Merge remote-tracking branch 'upstream/master' into templated-daemon-protocol

src/libexpr/common-eval-args.cc

@@ -29,7 +29,7 @@ MixEvalArgs::MixEvalArgs()
     addFlag({
         .longName = "include",
         .shortName = 'I',
-        .description = "add a path to the list of locations used to look up <...> file names",
+        .description = "add a path to the list of locations used to look up `<...>` file names",
         .labels = {"path"},
         .handler = {[&](std::string s) { searchPath.push_back(s); }}
     });

src/libexpr/eval-cache.cc

@@ -391,7 +391,8 @@ Value & AttrCursor::forceValue()
 
     if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
         if (v.type == tString)
-            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s};
+            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
+                           string_t{v.string.s, {}}};
         else if (v.type == tPath)
             cachedValue = {root->db->setString(getKey(), v.path), v.path};
         else if (v.type == tBool)

src/libexpr/eval.cc

@@ -87,6 +87,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
             else if (*i == '\n') str << "\\n";
             else if (*i == '\r') str << "\\r";
             else if (*i == '\t') str << "\\t";
+            else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
             else str << *i;
         str << "\"";
         break;
@@ -355,6 +356,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sEpsilon(symbols.create(""))
     , repair(NoRepair)
     , store(store)
+    , regexCache(makeRegexCache())
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
 {
@@ -369,7 +371,11 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
     }
-    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
+
+    try {
+        addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
+    } catch (Error &) {
+    }
 
     if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
@@ -381,10 +387,14 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
             auto path = r.second;
 
             if (store->isInStore(r.second)) {
-                StorePathSet closure;
-                store->computeFSClosure(store->toStorePath(r.second).first, closure);
-                for (auto & path : closure)
-                    allowedPaths->insert(store->printStorePath(path));
+                try {
+                    StorePathSet closure;
+                    store->computeFSClosure(store->toStorePath(r.second).first, closure);
+                    for (auto & path : closure)
+                        allowedPaths->insert(store->printStorePath(path));
+                } catch (InvalidPath &) {
+                    allowedPaths->insert(r.second);
+                }
             } else
                 allowedPaths->insert(r.second);
         }
@@ -509,7 +519,7 @@ Value * EvalState::addPrimOp(const string & name,
     if (arity == 0) {
         auto vPrimOp = allocValue();
         vPrimOp->type = tPrimOp;
-        vPrimOp->primOp = new PrimOp(primOp, 1, sym);
+        vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym };
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(name, v);
@@ -517,7 +527,7 @@ Value * EvalState::addPrimOp(const string & name,
 
     Value * v = allocValue();
     v->type = tPrimOp;
-    v->primOp = new PrimOp(primOp, arity, sym);
+    v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym };
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@@ -525,12 +535,59 @@ Value * EvalState::addPrimOp(const string & name,
 }
 
 
+Value * EvalState::addPrimOp(PrimOp && primOp)
+{
+    /* Hack to make constants lazy: turn them into a application of
+       the primop to a dummy value. */
+    if (primOp.arity == 0) {
+        primOp.arity = 1;
+        auto vPrimOp = allocValue();
+        vPrimOp->type = tPrimOp;
+        vPrimOp->primOp = new PrimOp(std::move(primOp));
+        Value v;
+        mkApp(v, *vPrimOp, *vPrimOp);
+        return addConstant(primOp.name, v);
+    }
+
+    Symbol envName = primOp.name;
+    if (hasPrefix(primOp.name, "__"))
+        primOp.name = symbols.create(std::string(primOp.name, 2));
+
+    Value * v = allocValue();
+    v->type = tPrimOp;
+    v->primOp = new PrimOp(std::move(primOp));
+    staticBaseEnv.vars[envName] = baseEnvDispl;
+    baseEnv.values[baseEnvDispl++] = v;
+    baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
+    return v;
+}
+
+
 Value & EvalState::getBuiltin(const string & name)
 {
     return *baseEnv.values[0]->attrs->find(symbols.create(name))->value;
 }
 
 
+std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
+{
+    if (v.type == tPrimOp || v.type == tPrimOpApp) {
+        auto v2 = &v;
+        while (v2->type == tPrimOpApp)
+            v2 = v2->primOpApp.left;
+        if (v2->primOp->doc)
+            return Doc {
+                .pos = noPos,
+                .name = v2->primOp->name,
+                .arity = v2->primOp->arity,
+                .args = v2->primOp->args,
+                .doc = v2->primOp->doc,
+            };
+    }
+    return {};
+}
+
+
 /* Every "format" object (even temporary) takes up a few hundred bytes
    of stack space, which is a real killer in the recursive
    evaluator.  So here are some helper functions for throwing
@@ -1297,14 +1354,25 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     }
 
     Value * actualArgs = allocValue();
-    mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size());
+    mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size()));
 
-    for (auto & i : fun.lambda.fun->formals->formals) {
-        Bindings::iterator j = args.find(i.name);
-        if (j != args.end())
-            actualArgs->attrs->push_back(*j);
-        else if (!i.def)
-            throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+    if (fun.lambda.fun->formals->ellipsis) {
+        // If the formals have an ellipsis (eg the function accepts extra args) pass
+        // all available automatic arguments (which includes arguments specified on
+        // the command line via --arg/--argstr)
+        for (auto& v : args) {
+            actualArgs->attrs->push_back(v);
+        }
+    } else {
+        // Otherwise, only pass the arguments that the function accepts
+        for (auto & i : fun.lambda.fun->formals->formals) {
+            Bindings::iterator j = args.find(i.name);
+            if (j != args.end()) {
+                actualArgs->attrs->push_back(*j);
+            } else if (!i.def) {
+                throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+            }
+        }
     }
 
     actualArgs->attrs->sort();

src/libexpr/eval.hh

@@ -6,7 +6,6 @@
 #include "symbol-table.hh"
 #include "config.hh"
 
-#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
@@ -30,8 +29,8 @@ struct PrimOp
     PrimOpFun fun;
     size_t arity;
     Symbol name;
-    PrimOp(PrimOpFun fun, size_t arity, Symbol name)
-        : fun(fun), arity(arity), name(name) { }
+    std::vector<std::string> args;
+    const char * doc = nullptr;
 };
 
 
@@ -65,6 +64,11 @@ typedef std::list<SearchPathElem> SearchPath;
 void initGC();
 
 
+struct RegexCache;
+
+std::shared_ptr<RegexCache> makeRegexCache();
+
+
 class EvalState
 {
 public:
@@ -120,7 +124,7 @@ private:
     std::unordered_map<Path, Path> resolvedPaths;
 
     /* Cache used by prim_match(). */
-    std::unordered_map<std::string, std::regex> regexCache;
+    std::shared_ptr<RegexCache> regexCache;
 
 public:
 
@@ -242,10 +246,23 @@ private:
     Value * addPrimOp(const string & name,
         size_t arity, PrimOpFun primOp);
 
+    Value * addPrimOp(PrimOp && primOp);
+
 public:
 
     Value & getBuiltin(const string & name);
 
+    struct Doc
+    {
+        Pos pos;
+        std::optional<Symbol> name;
+        size_t arity;
+        std::vector<std::string> args;
+        const char * doc;
+    };
+
+    std::optional<Doc> getDoc(Value & v);
+
 private:
 
     inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval);
@@ -357,24 +374,57 @@ struct EvalSettings : Config
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
-    Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path",
-        "List of directories to be searched for <...> file references."};
+    Setting<Strings> nixPath{
+        this, getDefaultNixPath(), "nix-path",
+        "List of directories to be searched for `<...>` file references."};
 
-    Setting<bool> restrictEval{this, false, "restrict-eval",
-        "Whether to restrict file system access to paths in $NIX_PATH, "
-        "and network access to the URI prefixes listed in 'allowed-uris'."};
+    Setting<bool> restrictEval{
+        this, false, "restrict-eval",
+        R"(
+          If set to `true`, the Nix evaluator will not allow access to any
+          files outside of the Nix search path (as set via the `NIX_PATH`
+          environment variable or the `-I` option), or to URIs outside of
+          `allowed-uri`. The default is `false`.
+        )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",
         "Whether to restrict file system and network access to files specified by cryptographic hash."};
 
-    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
-        "Whether the evaluator allows importing the result of a derivation."};
+    Setting<bool> enableImportFromDerivation{
+        this, true, "allow-import-from-derivation",
+        R"(
+          By default, Nix allows you to `import` from a derivation, allowing
+          building at evaluation time. With this option set to false, Nix will
+          throw an error when evaluating an expression that uses this feature,
+          allowing users to ensure their evaluation will not require any
+          builds to take place.
+        )"};
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
-        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+        R"(
+          A list of URI prefixes to which access is allowed in restricted
+          evaluation mode. For example, when set to
+          `https://github.com/NixOS`, builtin functions such as `fetchGit` are
+          allowed to access `https://github.com/NixOS/patchelf.git`.
+        )"};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
+        R"(
+          If set to `true`, the Nix evaluator will trace every function call.
+          Nix will print a log message at the "vomit" level for every function
+          entrance and function exit.
+
+              function-trace entered undefined position at 1565795816999559622
+              function-trace exited undefined position at 1565795816999581277
+              function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+              function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+
+          The `undefined position` means the function call is a builtin.
+
+          Use the `contrib/stack-collapse.py` script distributed with the Nix
+          source code to convert the trace logs in to a format suitable for
+          `flamegraph.pl`.
+        )"};
 
     Setting<bool> useEvalCache{this, true, "eval-cache",
         "Whether to use the flake evaluation cache."};

src/libexpr/flake/flakeref.cc

@@ -1,6 +1,7 @@
 #include "flakeref.hh"
 #include "store-api.hh"
 #include "url.hh"
+#include "url-parts.hh"
 #include "fetchers.hh"
 #include "registry.hh"
 

src/libexpr/flake/lockfile.cc

@@ -1,5 +1,6 @@
 #include "lockfile.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <nlohmann/json.hpp>
 

src/libexpr/flake/lockfile.hh

@@ -6,7 +6,7 @@
 
 namespace nix {
 class Store;
-struct StorePath;
+class StorePath;
 }
 
 namespace nix::flake {

src/libexpr/get-drvs.cc

@@ -38,8 +38,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
     auto i = drv.outputs.find(outputName);
     if (i == drv.outputs.end())
         throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
+    auto & [outputName, output] = *i;
 
-    outPath = store->printStorePath(i->second.path(*store, drv.name));
+    auto optStorePath = output.path(*store, drv.name, outputName);
+    if (optStorePath)
+        outPath = store->printStorePath(*optStorePath);
 }
 
 
@@ -77,12 +80,15 @@ string DrvInfo::queryDrvPath() const
 
 string DrvInfo::queryOutPath() const
 {
-    if (outPath == "" && attrs) {
+    if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
         PathSet context;
-        outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
+        if (i != attrs->end())
+            outPath = state->coerceToPath(*i->pos, *i->value, context);
     }
-    return outPath;
+    if (!outPath)
+        throw UnimplementedError("CA derivations are not yet supported");
+    return *outPath;
 }
 
 

src/libexpr/get-drvs.hh

@@ -20,7 +20,7 @@ private:
     mutable string name;
     mutable string system;
     mutable string drvPath;
-    mutable string outPath;
+    mutable std::optional<string> outPath;
     mutable string outputName;
     Outputs outputs;
 

src/libexpr/local.mk

@@ -42,6 +42,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh
 
 $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh

src/libexpr/primops.hh

@@ -10,9 +10,11 @@ struct RegisterPrimOp
     struct Info
     {
         std::string name;
-        size_t arity;
-        PrimOpFun primOp;
+        std::vector<std::string> args;
+        size_t arity = 0;
+        const char * doc;
         std::optional<std::string> requiredFeature;
+        PrimOpFun fun;
     };
 
     typedef std::vector<Info> PrimOps;
@@ -26,6 +28,8 @@ struct RegisterPrimOp
         size_t arity,
         PrimOpFun fun,
         std::optional<std::string> requiredFeature = {});
+
+    RegisterPrimOp(Info && info);
 };
 
 /* These primops are disabled without enableNativeCode, but plugins

src/libexpr/primops/derivation.nix

@@ -0,0 +1,27 @@
+/* This is the implementation of the ‘derivation’ builtin function.
+   It's actually a wrapper around the ‘derivationStrict’ primop. */
+
+drvAttrs @ { outputs ? [ "out" ], ... }:
+
+let
+
+  strict = derivationStrict drvAttrs;
+
+  commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) //
+    { all = map (x: x.value) outputsList;
+      inherit drvAttrs;
+    };
+
+  outputToAttrListElement = outputName:
+    { name = outputName;
+      value = commonAttrs // {
+        outPath = builtins.getAttr outputName strict;
+        drvPath = strict.drvPath;
+        type = "derivation";
+        inherit outputName;
+      };
+    };
+
+  outputsList = map outputToAttrListElement outputs;
+
+in (builtins.head outputsList).value

src/libexpr/primops/fetchMercurial.cc

@@ -3,8 +3,7 @@
 #include "store-api.hh"
 #include "fetchers.hh"
 #include "url.hh"
-
-#include <regex>
+#include "url-parts.hh"
 
 namespace nix {
 

src/libexpr/primops/fetchTree.cc

@@ -226,18 +226,187 @@ static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Va
     fetch(state, pos, args, v, "fetchurl", false, "");
 }
 
+static RegisterPrimOp primop_fetchurl({
+    .name = "__fetchurl",
+    .args = {"url"},
+    .doc = R"(
+      Download the specified URL and return the path of the downloaded
+      file. This function is not available if [restricted evaluation
+      mode](../command-ref/conf-file.md) is enabled.
+    )",
+    .fun = prim_fetchurl,
+});
+
 static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     fetch(state, pos, args, v, "fetchTarball", true, "source");
 }
 
+static RegisterPrimOp primop_fetchTarball({
+    .name = "fetchTarball",
+    .args = {"args"},
+    .doc = R"(
+      Download the specified URL, unpack it and return the path of the
+      unpacked tree. The file must be a tape archive (`.tar`) compressed
+      with `gzip`, `bzip2` or `xz`. The top-level path component of the
+      files in the tarball is removed, so it is best if the tarball
+      contains a single directory at top level. The typical use of the
+      function is to obtain external Nix expression dependencies, such as
+      a particular version of Nixpkgs, e.g.
+
+      ```nix
+      with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
+
+      stdenv.mkDerivation { … }
+      ```
+
+      The fetched tarball is cached for a certain amount of time (1 hour
+      by default) in `~/.cache/nix/tarballs/`. You can change the cache
+      timeout either on the command line with `--option tarball-ttl number
+      of seconds` or in the Nix configuration file with this option: ` 
+      number of seconds to cache `.
+
+      Note that when obtaining the hash with ` nix-prefetch-url ` the
+      option `--unpack` is required.
+
+      This function can also verify the contents against a hash. In that
+      case, the function takes a set instead of a URL. The set requires
+      the attribute `url` and the attribute `sha256`, e.g.
+
+      ```nix
+      with import (fetchTarball {
+        url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
+        sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
+      }) {};
+
+      stdenv.mkDerivation { … }
+      ```
+
+      This function is not available if [restricted evaluation
+      mode](../command-ref/conf-file.md) is enabled.
+    )",
+    .fun = prim_fetchTarball,
+});
+
 static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
 {
     fetchTree(state, pos, args, v, "git", true);
 }
 
-static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
-static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
-static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
+static RegisterPrimOp primop_fetchGit({
+    .name = "fetchGit",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a path from git. *args* can be a URL, in which case the HEAD
+      of the repo at that URL is fetched. Otherwise, it can be an
+      attribute with the following attributes (all except `url` optional):
+
+        - url  
+          The URL of the repo.
+
+        - name  
+          The name of the directory the repo should be exported to in the
+          store. Defaults to the basename of the URL.
+
+        - rev  
+          The git revision to fetch. Defaults to the tip of `ref`.
+
+        - ref  
+          The git ref to look for the requested revision under. This is
+          often a branch or tag name. Defaults to `HEAD`.
+
+          By default, the `ref` value is prefixed with `refs/heads/`. As
+          of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts
+          with `refs/`.
+
+        - submodules  
+          A Boolean parameter that specifies whether submodules should be
+          checked out. Defaults to `false`.
+
+      Here are some examples of how to use `fetchGit`.
+
+        - To fetch a private repository over SSH:
+
+          ```nix
+          builtins.fetchGit {
+            url = "git@github.com:my-secret/repository.git";
+            ref = "master";
+            rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
+          }
+          ```
+
+        - To fetch an arbitrary reference:
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/NixOS/nix.git";
+            ref = "refs/heads/0.5-release";
+          }
+          ```
+
+        - If the revision you're looking for is in the default branch of
+          the git repository you don't strictly need to specify the branch
+          name in the `ref` attribute.
+
+          However, if the revision you're looking for is in a future
+          branch for the non-default branch you will need to specify the
+          the `ref` attribute as well.
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+            ref = "1.11-maintenance";
+          }
+          ```
+
+          > **Note**
+          > 
+          > It is nice to always specify the branch which a revision
+          > belongs to. Without the branch being specified, the fetcher
+          > might fail if the default branch changes. Additionally, it can
+          > be confusing to try a commit from a non-default branch and see
+          > the fetch fail. If the branch is specified the fault is much
+          > more obvious.
+
+        - If the revision you're looking for is in the default branch of
+          the git repository you may omit the `ref` attribute.
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+          }
+          ```
+
+        - To fetch a specific tag:
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            ref = "refs/tags/1.9";
+          }
+          ```
+
+        - To fetch the latest version of a remote branch:
+
+          ```nix
+          builtins.fetchGit {
+            url = "ssh://git@github.com/nixos/nix.git";
+            ref = "master";
+          }
+          ```
+
+          > **Note**
+          > 
+          > Nix will refetch the branch in accordance with
+          > the option `tarball-ttl`.
+
+          > **Note**
+          > 
+          > This behavior is disabled in *Pure evaluation mode*.
+    )",
+    .fun = prim_fetchGit,
+});
 
 }

src/libfetchers/fetchers.hh

@@ -23,7 +23,7 @@ struct InputScheme;
 
 struct Input
 {
-    friend class InputScheme;
+    friend struct InputScheme;
 
     std::shared_ptr<InputScheme> scheme; // note: can be null
     Attrs attrs;

src/libfetchers/git.cc

@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <sys/time.h>
 

src/libfetchers/github.cc

@@ -3,14 +3,15 @@
 #include "fetchers.hh"
 #include "globals.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
 
-// A github or gitlab url
-const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
-std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
+// A github or gitlab host
+const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
+std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
 
 struct GitArchiveInputScheme : InputScheme
 {
@@ -50,9 +51,9 @@ struct GitArchiveInputScheme : InputScheme
                     throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
                 ref = value;
             }
-            else if (name == "url") {
-                if (!std::regex_match(value, urlRegex))
-                    throw BadURL("URL '%s' contains an invalid instance url", url.url);
+            else if (name == "host") {
+                if (!std::regex_match(value, hostRegex))
+                    throw BadURL("URL '%s' contains an invalid instance host", url.url);
                 host_url = value;
             }
             // FIXME: barf on unsupported attributes
@@ -67,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("repo", path[1]);
         if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
         if (ref) input.attrs.insert_or_assign("ref", *ref);
-        if (host_url) input.attrs.insert_or_assign("url", *host_url);
+        if (host_url) input.attrs.insert_or_assign("host", *host_url);
 
         return input;
     }
@@ -77,7 +78,7 @@ struct GitArchiveInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != type()) return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
                 throw Error("unsupported input attribute '%s'", name);
 
         getStrAttr(attrs, "owner");
@@ -182,11 +183,21 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
+    void addAccessToken(std::string & url) const
+    {
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            url += "?access_token=" + accessToken;
+    }
+
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
         auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
         auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+
+        addAccessToken(url);
+
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
@@ -200,14 +211,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
     {
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
         auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            url += "?access_token=" + accessToken;
+        addAccessToken(url);
 
         return url;
     }
@@ -228,7 +237,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
             host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
         auto json = nlohmann::json::parse(

src/libfetchers/indirect.cc

@@ -1,4 +1,5 @@
 #include "fetchers.hh"
+#include "url-parts.hh"
 
 namespace nix::fetchers {
 

src/libfetchers/mercurial.cc

@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <sys/time.h>
 

src/libfetchers/registry.cc

@@ -147,7 +147,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
         if (!hasPrefix(path, "/")) {
             auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true);
+                store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");
             path = store->toRealPath(storePath);
         }
 

src/libmain/common-args.cc

@@ -28,7 +28,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     addFlag({
         .longName = "option",
-        .description = "set a Nix configuration option (overriding nix.conf)",
+        .description = "set a Nix configuration option (overriding `nix.conf`)",
         .labels = {"name", "value"},
         .handler = {[](std::string name, std::string value) {
             try {
@@ -51,8 +51,8 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     addFlag({
         .longName = "log-format",
-        .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
-                        "or \"bar-with-logs\"",
+        .description = "format of log output; `raw`, `internal-json`, `bar` "
+                        "or `bar-with-logs`",
         .labels = {"format"},
         .handler = {[](std::string format) { setLogFormat(format); }},
     });

src/libmain/shared.cc

@@ -277,6 +277,8 @@ void printVersion(const string & programName)
 #if HAVE_SODIUM
         cfg.push_back("signed-caches");
 #endif
+        std::cout << "System type: " << settings.thisSystem << "\n";
+        std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
         std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
         std::cout << "User configuration files: " <<

src/libstore/binary-cache-store.cc

@@ -11,6 +11,7 @@
 #include "nar-accessor.hh"
 #include "json.hh"
 #include "thread-pool.hh"
+#include "callback.hh"
 
 #include <chrono>
 #include <future>
@@ -22,7 +23,8 @@
 namespace nix {
 
 BinaryCacheStore::BinaryCacheStore(const Params & params)
-    : Store(params)
+    : BinaryCacheStoreConfig(params)
+    , Store(params)
 {
     if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));

src/libstore/binary-cache-store.hh

@@ -11,17 +11,21 @@ namespace nix {
 
 struct NarInfo;
 
-class BinaryCacheStore : public Store
+struct BinaryCacheStoreConfig : virtual StoreConfig
 {
-public:
+    using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
-    const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
-    const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
-    const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
-    const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
-    const Setting<bool> parallelCompression{this, false, "parallel-compression",
+    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
+    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
+    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
+    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
+    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
+    const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
         "enable multi-threading compression, available for xz only currently"};
+};
+
+class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
+{
 
 private:
 
@@ -58,7 +62,7 @@ public:
 
 public:
 
-    virtual void init();
+    virtual void init() override;
 
 private:
 

src/libstore/daemon.cc

@@ -232,7 +232,7 @@ struct ClientSettings
                 else if (setSubstituters(settings.extraSubstituters))
                     ;
                 else
-                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
+                    debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
             } catch (UsageError & e) {
                 warn(e.what());
             }

src/libstore/derivations.cc

@@ -7,7 +7,7 @@
 
 namespace nix {
 
-std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
+std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
 {
     return std::visit(overloaded {
         [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
@@ -15,7 +15,7 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
         },
         [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
             return {
-                store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
+                dof.path(store, drvName, outputName)
             };
         },
         [](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
@@ -25,6 +25,13 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
 }
 
 
+StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
+    return store.makeFixedOutputPath(
+        hash.method, hash.hash,
+        outputPathName(drvName, outputName));
+}
+
+
 bool derivationIsCA(DerivationType dt) {
     switch (dt) {
     case DerivationType::InputAddressed: return false;
@@ -61,7 +68,7 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
-StorePath writeDerivation(ref<Store> store,
+StorePath writeDerivation(Store & store,
     const Derivation & drv, RepairFlag repair)
 {
     auto references = drv.inputSrcs;
@@ -71,10 +78,10 @@ StorePath writeDerivation(ref<Store> store,
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
     auto suffix = std::string(drv.name) + drvExtension;
-    auto contents = drv.unparse(*store, false);
+    auto contents = drv.unparse(store, false);
     return settings.readOnlyMode
-        ? store->computeStorePathForText(suffix, contents, references)
-        : store->addTextToStore(suffix, contents, references, repair);
+        ? store.computeStorePathForText(suffix, contents, references)
+        : store.addTextToStore(suffix, contents, references, repair);
 }
 
 
@@ -106,12 +113,15 @@ static string parseString(std::istream & str)
     return res;
 }
 
+static void validatePath(std::string_view s) {
+    if (s.size() == 0 || s[0] != '/')
+        throw FormatError("bad path '%1%' in derivation", s);
+}
 
 static Path parsePath(std::istream & str)
 {
-    string s = parseString(str);
-    if (s.size() == 0 || s[0] != '/')
-        throw FormatError("bad path '%1%' in derivation", s);
+    auto s = parseString(str);
+    validatePath(s);
     return s;
 }
 
@@ -140,7 +150,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 
 static DerivationOutput parseDerivationOutput(const Store & store,
-    StorePath path, std::string_view hashAlgo, std::string_view hash)
+    std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
 {
     if (hashAlgo != "") {
         auto method = FileIngestionMethod::Flat;
@@ -148,44 +158,49 @@ static DerivationOutput parseDerivationOutput(const Store & store,
             method = FileIngestionMethod::Recursive;
             hashAlgo = hashAlgo.substr(2);
         }
-        const HashType hashType = parseHashType(hashAlgo);
-
-        return hash != ""
-            ? DerivationOutput {
-                  .output = DerivationOutputCAFixed {
-                      .hash = FixedOutputHash {
-                          .method = std::move(method),
-                          .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
-                      },
-                  }
-               }
-            : (settings.requireExperimentalFeature("ca-derivations"),
-              DerivationOutput {
-                  .output =  DerivationOutputCAFloating {
-                      .method = std::move(method),
-                      .hashType = std::move(hashType),
-                  },
-              });
-    } else
+        const auto hashType = parseHashType(hashAlgo);
+        if (hash != "") {
+            validatePath(pathS);
+            return DerivationOutput {
+                .output = DerivationOutputCAFixed {
+                    .hash = FixedOutputHash {
+                        .method = std::move(method),
+                        .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
+                    },
+                },
+            };
+        } else {
+            settings.requireExperimentalFeature("ca-derivations");
+            assert(pathS == "");
+            return DerivationOutput {
+                .output = DerivationOutputCAFloating {
+                    .method = std::move(method),
+                    .hashType = std::move(hashType),
+                },
+            };
+        }
+    } else {
+        validatePath(pathS);
         return DerivationOutput {
             .output = DerivationOutputInputAddressed {
-                .path = std::move(path),
+                .path = store.parseStorePath(pathS),
             }
         };
+    }
 }
 
 static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
 {
-    expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+    expect(str, ","); const auto pathS = parseString(str);
     expect(str, ","); const auto hashAlgo = parseString(str);
     expect(str, ","); const auto hash = parseString(str);
     expect(str, ")");
 
-    return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
+    return parseDerivationOutput(store, pathS, hashAlgo, hash);
 }
 
 
-static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
+Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
 {
     Derivation drv;
     drv.name = name;
@@ -233,34 +248,6 @@ static Derivation parseDerivation(const Store & store, std::string && s, std::st
 }
 
 
-Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name)
-{
-    try {
-        return parseDerivation(store, readFile(drvPath), name);
-    } catch (FormatError & e) {
-        throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
-    }
-}
-
-
-Derivation Store::derivationFromPath(const StorePath & drvPath)
-{
-    ensurePath(drvPath);
-    return readDerivation(drvPath);
-}
-
-
-Derivation Store::readDerivation(const StorePath & drvPath)
-{
-    auto accessor = getFSAccessor();
-    try {
-        return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath));
-    } catch (FormatError & e) {
-        throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
-    }
-}
-
-
 static void printString(string & res, std::string_view s)
 {
     char buf[s.size() * 2 + 2];
@@ -322,17 +309,19 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     for (auto & i : outputs) {
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
-        s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
         std::visit(overloaded {
             [&](DerivationOutputInputAddressed doi) {
+                s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
             },
             [&](DerivationOutputCAFixed dof) {
+                s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
                 s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
                 s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
             },
             [&](DerivationOutputCAFloating dof) {
+                s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
                 s += ','; printUnquotedString(s, "");
             },
@@ -388,6 +377,16 @@ bool isDerivation(const string & fileName)
 }
 
 
+std::string outputPathName(std::string_view drvName, std::string_view outputName) {
+    std::string res { drvName };
+    if (outputName != "out") {
+        res += "-";
+        res += outputName;
+    }
+    return res;
+}
+
+
 DerivationType BasicDerivation::type() const
 {
     std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
@@ -480,12 +479,12 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
         throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
     case DerivationType::CAFixed: {
         std::map<std::string, Hash> outputHashes;
-        for (const auto & i : drv.outputsAndPaths(store)) {
-            auto & dof = std::get<DerivationOutputCAFixed>(i.second.first.output);
+        for (const auto & i : drv.outputs) {
+            auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
             auto hash = hashString(htSHA256, "fixed:out:"
                 + dof.hash.printMethodAlgo() + ":"
                 + dof.hash.hash.to_string(Base16, false) + ":"
-                + store.printStorePath(i.second.second));
+                + store.printStorePath(dof.path(store, drv.name, i.first)));
             outputHashes.insert_or_assign(i.first, std::move(hash));
         }
         return outputHashes;
@@ -536,21 +535,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
 }
 
 
-StorePathSet BasicDerivation::outputPaths(const Store & store) const
-{
-    StorePathSet paths;
-    for (auto & i : outputsAndPaths(store))
-        paths.insert(i.second.second);
-    return paths;
-}
-
 static DerivationOutput readDerivationOutput(Source & in, const Store & store)
 {
-    auto path = store.parseStorePath(readString(in));
+    const auto pathS = readString(in);
     const auto hashAlgo = readString(in);
     const auto hash = readString(in);
 
-    return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
+    return parseDerivationOutput(store, pathS, hashAlgo, hash);
 }
 
 StringSet BasicDerivation::outputNames() const
@@ -561,23 +552,12 @@ StringSet BasicDerivation::outputNames() const
     return names;
 }
 
-DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const {
-    DerivationOutputsAndPaths outsAndPaths;
-    for (auto output : outputs)
-        outsAndPaths.insert(std::make_pair(
-            output.first,
-            std::make_pair(output.second, output.second.path(store, name))
-            )
-        );
-    return outsAndPaths;
-}
-
 DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
     DerivationOutputsAndOptPaths outsAndOptPaths;
     for (auto output : outputs)
         outsAndOptPaths.insert(std::make_pair(
             output.first,
-            std::make_pair(output.second, output.second.pathOpt(store, output.first))
+            std::make_pair(output.second, output.second.path(store, name, output.first))
             )
         );
     return outsAndOptPaths;
@@ -622,22 +602,25 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
 void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
-    for (auto & i : drv.outputsAndPaths(store)) {
-        out << i.first
-            << store.printStorePath(i.second.second);
+    for (auto & i : drv.outputs) {
+        out << i.first;
         std::visit(overloaded {
             [&](DerivationOutputInputAddressed doi) {
-                out << "" << "";
+                out << store.printStorePath(doi.path)
+                    << ""
+                    << "";
             },
             [&](DerivationOutputCAFixed dof) {
-                out << dof.hash.printMethodAlgo()
+                out << store.printStorePath(dof.path(store, drv.name, i.first))
+                    << dof.hash.printMethodAlgo()
                     << dof.hash.hash.to_string(Base16, false);
             },
             [&](DerivationOutputCAFloating dof) {
-                out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
+                out << ""
+                    << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
                     << "";
             },
-        }, i.second.first.output);
+        }, i.second.output);
     }
     WorkerProto<StorePathSet>::write(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
@@ -653,5 +636,12 @@ std::string hashPlaceholder(const std::string & outputName)
     return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
 }
 
+std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
+{
+    auto drvNameWithExtension = drvPath.name();
+    auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
+    auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
+    return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
+}
 
 }

src/libstore/derivations.hh

@@ -27,6 +27,7 @@ struct DerivationOutputInputAddressed
 struct DerivationOutputCAFixed
 {
     FixedOutputHash hash; /* hash used for expected hash computation */
+    StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 };
 
 /* Floating-output derivations, whose output paths are content addressed, but
@@ -49,14 +50,8 @@ struct DerivationOutput
     std::optional<HashType> hashAlgoOpt(const Store & store) const;
     /* Note, when you use this function you should make sure that you're passing
        the right derivation name. When in doubt, you should use the safer
-       interface provided by BasicDerivation::outputsAndPaths */
-    std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
-    /* DEPRECATED: Remove after CA drvs are fully implemented */
-    StorePath path(const Store & store, std::string_view drvName) const {
-        auto p = pathOpt(store, drvName);
-        if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
-        return *p;
-    }
+       interface provided by BasicDerivation::outputsAndOptPaths */
+    std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -113,17 +108,12 @@ struct BasicDerivation
     /* Return true iff this is a fixed-output derivation. */
     DerivationType type() const;
 
-    /* Return the output paths of a derivation. */
-    StorePathSet outputPaths(const Store & store) const;
-
     /* Return the output names of a derivation. */
     StringSet outputNames() const;
 
     /* Calculates the maps that contains all the DerivationOutputs, but
-       augmented with knowledge of the Store paths they would be written into.
-       The first one of these functions will be removed when the CA work is
-       completed */
-    DerivationOutputsAndPaths outputsAndPaths(const Store & store) const;
+       augmented with knowledge of the Store paths they would be written
+       into. */
     DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
 
     static std::string_view nameFromPath(const StorePath & storePath);
@@ -146,15 +136,22 @@ class Store;
 enum RepairFlag : bool { NoRepair = false, Repair = true };
 
 /* Write a derivation to the Nix store, and return its path. */
-StorePath writeDerivation(ref<Store> store,
+StorePath writeDerivation(Store & store,
     const Derivation & drv, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
-Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
+Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
 
 // FIXME: remove
 bool isDerivation(const string & fileName);
 
+/* Calculate the name that will be used for the store path for this
+   output.
+
+   This is usually <drv-name>-<output-name>, but is just <drv-name> when
+   the output name is "out". */
+std::string outputPathName(std::string_view drvName, std::string_view outputName);
+
 // known CA drv's output hashes, current just for fixed-output derivations
 // whose output hashes are always known since they are fixed up-front.
 typedef std::map<std::string, Hash> CaOutputHashes;
@@ -202,6 +199,21 @@ struct Sink;
 Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
 void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
 
+/* This creates an opaque and almost certainly unique string
+   deterministically from the output name.
+
+   It is used as a placeholder to allow derivations to refer to their
+   own outputs without needing to use the hash of a derivation in
+   itself, making the hash near-impossible to calculate. */
 std::string hashPlaceholder(const std::string & outputName);
 
+/* This creates an opaque and almost certainly unique string
+   deterministically from a derivation path and output name.
+
+   It is used as a placeholder to allow derivations to refer to
+   content-addressed paths whose content --- and thus the path
+   themselves --- isn't yet known. This occurs when a derivation has a
+   dependency which is a CA derivation. */
+std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
+
 }

src/libstore/dummy-store.cc

@@ -0,0 +1,68 @@
+#include "store-api.hh"
+#include "callback.hh"
+
+namespace nix {
+
+struct DummyStoreConfig : virtual StoreConfig {
+    using StoreConfig::StoreConfig;
+
+    const std::string name() override { return "Dummy Store"; }
+};
+
+struct DummyStore : public Store, public virtual DummyStoreConfig
+{
+    DummyStore(const std::string scheme, const std::string uri, const Params & params)
+        : DummyStore(params)
+    { }
+
+    DummyStore(const Params & params)
+        : StoreConfig(params)
+        , Store(params)
+    {
+    }
+
+    string getUri() override
+    {
+        return *uriSchemes().begin();
+    }
+
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
+    {
+        callback(nullptr);
+    }
+
+    static std::set<std::string> uriSchemes() {
+        return {"dummy"};
+    }
+
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
+    { unsupported("queryPathFromHashPart"); }
+
+    void addToStore(const ValidPathInfo & info, Source & source,
+        RepairFlag repair, CheckSigsFlag checkSigs) override
+    { unsupported("addToStore"); }
+
+    StorePath addToStore(const string & name, const Path & srcPath,
+        FileIngestionMethod method, HashType hashAlgo,
+        PathFilter & filter, RepairFlag repair) override
+    { unsupported("addToStore"); }
+
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override
+    { unsupported("addTextToStore"); }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    { unsupported("narFromPath"); }
+
+    void ensurePath(const StorePath & path) override
+    { unsupported("ensurePath"); }
+
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode) override
+    { unsupported("buildDerivation"); }
+};
+
+static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regStore;
+
+}

src/libstore/filetransfer.cc

@@ -5,6 +5,7 @@
 #include "s3.hh"
 #include "compression.hh"
 #include "finally.hh"
+#include "callback.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>

src/libstore/filetransfer.hh

@@ -17,15 +17,30 @@ struct FileTransferSettings : Config
     Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
         "String appended to the user agent in HTTP requests."};
 
-    Setting<size_t> httpConnections{this, 25, "http-connections",
-        "Number of parallel HTTP connections.",
+    Setting<size_t> httpConnections{
+        this, 25, "http-connections",
+        R"(
+          The maximum number of parallel TCP connections used to fetch
+          files from binary caches and by other downloads. It defaults
+          to 25. 0 means no limit.
+        )",
         {"binary-caches-parallel-connections"}};
 
-    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
-        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
+    Setting<unsigned long> connectTimeout{
+        this, 0, "connect-timeout",
+        R"(
+          The timeout (in seconds) for establishing connections in the
+          binary cache substituter. It corresponds to `curl`’s
+          `--connect-timeout` option.
+        )"};
 
-    Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
-        "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
+    Setting<unsigned long> stalledDownloadTimeout{
+        this, 300, "stalled-download-timeout",
+        R"(
+          The timeout (in seconds) for receiving data from servers
+          during download. Nix cancels idle downloads after this
+          timeout's duration.
+        )"};
 
     Setting<unsigned int> tries{this, 5, "download-attempts",
         "How often Nix will attempt to download a file before giving up."};

src/libstore/gc.cc

@@ -85,8 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path)
 }
 
 
-Path LocalFSStore::addPermRoot(const StorePath & storePath,
-    const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
+Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
 {
     Path gcRoot(canonPath(_gcRoot));
 
@@ -95,47 +94,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
                 "creating a garbage collector root (%1%) in the Nix store is forbidden "
                 "(are you running nix-build inside the store?)", gcRoot);
 
-    if (indirect) {
-        /* Don't clobber the link if it already exists and doesn't
-           point to the Nix store. */
-        if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
-            throw Error("cannot create symlink '%1%'; already exists", gcRoot);
-        makeSymlink(gcRoot, printStorePath(storePath));
-        addIndirectRoot(gcRoot);
-    }
-
-    else {
-        if (!allowOutsideRootsDir) {
-            Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
-
-            if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
-                throw Error(
-                    "path '%1%' is not a valid garbage collector root; "
-                    "it's not in the directory '%2%'",
-                    gcRoot, rootsDir);
-        }
-
-        if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
-            writeFile(gcRoot, "");
-        else
-            makeSymlink(gcRoot, printStorePath(storePath));
-    }
-
-    /* Check that the root can be found by the garbage collector.
-       !!! This can be very slow on machines that have many roots.
-       Instead of reading all the roots, it would be more efficient to
-       check if the root is in a directory in or linked from the
-       gcroots directory. */
-    if (settings.checkRootReachability) {
-        auto roots = findRoots(false);
-        if (roots[storePath].count(gcRoot) == 0)
-            logWarning({
-                .name = "GC root",
-                .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
-                    "therefore, '%2%' might be removed by the garbage collector",
-                    gcRoot, printStorePath(storePath))
-            });
-    }
+    /* Don't clobber the link if it already exists and doesn't
+       point to the Nix store. */
+    if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
+        throw Error("cannot create symlink '%1%'; already exists", gcRoot);
+    makeSymlink(gcRoot, printStorePath(storePath));
+    addIndirectRoot(gcRoot);
 
     /* Grab the global GC root, causing us to block while a GC is in
        progress.  This prevents the set of permanent roots from
@@ -610,9 +574,12 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
     /* If keep-derivations is set and this is a derivation, then
        don't delete the derivation if any of the outputs are alive. */
     if (state.gcKeepDerivations && path.isDerivation()) {
-        for (auto & i : queryDerivationOutputs(path))
-            if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                incoming.insert(i);
+        for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
+            if (maybeOutPath &&
+                isValidPath(*maybeOutPath) &&
+                queryPathInfo(*maybeOutPath)->deriver == path
+                )
+                incoming.insert(*maybeOutPath);
     }
 
     /* If keep-outputs is set, then don't delete this path if there

src/libstore/globals.cc

@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "archive.hh"
 #include "args.hh"
+#include "abstract-setting-to-json.hh"
 
 #include <algorithm>
 #include <map>
@@ -9,6 +10,8 @@
 #include <dlfcn.h>
 #include <sys/utsname.h>
 
+#include <nlohmann/json.hpp>
+
 
 namespace nix {
 
@@ -160,11 +163,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const
     else abort();
 }
 
-template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
-{
-    AbstractSetting::toJSON(out);
-}
-
 template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
 {
     args.addFlag({

src/libstore/globals.hh

@@ -80,89 +80,209 @@ public:
     Setting<bool> keepGoing{this, false, "keep-going",
         "Whether to keep building derivations when another build fails."};
 
-    Setting<bool> tryFallback{this, false, "fallback",
-        "Whether to fall back to building when substitution fails.",
+    Setting<bool> tryFallback{
+        this, false, "fallback",
+        R"(
+          If set to `true`, Nix will fall back to building from source if a
+          binary substitute fails. This is equivalent to the `--fallback`
+          flag. The default is `false`.
+        )",
         {"build-fallback"}};
 
     /* Whether to show build log output in real time. */
     bool verboseBuild = true;
 
     Setting<size_t> logLines{this, 10, "log-lines",
-        "If verbose-build is false, the number of lines of the tail of "
+        "If `verbose-build` is false, the number of lines of the tail of "
         "the log to show if a build fails."};
 
-    MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
-        "Maximum number of parallel build jobs. \"auto\" means use number of cores.",
+    MaxBuildJobsSetting maxBuildJobs{
+        this, 1, "max-jobs",
+        R"(
+          This option defines the maximum number of jobs that Nix will try to
+          build in parallel. The default is `1`. The special value `auto`
+          causes Nix to use the number of CPUs in your system. `0` is useful
+          when using remote builders to prevent any local builds (except for
+          `preferLocalBuild` derivation attribute which executes locally
+          regardless). It can be overridden using the `--max-jobs` (`-j`)
+          command line switch.
+        )",
         {"build-max-jobs"}};
 
-    Setting<unsigned int> buildCores{this, getDefaultCores(), "cores",
-        "Number of CPU cores to utilize in parallel within a build, "
-        "i.e. by passing this number to Make via '-j'. 0 means that the "
-        "number of actual CPU cores on the local host ought to be "
-        "auto-detected.", {"build-cores"}};
+    Setting<unsigned int> buildCores{
+        this, getDefaultCores(), "cores",
+        R"(
+          Sets the value of the `NIX_BUILD_CORES` environment variable in the
+          invocation of builders. Builders can use this variable at their
+          discretion to control the maximum amount of parallelism. For
+          instance, in Nixpkgs, if the derivation attribute
+          `enableParallelBuilding` is set to `true`, the builder passes the
+          `-jN` flag to GNU Make. It can be overridden using the `--cores`
+          command line switch and defaults to `1`. The value `0` means that
+          the builder should use all available CPU cores in the system.
+        )",
+        {"build-cores"}};
 
     /* Read-only mode.  Don't copy stuff to the store, don't change
        the database. */
     bool readOnlyMode = false;
 
-    Setting<std::string> thisSystem{this, SYSTEM, "system",
-        "The canonical Nix system name."};
+    Setting<std::string> thisSystem{
+        this, SYSTEM, "system",
+        R"(
+          This option specifies the canonical Nix system name of the current
+          installation, such as `i686-linux` or `x86_64-darwin`. Nix can only
+          build derivations whose `system` attribute equals the value
+          specified here. In general, it never makes sense to modify this
+          value from its default, since you can use it to ‘lie’ about the
+          platform you are building on (e.g., perform a Mac OS build on a
+          Linux machine; the result would obviously be wrong). It only makes
+          sense if the Nix binaries can run on multiple platforms, e.g.,
+          ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`.
 
-    Setting<time_t> maxSilentTime{this, 0, "max-silent-time",
-        "The maximum time in seconds that a builer can go without "
-        "producing any output on stdout/stderr before it is killed. "
-        "0 means infinity.",
+          It defaults to the canonical Nix system name detected by `configure`
+          at build time.
+        )"};
+
+    Setting<time_t> maxSilentTime{
+        this, 0, "max-silent-time",
+        R"(
+          This option defines the maximum number of seconds that a builder can
+          go without producing any data on standard output or standard error.
+          This is useful (for instance in an automated build system) to catch
+          builds that are stuck in an infinite loop, or to catch remote builds
+          that are hanging due to network problems. It can be overridden using
+          the `--max-silent-time` command line switch.
+
+          The value `0` means that there is no timeout. This is also the
+          default.
+        )",
         {"build-max-silent-time"}};
 
-    Setting<time_t> buildTimeout{this, 0, "timeout",
-        "The maximum duration in seconds that a builder can run. "
-        "0 means infinity.", {"build-timeout"}};
+    Setting<time_t> buildTimeout{
+        this, 0, "timeout",
+        R"(
+          This option defines the maximum number of seconds that a builder can
+          run. This is useful (for instance in an automated build system) to
+          catch builds that are stuck in an infinite loop but keep writing to
+          their standard output or standard error. It can be overridden using
+          the `--timeout` command line switch.
+
+          The value `0` means that there is no timeout. This is also the
+          default.
+        )",
+        {"build-timeout"}};
 
     PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
         "The path of the helper program that executes builds to remote machines."};
 
-    Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
-        "A semicolon-separated list of build machines, in the format of nix.machines."};
+    Setting<std::string> builders{
+        this, "@" + nixConfDir + "/machines", "builders",
+        "A semicolon-separated list of build machines, in the format of `nix.machines`."};
 
-    Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
-        "Whether build machines should use their own substitutes for obtaining "
-        "build dependencies if possible, rather than waiting for this host to "
-        "upload them."};
+    Setting<bool> buildersUseSubstitutes{
+        this, false, "builders-use-substitutes",
+        R"(
+          If set to `true`, Nix will instruct remote build machines to use
+          their own binary substitutes if available. In practical terms, this
+          means that remote hosts will fetch as many build dependencies as
+          possible from their own substitutes (e.g, from `cache.nixos.org`),
+          instead of waiting for this host to upload them all. This can
+          drastically reduce build times if the network connection between
+          this computer and the remote build host is slow.
+        )"};
 
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
-    Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
-        "Whether SQLite should use fsync()."};
+    Setting<bool> fsyncMetadata{
+        this, true, "fsync-metadata",
+        R"(
+          If set to `true`, changes to the Nix store metadata (in
+          `/nix/var/nix/db`) are synchronously flushed to disk. This improves
+          robustness in case of system crashes, but reduces performance. The
+          default is `true`.
+        )"};
 
     Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal",
         "Whether SQLite should use WAL mode."};
 
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
-        "Whether to call sync() before registering a path as valid."};
+        "Whether to call `sync()` before registering a path as valid."};
 
-    Setting<bool> useSubstitutes{this, true, "substitute",
-        "Whether to use substitutes.",
+    Setting<bool> useSubstitutes{
+        this, true, "substitute",
+        R"(
+          If set to `true` (default), Nix will use binary substitutes if
+          available. This option can be disabled to force building from
+          source.
+        )",
         {"build-use-substitutes"}};
 
-    Setting<std::string> buildUsersGroup{this, "", "build-users-group",
-        "The Unix group that contains the build users."};
+    Setting<std::string> buildUsersGroup{
+        this, "", "build-users-group",
+        R"(
+          This options specifies the Unix group containing the Nix build user
+          accounts. In multi-user Nix installations, builds should not be
+          performed by the Nix account since that would allow users to
+          arbitrarily modify the Nix store and database by supplying specially
+          crafted builders; and they cannot be performed by the calling user
+          since that would allow him/her to influence the build result.
+
+          Therefore, if this option is non-empty and specifies a valid group,
+          builds will be performed under the user accounts that are a member
+          of the group specified here (as listed in `/etc/group`). Those user
+          accounts should not be used for any other purpose\!
+
+          Nix will never run two builds under the same user account at the
+          same time. This is to prevent an obvious security hole: a malicious
+          user writing a Nix expression that modifies the build result of a
+          legitimate Nix expression being built by another user. Therefore it
+          is good to have as many Nix build user accounts as you can spare.
+          (Remember: uids are cheap.)
+
+          The build users should have permission to create files in the Nix
+          store, but not delete them. Therefore, `/nix/store` should be owned
+          by the Nix account, its group should be the group specified here,
+          and its mode should be `1775`.
+
+          If the build users group is empty, builds will be performed under
+          the uid of the Nix process (that is, the uid of the caller if
+          `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
+          `NIX_REMOTE` is `daemon`). Obviously, this should not be used in
+          multi-user settings with untrusted users.
+        )"};
 
     Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
         "Whether to impersonate a Linux 2.6 machine on newer kernels.",
         {"build-impersonate-linux-26"}};
 
-    Setting<bool> keepLog{this, true, "keep-build-log",
-        "Whether to store build logs.",
+    Setting<bool> keepLog{
+        this, true, "keep-build-log",
+        R"(
+          If set to `true` (the default), Nix will write the build log of a
+          derivation (i.e. the standard output and error of its builder) to
+          the directory `/nix/var/log/nix/drvs`. The build log can be
+          retrieved using the command `nix-store -l path`.
+        )",
         {"build-keep-log"}};
 
-    Setting<bool> compressLog{this, true, "compress-build-log",
-        "Whether to compress logs.",
+    Setting<bool> compressLog{
+        this, true, "compress-build-log",
+        R"(
+          If set to `true` (the default), build logs written to
+          `/nix/var/log/nix/drvs` will be compressed on the fly using bzip2.
+          Otherwise, they will not be compressed.
+        )",
         {"build-compress-log"}};
 
-    Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size",
-        "Maximum number of bytes a builder can write to stdout/stderr "
-        "before being killed (0 means no limit).",
+    Setting<unsigned long> maxLogSize{
+        this, 0, "max-build-log-size",
+        R"(
+          This option defines the maximum number of bytes that a builder can
+          write to its stdout/stderr. If the builder exceeds this limit, it’s
+          killed. A value of `0` (the default) means that there is no limit.
+        )",
         {"build-max-log-size"}};
 
     /* When buildRepeat > 0 and verboseBuild == true, whether to print
@@ -173,57 +293,156 @@ public:
     Setting<unsigned int> pollInterval{this, 5, "build-poll-interval",
         "How often (in seconds) to poll for locks."};
 
-    Setting<bool> checkRootReachability{this, false, "gc-check-reachability",
-        "Whether to check if new GC roots can in fact be found by the "
-        "garbage collector."};
+    Setting<bool> gcKeepOutputs{
+        this, false, "keep-outputs",
+        R"(
+          If `true`, the garbage collector will keep the outputs of
+          non-garbage derivations. If `false` (default), outputs will be
+          deleted unless they are GC roots themselves (or reachable from other
+          roots).
 
-    Setting<bool> gcKeepOutputs{this, false, "keep-outputs",
-        "Whether the garbage collector should keep outputs of live derivations.",
+          In general, outputs must be registered as roots separately. However,
+          even if the output of a derivation is registered as a root, the
+          collector will still delete store paths that are used only at build
+          time (e.g., the C compiler, or source tarballs downloaded from the
+          network). To prevent it from doing so, set this option to `true`.
+        )",
         {"gc-keep-outputs"}};
 
-    Setting<bool> gcKeepDerivations{this, true, "keep-derivations",
-        "Whether the garbage collector should keep derivers of live paths.",
+    Setting<bool> gcKeepDerivations{
+        this, true, "keep-derivations",
+        R"(
+          If `true` (default), the garbage collector will keep the derivations
+          from which non-garbage store paths were built. If `false`, they will
+          be deleted unless explicitly registered as a root (or reachable from
+          other roots).
+
+          Keeping derivation around is useful for querying and traceability
+          (e.g., it allows you to ask with what dependencies or options a
+          store path was built), so by default this option is on. Turn it off
+          to save a bit of disk space (or a lot if `keep-outputs` is also
+          turned on).
+        )",
         {"gc-keep-derivations"}};
 
-    Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
-        "Whether to automatically replace files with identical contents with hard links."};
+    Setting<bool> autoOptimiseStore{
+        this, false, "auto-optimise-store",
+        R"(
+          If set to `true`, Nix automatically detects files in the store
+          that have identical contents, and replaces them with hard links to
+          a single copy. This saves disk space. If set to `false` (the
+          default), you can still run `nix-store --optimise` to get rid of
+          duplicate files.
+        )"};
 
-    Setting<bool> envKeepDerivations{this, false, "keep-env-derivations",
-        "Whether to add derivations as a dependency of user environments "
-        "(to prevent them from being GCed).",
+    Setting<bool> envKeepDerivations{
+        this, false, "keep-env-derivations",
+        R"(
+          If `false` (default), derivations are not stored in Nix user
+          environments. That is, the derivations of any build-time-only
+          dependencies may be garbage-collected.
+
+          If `true`, when you add a Nix derivation to a user environment, the
+          path of the derivation is stored in the user environment. Thus, the
+          derivation will not be garbage-collected until the user environment
+          generation is deleted (`nix-env --delete-generations`). To prevent
+          build-time-only dependencies from being collected, you should also
+          turn on `keep-outputs`.
+
+          The difference between this option and `keep-derivations` is that
+          this one is “sticky”: it applies to any user environment created
+          while this option was enabled, while `keep-derivations` only applies
+          at the moment the garbage collector is run.
+        )",
         {"env-keep-derivations"}};
 
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
 
-    Setting<SandboxMode> sandboxMode{this,
+    Setting<SandboxMode> sandboxMode{
+        this,
         #if __linux__
           smEnabled
         #else
           smDisabled
         #endif
         , "sandbox",
-        "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
+        R"(
+          If set to `true`, builds will be performed in a *sandboxed
+          environment*, i.e., they’re isolated from the normal file system
+          hierarchy and will only see their dependencies in the Nix store,
+          the temporary build directory, private versions of `/proc`,
+          `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths
+          configured with the `sandbox-paths` option. This is useful to
+          prevent undeclared dependencies on files in directories such as
+          `/usr/bin`. In addition, on Linux, builds run in private PID,
+          mount, network, IPC and UTS namespaces to isolate them from other
+          processes in the system (except that fixed-output derivations do
+          not run in private network namespace to ensure they can access the
+          network).
+
+          Currently, sandboxing only work on Linux and macOS. The use of a
+          sandbox requires that Nix is run as root (so you should use the
+          “build users” feature to perform the actual builds under different
+          users than root).
+
+          If this option is set to `relaxed`, then fixed-output derivations
+          and derivations that have the `__noChroot` attribute set to `true`
+          do not run in sandboxes.
+
+          The default is `true` on Linux and `false` on all other platforms.
+        )",
         {"build-use-chroot", "build-use-sandbox"}};
 
-    Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths",
-        "The paths to make available inside the build sandbox.",
+    Setting<PathSet> sandboxPaths{
+        this, {}, "sandbox-paths",
+        R"(
+          A list of paths bind-mounted into Nix sandbox environments. You can
+          use the syntax `target=source` to mount a path in a different
+          location in the sandbox; for instance, `/bin=/nix-bin` will mount
+          the path `/nix-bin` as `/bin` inside the sandbox. If *source* is
+          followed by `?`, then it is not an error if *source* does not exist;
+          for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will
+          only be mounted in the sandbox if it exists in the host filesystem.
+
+          Depending on how Nix was built, the default value for this option
+          may be empty or provide `/bin/sh` as a bind-mount of `bash`.
+        )",
         {"build-chroot-dirs", "build-sandbox-paths"}};
 
     Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
         "Whether to disable sandboxing when the kernel doesn't allow it."};
 
-    Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
-        "Additional paths to make available inside the build sandbox.",
+    Setting<PathSet> extraSandboxPaths{
+        this, {}, "extra-sandbox-paths",
+        R"(
+          A list of additional paths appended to `sandbox-paths`. Useful if
+          you want to extend its default value.
+        )",
         {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
 
-    Setting<size_t> buildRepeat{this, 0, "repeat",
-        "The number of times to repeat a build in order to verify determinism.",
+    Setting<size_t> buildRepeat{
+        this, 0, "repeat",
+        R"(
+          How many times to repeat builds to check whether they are
+          deterministic. The default value is 0. If the value is non-zero,
+          every build is repeated the specified number of times. If the
+          contents of any of the runs differs from the previous ones and
+          `enforce-determinism` is true, the build is rejected and the
+          resulting store paths are not registered as “valid” in Nix’s
+          database.
+        )",
         {"build-repeat"}};
 
 #if __linux__
-    Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size",
-        "The size of /dev/shm in the build sandbox."};
+    Setting<std::string> sandboxShmSize{
+        this, "50%", "sandbox-dev-shm-size",
+        R"(
+          This option determines the maximum size of the `tmpfs` filesystem
+          mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+          description of the `size` option of `tmpfs` in mount8. The default
+          is `50%`.
+        )"};
 
     Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
         "The build directory inside the sandbox."};
@@ -237,121 +456,411 @@ public:
         "Whether to log Darwin sandbox access violations to the system log."};
 #endif
 
-    Setting<bool> runDiffHook{this, false, "run-diff-hook",
-        "Whether to run the program specified by the diff-hook setting "
-        "repeated builds produce a different result. Typically used to "
-        "plug in diffoscope."};
+    Setting<bool> runDiffHook{
+        this, false, "run-diff-hook",
+        R"(
+          If true, enable the execution of the `diff-hook` program.
 
-    PathSetting diffHook{this, true, "", "diff-hook",
-        "A program that prints out the differences between the two paths "
-        "specified on its command line."};
+          When using the Nix daemon, `run-diff-hook` must be set in the
+          `nix.conf` configuration file, and cannot be passed at the command
+          line.
+        )"};
 
-    Setting<bool> enforceDeterminism{this, true, "enforce-determinism",
-        "Whether to fail if repeated builds produce different output."};
+    PathSetting diffHook{
+        this, true, "", "diff-hook",
+        R"(
+          Absolute path to an executable capable of diffing build
+          results. The hook is executed if `run-diff-hook` is true, and the
+          output of a build is known to not be the same. This program is not
+          executed to determine if two results are the same.
 
-    Setting<Strings> trustedPublicKeys{this,
+          The diff hook is executed by the same user and group who ran the
+          build. However, the diff hook does not have write access to the
+          store path just built.
+
+          The diff hook program receives three parameters:
+
+          1.  A path to the previous build's results
+
+          2.  A path to the current build's results
+
+          3.  The path to the build's derivation
+
+          4.  The path to the build's scratch directory. This directory will
+              exist only if the build was run with `--keep-failed`.
+
+          The stderr and stdout output from the diff hook will not be
+          displayed to the user. Instead, it will print to the nix-daemon's
+          log.
+
+          When using the Nix daemon, `diff-hook` must be set in the `nix.conf`
+          configuration file, and cannot be passed at the command line.
+        )"};
+
+    Setting<bool> enforceDeterminism{
+        this, true, "enforce-determinism",
+        "Whether to fail if repeated builds produce different output. See `repeat`."};
+
+    Setting<Strings> trustedPublicKeys{
+        this,
         {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
         "trusted-public-keys",
-        "Trusted public keys for secure substitution.",
+        R"(
+          A whitespace-separated list of public keys. When paths are copied
+          from another Nix store (such as a binary cache), they must be
+          signed with one of these keys. For example:
+          `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+          hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`.
+        )",
         {"binary-cache-public-keys"}};
 
-    Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
-        "Secret keys with which to sign local builds."};
+    Setting<Strings> secretKeyFiles{
+        this, {}, "secret-key-files",
+        R"(
+          A whitespace-separated list of files containing secret (private)
+          keys. These are used to sign locally-built paths. They can be
+          generated using `nix-store --generate-binary-cache-key`. The
+          corresponding public key can be distributed to other users, who
+          can add it to `trusted-public-keys` in their `nix.conf`.
+        )"};
 
-    Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
-        "How long downloaded files are considered up-to-date."};
+    Setting<unsigned int> tarballTtl{
+        this, 60 * 60, "tarball-ttl",
+        R"(
+          The number of seconds a downloaded tarball is considered fresh. If
+          the cached tarball is stale, Nix will check whether it is still up
+          to date using the ETag header. Nix will download a new version if
+          the ETag header is unsupported, or the cached ETag doesn't match.
 
-    Setting<bool> requireSigs{this, true, "require-sigs",
-        "Whether to check that any non-content-addressed path added to the "
-        "Nix store has a valid signature (that is, one signed using a key "
-        "listed in 'trusted-public-keys'."};
+          Setting the TTL to `0` forces Nix to always check if the tarball is
+          up to date.
 
-    Setting<StringSet> extraPlatforms{this,
+          Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
+
+          Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
+          `fetchTarball`, and `fetchurl` respect this TTL.
+        )"};
+
+    Setting<bool> requireSigs{
+        this, true, "require-sigs",
+        R"(
+          If set to `true` (the default), any non-content-addressed path added
+          or copied to the Nix store (e.g. when substituting from a binary
+          cache) must have a valid signature, that is, be signed using one of
+          the keys listed in `trusted-public-keys` or `secret-key-files`. Set
+          to `false` to disable signature checking.
+        )"};
+
+    Setting<StringSet> extraPlatforms{
+        this,
         std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
         "extra-platforms",
-        "Additional platforms that can be built on the local system. "
-        "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
-        "or using hacks like qemu-user."};
+        R"(
+          Platforms other than the native one which this machine is capable of
+          building for. This can be useful for supporting additional
+          architectures on compatible machines: i686-linux can be built on
+          x86\_64-linux machines (and the default for this setting reflects
+          this); armv7 is backwards-compatible with armv6 and armv5tel; some
+          aarch64 machines can also natively run 32-bit ARM code; and
+          qemu-user may be used to support non-native platforms (though this
+          may be slow and buggy). Most values for this are not enabled by
+          default because build systems will often misdetect the target
+          platform and generate incompatible code, so you may wish to
+          cross-check the results of using this option against proper
+          natively-built versions of your derivations.
+        )"};
 
-    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
+    Setting<StringSet> systemFeatures{
+        this, getDefaultSystemFeatures(),
         "system-features",
-        "Optional features that this system implements (like \"kvm\")."};
+        R"(
+          A set of system “features” supported by this machine, e.g. `kvm`.
+          Derivations can express a dependency on such features through the
+          derivation attribute `requiredSystemFeatures`. For example, the
+          attribute
 
-    Setting<Strings> substituters{this,
+              requiredSystemFeatures = [ "kvm" ];
+
+          ensures that the derivation can only be built on a machine with the
+          `kvm` feature.
+
+          This setting by default includes `kvm` if `/dev/kvm` is accessible,
+          and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
+          that are used in Nixpkgs to route builds to specific machines.
+        )"};
+
+    Setting<Strings> substituters{
+        this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
         "substituters",
-        "The URIs of substituters (such as https://cache.nixos.org/).",
+        R"(
+          A list of URLs of substituters, separated by whitespace. The default
+          is `https://cache.nixos.org`.
+        )",
         {"binary-caches"}};
 
     // FIXME: provide a way to add to option values.
-    Setting<Strings> extraSubstituters{this, {}, "extra-substituters",
-        "Additional URIs of substituters.",
+    Setting<Strings> extraSubstituters{
+        this, {}, "extra-substituters",
+        R"(
+          Additional binary caches appended to those specified in
+          `substituters`. When used by unprivileged users, untrusted
+          substituters (i.e. those not listed in `trusted-substituters`) are
+          silently ignored.
+        )",
         {"extra-binary-caches"}};
 
-    Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters",
-        "Disabled substituters that may be enabled via the substituters option by untrusted users.",
+    Setting<StringSet> trustedSubstituters{
+        this, {}, "trusted-substituters",
+        R"(
+          A list of URLs of substituters, separated by whitespace. These are
+          not used by default, but can be enabled by users of the Nix daemon
+          by specifying `--option substituters urls` on the command
+          line. Unprivileged users are only allowed to pass a subset of the
+          URLs listed in `substituters` and `trusted-substituters`.
+        )",
         {"trusted-binary-caches"}};
 
-    Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
-        "Which users or groups are trusted to ask the daemon to do unsafe things."};
+    Setting<Strings> trustedUsers{
+        this, {"root"}, "trusted-users",
+        R"(
+          A list of names of users (separated by whitespace) that have
+          additional rights when connecting to the Nix daemon, such as the
+          ability to specify additional binary caches, or to import unsigned
+          NARs. You can also specify groups by prefixing them with `@`; for
+          instance, `@wheel` means all users in the `wheel` group. The default
+          is `root`.
 
-    Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl",
-        "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that "
-        "return an invalid path result"};
+          > **Warning**
+          > 
+          > Adding a user to `trusted-users` is essentially equivalent to
+          > giving that user root access to the system. For example, the user
+          > can set `sandbox-paths` and thereby obtain read access to
+          > directories that are otherwise inacessible to them.
+        )"};
 
-    Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
-        "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that "
-        "return a valid path result."};
+    Setting<unsigned int> ttlNegativeNarInfoCache{
+        this, 3600, "narinfo-cache-negative-ttl",
+        R"(
+          The TTL in seconds for negative lookups. If a store path is queried
+          from a substituter but was not found, there will be a negative
+          lookup cached in the local disk cache database for the specified
+          duration.
+        )"};
+
+    Setting<unsigned int> ttlPositiveNarInfoCache{
+        this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
+        R"(
+          The TTL in seconds for positive lookups. If a store path is queried
+          from a substituter, the result of the query will be cached in the
+          local disk cache database including some of the NAR metadata. The
+          default TTL is a month, setting a shorter TTL for positive lookups
+          can be useful for binary caches that have frequent garbage
+          collection, in which case having a more frequent cache invalidation
+          would prevent trying to pull the path again and failing with a hash
+          mismatch if the build isn't reproducible.
+        )"};
 
     /* ?Who we trust to use the daemon in safe ways */
-    Setting<Strings> allowedUsers{this, {"*"}, "allowed-users",
-        "Which users or groups are allowed to connect to the daemon."};
+    Setting<Strings> allowedUsers{
+        this, {"*"}, "allowed-users",
+        R"(
+          A list of names of users (separated by whitespace) that are allowed
+          to connect to the Nix daemon. As with the `trusted-users` option,
+          you can specify groups by prefixing them with `@`. Also, you can
+          allow all users by specifying `*`. The default is `*`.
+
+          Note that trusted users are always allowed to connect.
+        )"};
 
     Setting<bool> printMissing{this, true, "print-missing",
         "Whether to print what paths need to be built or downloaded."};
 
-    Setting<std::string> preBuildHook{this, "",
-        "pre-build-hook",
-        "A program to run just before a build to set derivation-specific build settings."};
+    Setting<std::string> preBuildHook{
+        this, "", "pre-build-hook",
+        R"(
+          If set, the path to a program that can set extra derivation-specific
+          settings for this system. This is used for settings that can't be
+          captured by the derivation model itself and are too variable between
+          different versions of the same system to be hard-coded into nix.
 
-    Setting<std::string> postBuildHook{this, "", "post-build-hook",
-        "A program to run just after each successful build."};
+          The hook is passed the derivation path and, if sandboxes are
+          enabled, the sandbox directory. It can then modify the sandbox and
+          send a series of commands to modify various settings to stdout. The
+          currently recognized commands are:
 
-    Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
-        "Path to the netrc file used to obtain usernames/passwords for downloads."};
+            - `extra-sandbox-paths`  
+              Pass a list of files and directories to be included in the
+              sandbox for this build. One entry per line, terminated by an
+              empty line. Entries have the same format as `sandbox-paths`.
+        )"};
+
+    Setting<std::string> postBuildHook{
+        this, "", "post-build-hook",
+        R"(
+          Optional. The path to a program to execute after each build.
+
+          This option is only settable in the global `nix.conf`, or on the
+          command line by trusted users.
+
+          When using the nix-daemon, the daemon executes the hook as `root`.
+          If the nix-daemon is not involved, the hook runs as the user
+          executing the nix-build.
+
+            - The hook executes after an evaluation-time build.
+
+            - The hook does not execute on substituted paths.
+
+            - The hook's output always goes to the user's terminal.
+
+            - If the hook fails, the build succeeds but no further builds
+              execute.
+
+            - The hook executes synchronously, and blocks other builds from
+              progressing while it runs.
+
+          The program executes with no arguments. The program's environment
+          contains the following environment variables:
+
+            - `DRV_PATH`  
+              The derivation for the built paths.
+
+              Example:
+              `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
+
+            - `OUT_PATHS`  
+              Output paths of the built derivation, separated by a space
+              character.
+
+              Example:
+              `/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
+              /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+              /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+              /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+              /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
+        )"};
+
+    Setting<std::string> netrcFile{
+        this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
+        R"(
+          If set to an absolute path to a `netrc` file, Nix will use the HTTP
+          authentication credentials in this file when trying to download from
+          a remote host through HTTP or HTTPS. Defaults to
+          `$NIX_CONF_DIR/netrc`.
+
+          The `netrc` file consists of a list of accounts in the following
+          format:
+
+              machine my-machine
+              login my-username
+              password my-password
+
+          For the exact syntax, see [the `curl`
+          documentation](https://ec.haxx.se/usingcurl-netrc.html).
+
+          > **Note**
+          > 
+          > This must be an absolute path, and `~` is not resolved. For
+          > example, `~/.netrc` won't resolve to your home directory's
+          > `.netrc`.
+        )"};
 
     /* Path to the SSL CA file used */
     Path caFile;
 
 #if __linux__
-    Setting<bool> filterSyscalls{this, true, "filter-syscalls",
-            "Whether to prevent certain dangerous system calls, such as "
-            "creation of setuid/setgid files or adding ACLs or extended "
-            "attributes. Only disable this if you're aware of the "
-            "security implications."};
+    Setting<bool> filterSyscalls{
+        this, true, "filter-syscalls",
+        R"(
+          Whether to prevent certain dangerous system calls, such as
+          creation of setuid/setgid files or adding ACLs or extended
+          attributes. Only disable this if you're aware of the
+          security implications.
+        )"};
 
-    Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges",
-        "Whether builders can acquire new privileges by calling programs with "
-        "setuid/setgid bits or with file capabilities."};
+    Setting<bool> allowNewPrivileges{
+        this, false, "allow-new-privileges",
+        R"(
+          (Linux-specific.) By default, builders on Linux cannot acquire new
+          privileges by calling setuid/setgid programs or programs that have
+          file capabilities. For example, programs such as `sudo` or `ping`
+          will fail. (Note that in sandbox builds, no such programs are
+          available unless you bind-mount them into the sandbox via the
+          `sandbox-paths` option.) You can allow the use of such programs by
+          enabling this option. This is impure and usually undesirable, but
+          may be useful in certain scenarios (e.g. to spin up containers or
+          set up userspace network interfaces in tests).
+        )"};
 #endif
 
-    Setting<Strings> hashedMirrors{this, {}, "hashed-mirrors",
-        "A list of servers used by builtins.fetchurl to fetch files by hash."};
+    Setting<Strings> hashedMirrors{
+        this, {}, "hashed-mirrors",
+        R"(
+          A list of web servers used by `builtins.fetchurl` to obtain files by
+          hash. The default is `http://tarballs.nixos.org/`. Given a hash type
+          *ht* and a base-16 hash *h*, Nix will try to download the file from
+          *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
+          they have disappeared from their original URI. For example, given
+          the default mirror `http://tarballs.nixos.org/`, when building the
+          derivation
 
-    Setting<uint64_t> minFree{this, 0, "min-free",
-        "Automatically run the garbage collector when free disk space drops below the specified amount."};
+          ```nix
+          builtins.fetchurl {
+            url = "https://example.org/foo-1.2.3.tar.xz";
+            sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+          }
+          ```
 
-    Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
-        "Stop deleting garbage when free disk space is above the specified amount."};
+          Nix will attempt to download this file from
+          `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae`
+          first. If it is not available there, if will try the original URI.
+        )"};
+
+    Setting<uint64_t> minFree{
+        this, 0, "min-free",
+        R"(
+          When free disk space in `/nix/store` drops below `min-free` during a
+          build, Nix performs a garbage-collection until `max-free` bytes are
+          available or there is no more garbage. A value of `0` (the default)
+          disables this feature.
+        )"};
+
+    Setting<uint64_t> maxFree{
+        this, std::numeric_limits<uint64_t>::max(), "max-free",
+        R"(
+          When a garbage collection is triggered by the `min-free` option, it
+          stops as soon as `max-free` bytes are available. The default is
+          infinity (i.e. delete all garbage).
+        )"};
 
     Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
         "Number of seconds between checking free disk space."};
 
-    Setting<Paths> pluginFiles{this, {}, "plugin-files",
-        "Plugins to dynamically load at nix initialization time."};
+    Setting<Paths> pluginFiles{
+        this, {}, "plugin-files",
+        R"(
+          A list of plugin files to be loaded by Nix. Each of these files will
+          be dlopened by Nix, allowing them to affect execution through static
+          initialization. In particular, these plugins may construct static
+          instances of RegisterPrimOp to add new primops or constants to the
+          expression language, RegisterStoreImplementation to add new store
+          implementations, RegisterCommand to add new subcommands to the `nix`
+          command, and RegisterSetting to add new nix config settings. See the
+          constructors for those types for more details.
+
+          Since these files are loaded into the same address space as Nix
+          itself, they must be DSOs compatible with the instance of Nix
+          running at the time (i.e. compiled against the same headers, not
+          linked to any incompatible libraries). They should not be linked to
+          any Nix libs directly, as those will be available already at load
+          time.
+
+          If an entry in the list is a directory, all files in the directory
+          are loaded as plugins (non-recursively).
+        )"};
 
     Setting<std::string> githubAccessToken{this, "", "github-access-token",
-        "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
+        "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
 
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};

src/libstore/http-binary-cache-store.cc

@@ -2,12 +2,20 @@
 #include "filetransfer.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
+#include "callback.hh"
 
 namespace nix {
 
 MakeError(UploadToHTTP, Error);
 
-class HttpBinaryCacheStore : public BinaryCacheStore
+struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
+{
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+
+    const std::string name() override { return "Http Binary Cache Store"; }
+};
+
+class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
 {
 private:
 
@@ -24,9 +32,12 @@ private:
 public:
 
     HttpBinaryCacheStore(
-        const Params & params, const Path & _cacheUri)
-        : BinaryCacheStore(params)
-        , cacheUri(_cacheUri)
+        const std::string & scheme,
+        const Path & _cacheUri,
+        const Params & params)
+        : StoreConfig(params)
+        , BinaryCacheStore(params)
+        , cacheUri(scheme + "://" + _cacheUri)
     {
         if (cacheUri.back() == '/')
             cacheUri.pop_back();
@@ -55,6 +66,13 @@ public:
         }
     }
 
+    static std::set<std::string> uriSchemes()
+    {
+        static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
+        auto ret = std::set<std::string>({"http", "https"});
+        if (forceHttp) ret.insert("file");
+        return ret;
+    }
 protected:
 
     void maybeDisable()
@@ -85,7 +103,7 @@ protected:
         checkEnabled();
 
         try {
-            FileTransferRequest request(cacheUri + "/" + path);
+            FileTransferRequest request(makeRequest(path));
             request.head = true;
             getFileTransfer()->download(request);
             return true;
@@ -103,7 +121,7 @@ protected:
         std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) override
     {
-        auto req = FileTransferRequest(cacheUri + "/" + path);
+        auto req = makeRequest(path);
         req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
         req.mimeType = mimeType;
         try {
@@ -115,8 +133,11 @@ protected:
 
     FileTransferRequest makeRequest(const std::string & path)
     {
-        FileTransferRequest request(cacheUri + "/" + path);
-        return request;
+        return FileTransferRequest(
+            hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")
+            ? path
+            : cacheUri + "/" + path);
+
     }
 
     void getFile(const std::string & path, Sink & sink) override
@@ -159,18 +180,6 @@ protected:
 
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
-    if (std::string(uri, 0, 7) != "http://" &&
-        std::string(uri, 0, 8) != "https://" &&
-        (!forceHttp || std::string(uri, 0, 7) != "file://"))
-        return 0;
-    auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
-    store->init();
-    return store;
-});
+static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regStore;
 
 }

src/libstore/legacy-ssh-store.cc

@@ -6,21 +6,28 @@
 #include "worker-protocol.hh"
 #include "ssh.hh"
 #include "derivations.hh"
+#include "callback.hh"
 
 namespace nix {
 
-static std::string uriScheme = "ssh://";
-
-struct LegacySSHStore : public Store
+struct LegacySSHStoreConfig : virtual StoreConfig
 {
-    const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
-    const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
-    const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
-    const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
-    const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
+    using StoreConfig::StoreConfig;
+    const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
+    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
+    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
+    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
 
+    const std::string name() override { return "Legacy SSH Store"; }
+};
+
+struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
+{
     // Hack for getting remote build log output.
-    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
+    // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
+    // the documentation
+    const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
 
     struct Connection
     {
@@ -37,8 +44,11 @@ struct LegacySSHStore : public Store
 
     SSHMaster master;
 
-    LegacySSHStore(const string & host, const Params & params)
-        : Store(params)
+    static std::set<std::string> uriSchemes() { return {"ssh"}; }
+
+    LegacySSHStore(const string & scheme, const string & host, const Params & params)
+        : StoreConfig(params)
+        , Store(params)
         , host(host)
         , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
@@ -84,7 +94,7 @@ struct LegacySSHStore : public Store
 
     string getUri() override
     {
-        return uriScheme + host;
+        return *uriSchemes().begin() + "://" + host;
     }
 
     void queryPathInfoUncached(const StorePath & path,
@@ -325,12 +335,6 @@ public:
     }
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
-});
+static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regStore;
 
 }

src/libstore/local-binary-cache-store.cc

@@ -4,7 +4,14 @@
 
 namespace nix {
 
-class LocalBinaryCacheStore : public BinaryCacheStore
+struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
+{
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+
+    const std::string name() override { return "Local Binary Cache Store"; }
+};
+
+class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
 {
 private:
 
@@ -13,8 +20,11 @@ private:
 public:
 
     LocalBinaryCacheStore(
-        const Params & params, const Path & binaryCacheDir)
-        : BinaryCacheStore(params)
+        const std::string scheme,
+        const Path & binaryCacheDir,
+        const Params & params)
+        : StoreConfig(params)
+        , BinaryCacheStore(params)
         , binaryCacheDir(binaryCacheDir)
     {
     }
@@ -26,6 +36,8 @@ public:
         return "file://" + binaryCacheDir;
     }
 
+    static std::set<std::string> uriSchemes();
+
 protected:
 
     bool fileExists(const std::string & path) override;
@@ -85,16 +97,14 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path)
     return pathExists(binaryCacheDir + "/" + path);
 }
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
+std::set<std::string> LocalBinaryCacheStore::uriSchemes()
 {
-    if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
-        std::string(uri, 0, 7) != "file://")
-        return 0;
-    auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
-    store->init();
-    return store;
-});
+    if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1")
+        return {};
+    else
+        return {"file"};
+}
+
+static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regStore;
 
 }

src/libstore/local-store.cc

@@ -6,6 +6,7 @@
 #include "derivations.hh"
 #include "nar-info.hh"
 #include "references.hh"
+#include "callback.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -42,7 +43,8 @@ namespace nix {
 
 
 LocalStore::LocalStore(const Params & params)
-    : Store(params)
+    : StoreConfig(params)
+    , Store(params)
     , LocalFSStore(params)
     , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
         "physical path to the Nix store"}
@@ -578,13 +580,32 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
                 envHasRightPath(path, i.first);
             },
             [&](DerivationOutputCAFloating _) {
-                throw UnimplementedError("floating CA output derivations are not yet implemented");
+                /* Nothing to check */
             },
         }, i.second.output);
     }
 }
 
 
+void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
+{
+    auto state(_state.lock());
+    return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
+}
+
+void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
+{
+    retrySQLite<void>([&]() {
+        state.stmtAddDerivationOutput.use()
+            (deriver)
+            (outputName)
+            (printStorePath(output))
+            .exec();
+    });
+
+}
+
+
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
@@ -618,12 +639,11 @@ uint64_t LocalStore::addValidPath(State & state,
            registration above is undone. */
         if (checkOutputs) checkDerivationOutputs(info.path, drv);
 
-        for (auto & i : drv.outputsAndPaths(*this)) {
-            state.stmtAddDerivationOutput.use()
-                (id)
-                (i.first)
-                (printStorePath(i.second.second))
-                .exec();
+        for (auto & i : drv.outputsAndOptPaths(*this)) {
+            /* Floating CA derivations have indeterminate output paths until
+               they are built, so don't register anything in that case */
+            if (i.second.second)
+                linkDeriverToPath(state, id, i.first, *i.second.second);
         }
     }
 

src/libstore/local-store.hh

@@ -23,9 +23,6 @@ namespace nix {
 const int nixSchemaVersion = 10;
 
 
-struct Derivation;
-
-
 struct OptimiseStats
 {
     unsigned long filesLinked = 0;
@@ -33,8 +30,19 @@ struct OptimiseStats
     uint64_t blocksFreed = 0;
 };
 
+struct LocalStoreConfig : virtual LocalFSStoreConfig
+{
+    using LocalFSStoreConfig::LocalFSStoreConfig;
 
-class LocalStore : public LocalFSStore
+    Setting<bool> requireSigs{(StoreConfig*) this,
+        settings.requireSigs,
+        "require-sigs", "whether store paths should have a trusted signature on import"};
+
+    const std::string name() override { return "Local Store"; }
+};
+
+
+class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
 {
 private:
 
@@ -98,10 +106,6 @@ public:
 
 private:
 
-    Setting<bool> requireSigs{(Store*) this,
-        settings.requireSigs,
-        "require-sigs", "whether store paths should have a trusted signature on import"};
-
     const PublicKeys & getPublicKeys();
 
 public:
@@ -282,6 +286,11 @@ private:
        specified by the ‘secret-key-files’ option. */
     void signPathInfo(ValidPathInfo & info);
 
+    /* Register the store path 'output' as the output named 'outputName' of
+       derivation 'deriver'. */
+    void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
+    void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
+
     Path getRealStoreDir() override { return realStoreDir; }
 
     void createUser(const std::string & userName, uid_t userId) override;

src/libstore/misc.cc

@@ -5,7 +5,7 @@
 #include "store-api.hh"
 #include "thread-pool.hh"
 #include "topo-sort.hh"
-
+#include "callback.hh"
 
 namespace nix {
 
@@ -203,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                 return;
             }
 
+            PathSet invalid;
+            /* true for regular derivations, and CA derivations for which we
+               have a trust mapping for all wanted outputs. */
+            auto knownOutputPaths = true;
+            for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
+                if (!pathOpt) {
+                    knownOutputPaths = false;
+                    break;
+                }
+                if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
+                    invalid.insert(printStorePath(*pathOpt));
+            }
+            if (knownOutputPaths && invalid.empty()) return;
+
             auto drv = make_ref<Derivation>(derivationFromPath(path.path));
             ParsedDerivation parsedDrv(StorePath(path.path), *drv);
 
-            PathSet invalid;
-            for (auto & j : drv->outputsAndPaths(*this))
-                if (wantOutput(j.first, path.outputs)
-                    && !isValidPath(j.second.second))
-                    invalid.insert(printStorePath(j.second.second));
-            if (invalid.empty()) return;
-
-            if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
+            if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
                     pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));

src/libstore/names.cc

@@ -1,10 +1,18 @@
 #include "names.hh"
 #include "util.hh"
 
+#include <regex>
+
 
 namespace nix {
 
 
+struct Regex
+{
+    std::regex regex;
+};
+
+
 DrvName::DrvName()
 {
     name = "";
@@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0)
 }
 
 
+DrvName::~DrvName()
+{ }
+
+
 bool DrvName::matches(DrvName & n)
 {
     if (name != "*") {
-        if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended));
-        if (!std::regex_match(n.name, *regex)) return false;
+        if (!regex) {
+            regex = std::make_unique<Regex>();
+            regex->regex = std::regex(name, std::regex::extended);
+        }
+        if (!std::regex_match(n.name, regex->regex)) return false;
     }
     if (version != "" && version != n.version) return false;
     return true;
@@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs)
 {
     DrvNames result;
     for (auto & i : opArgs)
-        result.push_back(DrvName(i));
+        result.emplace_back(i);
     return result;
 }
 

src/libstore/names.hh

@@ -3,10 +3,11 @@
 #include <memory>
 
 #include "types.hh"
-#include <regex>
 
 namespace nix {
 
+struct Regex;
+
 struct DrvName
 {
     string fullName;
@@ -16,10 +17,12 @@ struct DrvName
 
     DrvName();
     DrvName(std::string_view s);
+    ~DrvName();
+
     bool matches(DrvName & n);
 
 private:
-    std::unique_ptr<std::regex> regex;
+    std::unique_ptr<Regex> regex;
 };
 
 typedef list<DrvName> DrvNames;

src/libstore/nar-accessor.cc

@@ -49,7 +49,8 @@ struct NarAccessor : public FSAccessor
             : acc(acc), source(source)
         { }
 
-        void createMember(const Path & path, NarMember member) {
+        void createMember(const Path & path, NarMember member)
+        {
             size_t level = std::count(path.begin(), path.end(), '/');
             while (parents.size() > level) parents.pop();
 

src/libstore/profiles.cc

@@ -72,7 +72,7 @@ static void makeName(const Path & profile, GenerationNumber num,
 }
 
 
-Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
+Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
 {
     /* The new generation number should be higher than old the
        previous ones. */
@@ -82,7 +82,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
     if (gens.size() > 0) {
         Generation last = gens.back();
 
-        if (readLink(last.path) == outPath) {
+        if (readLink(last.path) == store->printStorePath(outPath)) {
             /* We only create a new generation symlink if it differs
                from the last one.
 
@@ -105,7 +105,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
        user environment etc. we've just built. */
     Path generation;
     makeName(profile, num + 1, generation);
-    store->addPermRoot(store->parseStorePath(outPath), generation, false, true);
+    store->addPermRoot(outPath, generation);
 
     return generation;
 }

src/libstore/profiles.hh

@@ -8,6 +8,8 @@
 
 namespace nix {
 
+class StorePath;
+
 
 typedef unsigned int GenerationNumber;
 
@@ -28,7 +30,7 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
 
 class LocalFSStore;
 
-Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
+Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath);
 
 void deleteGeneration(const Path & profile, GenerationNumber gen);
 

src/libstore/references.cc

@@ -79,9 +79,17 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
 std::pair<PathSet, HashResult> scanForReferences(const string & path,
     const PathSet & refs)
 {
-    RefScanSink refsSink;
     HashSink hashSink { htSHA256 };
-    TeeSink sink { refsSink, hashSink };
+    auto found = scanForReferences(hashSink, path, refs);
+    auto hash = hashSink.finish();
+    return std::pair<PathSet, HashResult>(found, hash);
+}
+
+PathSet scanForReferences(Sink & toTee,
+    const string & path, const PathSet & refs)
+{
+    RefScanSink refsSink;
+    TeeSink sink { refsSink, toTee };
     std::map<string, Path> backMap;
 
     /* For efficiency (and a higher hit rate), just search for the
@@ -111,9 +119,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
         found.insert(j->second);
     }
 
-    auto hash = hashSink.finish();
-
-    return std::pair<PathSet, HashResult>(found, hash);
+    return found;
 }
 
 

src/libstore/references.hh

@@ -7,6 +7,8 @@ namespace nix {
 
 std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
 
+PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
+
 struct RewritingSink : Sink
 {
     std::string from, to, prev;

src/libstore/remote-store.cc

@@ -1,5 +1,6 @@
 #include "serialise.hh"
 #include "util.hh"
+#include "remote-fs-accessor.hh"
 #include "remote-store.hh"
 #include "worker-protocol.hh"
 #include "archive.hh"
@@ -9,6 +10,7 @@
 #include "pool.hh"
 #include "finally.hh"
 #include "logging.hh"
+#include "callback.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -57,7 +59,7 @@ void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const C
 
 std::optional<StorePath> WorkerProto<std::optional<StorePath>>::read(const Store & store, Source & from)
 {
-	auto s = readString(from);
+    auto s = readString(from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
@@ -81,9 +83,19 @@ void WorkerProto<std::optional<ContentAddress>>::write(const Store & store, Sink
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : Store(params)
+    , RemoteStoreConfig(params)
     , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
-            [this]() { return openConnectionWrapper(); },
+            [this]() {
+                auto conn = openConnectionWrapper();
+                try {
+                    initConnection(*conn);
+                } catch (...) {
+                    failed = true;
+                    throw;
+                }
+                return conn;
+            },
             [this](const ref<Connection> & r) {
                 return
                     r->to.good()
@@ -110,19 +122,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
 
 
 UDSRemoteStore::UDSRemoteStore(const Params & params)
-    : Store(params)
+    : StoreConfig(params)
+    , Store(params)
     , LocalFSStore(params)
     , RemoteStore(params)
 {
 }
 
 
-UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
-    : Store(params)
-    , LocalFSStore(params)
-    , RemoteStore(params)
-    , path(socket_path)
+UDSRemoteStore::UDSRemoteStore(
+        const std::string scheme,
+        std::string socket_path,
+        const Params & params)
+    : UDSRemoteStore(params)
 {
+    path.emplace(socket_path);
 }
 
 
@@ -166,8 +180,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
 
     conn->startTime = std::chrono::steady_clock::now();
 
-    initConnection(*conn);
-
     return conn;
 }
 
@@ -275,9 +287,9 @@ struct ConnectionHandle
 
     RemoteStore::Connection * operator -> () { return &*handle; }
 
-    void processStderr(Sink * sink = 0, Source * source = 0)
+    void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
     {
-        auto ex = handle->processStderr(sink, source);
+        auto ex = handle->processStderr(sink, source, flush);
         if (ex) {
             daemonException = true;
             std::rethrow_exception(ex);
@@ -467,10 +479,26 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 
 std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
 {
-    auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputMap << printStorePath(path);
-    conn.processStderr();
-    return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
+    if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
+        auto conn(getConnection());
+        conn->to << wopQueryDerivationOutputMap << printStorePath(path);
+        conn.processStderr();
+        return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
+    } else {
+        // Fallback for old daemon versions.
+        // For floating-CA derivations (and their co-dependencies) this is an
+        // under-approximation as it only returns the paths that can be inferred
+        // from the derivation itself (and not the ones that are known because
+        // the have been built), but as old stores don't handle floating-CA
+        // derivations this shouldn't matter
+        auto derivation = readDerivation(path);
+        auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
+        std::map<std::string, std::optional<StorePath>> ret;
+        for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
+            ret.emplace(outputName, outputAndPath.second);
+        }
+        return ret;
+    }
 }
 
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
@@ -525,6 +553,8 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
 
+            conn->to.flush();
+
             std::exception_ptr ex;
 
             struct FramedSink : BufferedSink
@@ -564,7 +594,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             std::thread stderrThread([&]()
             {
                 try {
-                    conn.processStderr();
+                    conn.processStderr(nullptr, nullptr, false);
                 } catch (...) {
                     ex = std::current_exception();
                 }
@@ -856,6 +886,18 @@ RemoteStore::Connection::~Connection()
     }
 }
 
+void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
+{
+    auto conn(connections->get());
+    conn->to << wopNarFromPath << printStorePath(path);
+    conn->processStderr();
+    copyNAR(conn->from, sink);
+}
+
+ref<FSAccessor> RemoteStore::getFSAccessor()
+{
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
+}
 
 static Logger::Fields readFields(Source & from)
 {
@@ -874,9 +916,10 @@ static Logger::Fields readFields(Source & from)
 }
 
 
-std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
+std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
 {
-    to.flush();
+    if (flush)
+        to.flush();
 
     while (true) {
 
@@ -937,14 +980,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
-static std::string uriScheme = "unix://";
-
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
-});
+static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore;
 
 }

src/libstore/remote-store.hh

@@ -16,19 +16,23 @@ struct FdSource;
 template<typename T> class Pool;
 struct ConnectionHandle;
 
+struct RemoteStoreConfig : virtual StoreConfig
+{
+    using StoreConfig::StoreConfig;
+
+    const Setting<int> maxConnections{(StoreConfig*) this, 1,
+            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
+
+    const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
+            "max-connection-age", "number of seconds to reuse a connection"};
+};
 
 /* FIXME: RemoteStore is a misnomer - should be something like
    DaemonStore. */
-class RemoteStore : public virtual Store
+class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
 {
 public:
 
-    const Setting<int> maxConnections{(Store*) this, 1,
-            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
-
-    const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
-            "max-connection-age", "number of seconds to reuse a connection"};
-
     virtual bool sameMachine() = 0;
 
     RemoteStore(const Params & params);
@@ -102,8 +106,6 @@ public:
 
     void flushBadConnections();
 
-protected:
-
     struct Connection
     {
         AutoCloseFD fd;
@@ -114,11 +116,13 @@ protected:
 
         virtual ~Connection();
 
-        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0);
+        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
     };
 
     ref<Connection> openConnectionWrapper();
 
+protected:
+
     virtual ref<Connection> openConnection() = 0;
 
     void initConnection(Connection & conn);
@@ -131,24 +135,54 @@ protected:
 
     friend struct ConnectionHandle;
 
+    virtual ref<FSAccessor> getFSAccessor() override;
+
+    virtual void narFromPath(const StorePath & path, Sink & sink) override;
+
 private:
 
     std::atomic_bool failed{false};
 
 };
 
-class UDSRemoteStore : public LocalFSStore, public RemoteStore
+struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
+{
+    UDSRemoteStoreConfig(const Store::Params & params)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , RemoteStoreConfig(params)
+    {
+    }
+
+    UDSRemoteStoreConfig()
+        : UDSRemoteStoreConfig(Store::Params({}))
+    {
+    }
+
+    const std::string name() override { return "Local Daemon Store"; }
+};
+
+class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
 {
 public:
 
     UDSRemoteStore(const Params & params);
-    UDSRemoteStore(std::string path, const Params & params);
+    UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
 
     std::string getUri() override;
 
+    static std::set<std::string> uriSchemes()
+    { return {"unix"}; }
+
     bool sameMachine() override
     { return true; }
 
+    ref<FSAccessor> getFSAccessor() override
+    { return LocalFSStore::getFSAccessor(); }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    { LocalFSStore::narFromPath(path, sink); }
+
 private:
 
     ref<RemoteStore::Connection> openConnection() override;

src/libstore/s3-binary-cache-store.cc

@@ -172,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject(
     return res;
 }
 
-struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
+struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
-    const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
-    const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
-    const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
-    const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
-    const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
-    const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
-    const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+    const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
+    const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
+    const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
+    const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
+    const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
+    const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
     const Setting<bool> multipartUpload{
-        this, false, "multipart-upload", "whether to use multi-part uploads"};
+        (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
     const Setting<uint64_t> bufferSize{
-        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
+        (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
 
+    const std::string name() override { return "S3 Binary Cache Store"; }
+};
+
+struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
+{
     std::string bucketName;
 
     Stats stats;
@@ -193,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     S3Helper s3Helper;
 
     S3BinaryCacheStoreImpl(
-        const Params & params, const std::string & bucketName)
-        : S3BinaryCacheStore(params)
+        const std::string & scheme,
+        const std::string & bucketName,
+        const Params & params)
+        : StoreConfig(params)
+        , S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(profile, region, scheme, endpoint)
     {
@@ -426,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         return paths;
     }
 
+    static std::set<std::string> uriSchemes() { return {"s3"}; }
+
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, 5) != "s3://") return 0;
-    auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
-    store->init();
-    return store;
-});
+static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regStore;
 
 }
 

src/libstore/ssh-store.cc

@@ -8,19 +8,25 @@
 
 namespace nix {
 
-static std::string uriScheme = "ssh-ng://";
+struct SSHStoreConfig : virtual RemoteStoreConfig
+{
+    using RemoteStoreConfig::RemoteStoreConfig;
 
-class SSHStore : public RemoteStore
+    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
+    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
+    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
+
+    const std::string name() override { return "SSH Store"; }
+};
+
+class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
 {
 public:
 
-    const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
-    const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
-    const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
-    const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"};
-
-    SSHStore(const std::string & host, const Params & params)
-        : Store(params)
+    SSHStore(const std::string & scheme, const std::string & host, const Params & params)
+        : StoreConfig(params)
+        , Store(params)
         , RemoteStore(params)
         , host(host)
         , master(
@@ -32,18 +38,16 @@ public:
     {
     }
 
+    static std::set<std::string> uriSchemes() { return {"ssh-ng"}; }
+
     std::string getUri() override
     {
-        return uriScheme + host;
+        return *uriSchemes().begin() + "://" + host;
     }
 
     bool sameMachine() override
     { return false; }
 
-    void narFromPath(const StorePath & path, Sink & sink) override;
-
-    ref<FSAccessor> getFSAccessor() override;
-
 private:
 
     struct Connection : RemoteStore::Connection
@@ -68,19 +72,6 @@ private:
     };
 };
 
-void SSHStore::narFromPath(const StorePath & path, Sink & sink)
-{
-    auto conn(connections->get());
-    conn->to << wopNarFromPath << printStorePath(path);
-    conn->processStderr();
-    copyNAR(conn->from, sink);
-}
-
-ref<FSAccessor> SSHStore::getFSAccessor()
-{
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
-}
-
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     auto conn = make_ref<Connection>();
@@ -89,16 +80,9 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
         + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
     conn->to = FdSink(conn->sshConn->in.get());
     conn->from = FdSource(conn->sshConn->out.get());
-    initConnection(*conn);
     return conn;
 }
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
-});
+static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regStore;
 
 }

src/libstore/store-api.cc

@@ -1,16 +1,14 @@
 #include "crypto.hh"
+#include "fs-accessor.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
 #include "json.hh"
-#include "derivations.hh"
 #include "url.hh"
 #include "archive.hh"
-
-#include <future>
-
+#include "callback.hh"
 
 namespace nix {
 
@@ -140,21 +138,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
 */
 
 
-StorePath Store::makeStorePath(const string & type,
-    const Hash & hash, std::string_view name) const
+StorePath Store::makeStorePath(std::string_view type,
+    std::string_view hash, std::string_view name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
+    string s = std::string { type } + ":" + std::string { hash }
+        + ":" + storeDir + ":" + std::string { name };
     auto h = compressHash(hashString(htSHA256, s), 20);
     return StorePath(h, name);
 }
 
 
-StorePath Store::makeOutputPath(const string & id,
+StorePath Store::makeStorePath(std::string_view type,
     const Hash & hash, std::string_view name) const
 {
-    return makeStorePath("output:" + id, hash,
-        std::string(name) + (id == "out" ? "" : "-" + id));
+    return makeStorePath(type, hash.to_string(Base16, true), name);
+}
+
+
+StorePath Store::makeOutputPath(std::string_view id,
+    const Hash & hash, std::string_view name) const
+{
+    return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
 }
 
 
@@ -339,7 +344,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
 
 
 Store::Store(const Params & params)
-    : Config(params)
+    : StoreConfig(params)
     , state({(size_t) pathInfoCacheSize})
 {
 }
@@ -983,6 +988,25 @@ Strings ValidPathInfo::shortRefs() const
 }
 
 
+Derivation Store::derivationFromPath(const StorePath & drvPath)
+{
+    ensurePath(drvPath);
+    return readDerivation(drvPath);
+}
+
+
+Derivation Store::readDerivation(const StorePath & drvPath)
+{
+    auto accessor = getFSAccessor();
+    try {
+        return parseDerivation(*this,
+            accessor->readFile(printStorePath(drvPath)),
+            Derivation::nameFromPath(drvPath));
+    } catch (FormatError & e) {
+        throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
+    }
+}
+
 }
 
 
@@ -992,9 +1016,6 @@ Strings ValidPathInfo::shortRefs() const
 
 namespace nix {
 
-
-RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
-
 /* Split URI into protocol+hierarchy part and its parameter set. */
 std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
 {
@@ -1008,24 +1029,6 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
     return {uri, params};
 }
 
-ref<Store> openStore(const std::string & uri_,
-    const Store::Params & extraParams)
-{
-    auto [uri, uriParams] = splitUriAndParams(uri_);
-    auto params = extraParams;
-    params.insert(uriParams.begin(), uriParams.end());
-
-    for (auto fun : *RegisterStoreImplementation::implementations) {
-        auto store = fun(uri, params);
-        if (store) {
-            store->warnUnknownSettings();
-            return ref<Store>(store);
-        }
-    }
-
-    throw Error("don't know how to open Nix store '%s'", uri);
-}
-
 static bool isNonUriPath(const std::string & spec) {
     return
         // is not a URL
@@ -1035,44 +1038,62 @@ static bool isNonUriPath(const std::string & spec) {
         && spec.find("/") != std::string::npos;
 }
 
-StoreType getStoreType(const std::string & uri, const std::string & stateDir)
+std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
 {
-    if (uri == "daemon") {
-        return tDaemon;
-    } else if (uri == "local" || isNonUriPath(uri)) {
-        return tLocal;
-    } else if (uri == "" || uri == "auto") {
+    if (uri == "" || uri == "auto") {
+        auto stateDir = get(params, "state").value_or(settings.nixStateDir);
         if (access(stateDir.c_str(), R_OK | W_OK) == 0)
-            return tLocal;
+            return std::make_shared<LocalStore>(params);
         else if (pathExists(settings.nixDaemonSocketFile))
-            return tDaemon;
+            return std::make_shared<UDSRemoteStore>(params);
         else
-            return tLocal;
+            return std::make_shared<LocalStore>(params);
+    } else if (uri == "daemon") {
+        return std::make_shared<UDSRemoteStore>(params);
+    } else if (uri == "local") {
+        return std::make_shared<LocalStore>(params);
+    } else if (isNonUriPath(uri)) {
+        Store::Params params2 = params;
+        params2["root"] = absPath(uri);
+        return std::make_shared<LocalStore>(params2);
     } else {
-        return tOther;
+        return nullptr;
     }
 }
 
-
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
+ref<Store> openStore(const std::string & uri_,
+    const Store::Params & extraParams)
 {
-    switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) {
-        case tDaemon:
-            return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
-        case tLocal: {
-            Store::Params params2 = params;
-            if (isNonUriPath(uri)) {
-                params2["root"] = absPath(uri);
-            }
-            return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
-        }
-        default:
-            return nullptr;
-    }
-});
+    auto params = extraParams;
+    try {
+        auto parsedUri = parseURL(uri_);
+        params.insert(parsedUri.query.begin(), parsedUri.query.end());
 
+        auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
+
+        for (auto implem : *Implementations::registered) {
+            if (implem.uriSchemes.count(parsedUri.scheme)) {
+                auto store = implem.create(parsedUri.scheme, baseURI, params);
+                if (store) {
+                    store->init();
+                    store->warnUnknownSettings();
+                    return ref<Store>(store);
+                }
+            }
+        }
+    }
+    catch (BadURL &) {
+        auto [uri, uriParams] = splitUriAndParams(uri_);
+        params.insert(uriParams.begin(), uriParams.end());
+
+        if (auto store = openFromNonUri(uri, params)) {
+            store->warnUnknownSettings();
+            return ref<Store>(store);
+        }
+    }
+
+    throw Error("don't know how to open Nix store '%s'", uri_);
+}
 
 std::list<ref<Store>> getDefaultSubstituters()
 {
@@ -1106,5 +1127,6 @@ std::list<ref<Store>> getDefaultSubstituters()
     return stores;
 }
 
+std::vector<StoreFactory> * Implementations::registered = 0;
 
 }

src/libstore/store-api.hh

@@ -24,6 +24,31 @@
 
 namespace nix {
 
+/**
+ * About the class hierarchy of the store implementations:
+ *
+ * Each store type `Foo` consists of two classes:
+ *
+ * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration
+ *   for the store
+ *
+ *   It should only contain members of type `const Setting<T>` (or subclasses
+ *   of it) and inherit the constructors of `StoreConfig`
+ *   (`using StoreConfig::StoreConfig`).
+ *
+ * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the
+ *   implementation of the store.
+ *
+ *   This class is expected to have a constructor `Foo(const Params & params)`
+ *   that calls `StoreConfig(params)` (otherwise you're gonna encounter an
+ *   `assertion failure` when trying to instantiate it).
+ *
+ * You can then register the new store using:
+ *
+ * ```
+ * cpp static RegisterStoreImplementation<Foo, FooConfig> regStore;
+ * ```
+ */
 
 MakeError(SubstError, Error);
 MakeError(BuildError, Error); // denotes a permanent build failure
@@ -33,6 +58,7 @@ MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
 MakeError(BadStorePath, Error);
 
+MakeError(InvalidStoreURI, Error);
 
 class FSAccessor;
 class NarInfoDiskCache;
@@ -144,12 +170,31 @@ struct BuildResult
     }
 };
 
-
-class Store : public std::enable_shared_from_this<Store>, public Config
+struct StoreConfig : public Config
 {
-public:
+    using Config::Config;
 
-    typedef std::map<std::string, std::string> Params;
+    /**
+     * When constructing a store implementation, we pass in a map `params` of
+     * parameters that's supposed to initialize the associated config.
+     * To do that, we must use the `StoreConfig(StringMap & params)`
+     * constructor, so we'd like to `delete` its default constructor to enforce
+     * it.
+     *
+     * However, actually deleting it means that all the subclasses of
+     * `StoreConfig` will have their default constructor deleted (because it's
+     * supposed to call the deleted default constructor of `StoreConfig`). But
+     * because we're always using virtual inheritance, the constructors of
+     * child classes will never implicitely call this one, so deleting it will
+     * be more painful than anything else.
+     *
+     * So we `assert(false)` here to ensure at runtime that the right
+     * constructor is always called without having to redefine a custom
+     * constructor for each `*Config` class.
+     */
+    StoreConfig() { assert(false); }
+
+    virtual const std::string name() = 0;
 
     const PathSetting storeDir_{this, false, settings.nixStore,
         "store", "path to the Nix store"};
@@ -167,6 +212,14 @@ public:
         "system-features",
         "Optional features that the system this store builds on implements (like \"kvm\")."};
 
+};
+
+class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
+{
+public:
+
+    typedef std::map<std::string, std::string> Params;
+
 protected:
 
     struct PathInfoCacheValue {
@@ -200,6 +253,11 @@ protected:
     Store(const Params & params);
 
 public:
+    /**
+     * Perform any necessary effectful operation to make the store up and
+     * running
+     */
+    virtual void init() {};
 
     virtual ~Store() { }
 
@@ -247,10 +305,12 @@ public:
     StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
 
     /* Constructs a unique store path name. */
-    StorePath makeStorePath(const string & type,
+    StorePath makeStorePath(std::string_view type,
+        std::string_view hash, std::string_view name) const;
+    StorePath makeStorePath(std::string_view type,
         const Hash & hash, std::string_view name) const;
 
-    StorePath makeOutputPath(const string & id,
+    StorePath makeOutputPath(std::string_view id,
         const Hash & hash, std::string_view name) const;
 
     StorePath makeFixedOutputPath(FileIngestionMethod method,
@@ -624,22 +684,25 @@ protected:
 
 };
 
-
-class LocalFSStore : public virtual Store
+struct LocalFSStoreConfig : virtual StoreConfig
 {
-public:
-
-    // FIXME: the (Store*) cast works around a bug in gcc that causes
+    using StoreConfig::StoreConfig;
+    // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
     // it to omit the call to the Setting constructor. Clang works fine
     // either way.
-    const PathSetting rootDir{(Store*) this, true, "",
+    const PathSetting rootDir{(StoreConfig*) this, true, "",
         "root", "directory prefixed to all other paths"};
-    const PathSetting stateDir{(Store*) this, false,
+    const PathSetting stateDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
         "state", "directory where Nix will store state"};
-    const PathSetting logDir{(Store*) this, false,
+    const PathSetting logDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
         "log", "directory where Nix will store state"};
+};
+
+class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
+{
+public:
 
     const static string drvsLogDir;
 
@@ -649,8 +712,7 @@ public:
     ref<FSAccessor> getFSAccessor() override;
 
     /* Register a permanent GC root. */
-    Path addPermRoot(const StorePath & storePath,
-        const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
+    Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
 
     virtual Path getRealStoreDir() { return storeDir; }
 
@@ -729,37 +791,47 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
     const Store::Params & extraParams = Store::Params());
 
 
-enum StoreType {
-    tDaemon,
-    tLocal,
-    tOther
-};
-
-
-StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
-    const std::string & stateDir = settings.nixStateDir);
-
 /* Return the default substituter stores, defined by the
    ‘substituters’ option and various legacy options. */
 std::list<ref<Store>> getDefaultSubstituters();
 
-
-/* Store implementation registration. */
-typedef std::function<std::shared_ptr<Store>(
-    const std::string & uri, const Store::Params & params)> OpenStore;
-
-struct RegisterStoreImplementation
+struct StoreFactory
 {
-    typedef std::vector<OpenStore> Implementations;
-    static Implementations * implementations;
+    std::set<std::string> uriSchemes;
+    std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
+    std::function<std::shared_ptr<StoreConfig> ()> getConfig;
+};
+struct Implementations
+{
+    static std::vector<StoreFactory> * registered;
 
-    RegisterStoreImplementation(OpenStore fun)
+    template<typename T, typename TConfig>
+    static void add()
     {
-        if (!implementations) implementations = new Implementations;
-        implementations->push_back(fun);
+        if (!registered) registered = new std::vector<StoreFactory>();
+        StoreFactory factory{
+            .uriSchemes = T::uriSchemes(),
+            .create =
+                ([](const std::string & scheme, const std::string & uri, const Store::Params & params)
+                 -> std::shared_ptr<Store>
+                 { return std::make_shared<T>(scheme, uri, params); }),
+            .getConfig =
+                ([]()
+                 -> std::shared_ptr<StoreConfig>
+                 { return std::make_shared<TConfig>(StringMap({})); })
+        };
+        registered->push_back(factory);
     }
 };
 
+template<typename T, typename TConfig>
+struct RegisterStoreImplementation
+{
+    RegisterStoreImplementation()
+    {
+        Implementations::add<T, TConfig>();
+    }
+};
 
 
 /* Display a set of paths in human-readable form (i.e., between quotes

src/libutil/abstract-setting-to-json.hh

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+#include "config.hh"
+
+namespace nix {
+template<typename T>
+std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject()
+{
+    auto obj = AbstractSetting::toJSONObject();
+    obj.emplace("value", value);
+    obj.emplace("defaultValue", defaultValue);
+    return obj;
+}
+}

src/libutil/args.cc

@@ -3,6 +3,8 @@
 
 #include <glob.h>
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 void Args::addFlag(Flag && flag_)
@@ -205,6 +207,43 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
+nlohmann::json Args::toJSON()
+{
+    auto flags = nlohmann::json::object();
+
+    for (auto & [name, flag] : longFlags) {
+        auto j = nlohmann::json::object();
+        if (flag->shortName)
+            j["shortName"] = std::string(1, flag->shortName);
+        if (flag->description != "")
+            j["description"] = flag->description;
+        if (flag->category != "")
+            j["category"] = flag->category;
+        if (flag->handler.arity != ArityAny)
+            j["arity"] = flag->handler.arity;
+        if (!flag->labels.empty())
+            j["labels"] = flag->labels;
+        flags[name] = std::move(j);
+    }
+
+    auto args = nlohmann::json::array();
+
+    for (auto & arg : expectedArgs) {
+        auto j = nlohmann::json::object();
+        j["label"] = arg.label;
+        j["optional"] = arg.optional;
+        if (arg.handler.arity != ArityAny)
+            j["arity"] = arg.handler.arity;
+        args.push_back(std::move(j));
+    }
+
+    auto res = nlohmann::json::object();
+    res["description"] = description();
+    res["flags"] = std::move(flags);
+    res["args"] = std::move(args);
+    return res;
+}
+
 static void hashTypeCompleter(size_t index, std::string_view prefix) 
 {
     for (auto & type : hashTypes)
@@ -313,11 +352,29 @@ void Command::printHelp(const string & programName, std::ostream & out)
     }
 }
 
+nlohmann::json Command::toJSON()
+{
+    auto exs = nlohmann::json::array();
+
+    for (auto & example : examples()) {
+        auto ex = nlohmann::json::object();
+        ex["description"] = example.description;
+        ex["command"] = chomp(stripIndentation(example.command));
+        exs.push_back(std::move(ex));
+    }
+
+    auto res = Args::toJSON();
+    res["examples"] = std::move(exs);
+    auto s = doc();
+    if (s != "") res.emplace("doc", stripIndentation(s));
+    return res;
+}
+
 MultiCommand::MultiCommand(const Commands & commands)
     : commands(commands)
 {
     expectArgs({
-        .label = "command",
+        .label = "subcommand",
         .optional = true,
         .handler = {[=](std::string s) {
             assert(!command);
@@ -387,4 +444,20 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
+nlohmann::json MultiCommand::toJSON()
+{
+    auto cmds = nlohmann::json::object();
+
+    for (auto & [name, commandFun] : commands) {
+        auto command = commandFun();
+        auto j = command->toJSON();
+        j["category"] = categories[command->category()];
+        cmds[name] = std::move(j);
+    }
+
+    auto res = Args::toJSON();
+    res["commands"] = std::move(cmds);
+    return res;
+}
+
 }

src/libutil/args.hh

@@ -4,6 +4,8 @@
 #include <map>
 #include <memory>
 
+#include <nlohmann/json_fwd.hpp>
+
 #include "util.hh"
 
 namespace nix {
@@ -20,6 +22,7 @@ public:
 
     virtual void printHelp(const string & programName, std::ostream & out);
 
+    /* Return a short one-line description of the command. */
     virtual std::string description() { return ""; }
 
 protected:
@@ -203,6 +206,8 @@ public:
         });
     }
 
+    virtual nlohmann::json toJSON();
+
     friend class MultiCommand;
 };
 
@@ -217,6 +222,9 @@ struct Command : virtual Args
     virtual void prepare() { };
     virtual void run() = 0;
 
+    /* Return documentation about this command, in Markdown format. */
+    virtual std::string doc() { return ""; }
+
     struct Example
     {
         std::string description;
@@ -234,6 +242,8 @@ struct Command : virtual Args
     virtual Category category() { return catDefault; }
 
     void printHelp(const string & programName, std::ostream & out) override;
+
+    nlohmann::json toJSON() override;
 };
 
 typedef std::map<std::string, std::function<ref<Command>()>> Commands;
@@ -259,6 +269,8 @@ public:
     bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
 
     bool processArgs(const Strings & args, bool finish) override;
+
+    nlohmann::json toJSON() override;
 };
 
 Strings argvToStrings(int argc, char * * argv);

src/libutil/callback.hh

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <future>
+#include <functional>
+
+namespace nix {
+
+/* A callback is a wrapper around a lambda that accepts a valid of
+   type T or an exception. (We abuse std::future<T> to pass the value or
+   exception.) */
+template<typename T>
+class Callback
+{
+    std::function<void(std::future<T>)> fun;
+    std::atomic_flag done = ATOMIC_FLAG_INIT;
+
+public:
+
+    Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
+
+    Callback(Callback && callback) : fun(std::move(callback.fun))
+    {
+        auto prev = callback.done.test_and_set();
+        if (prev) done.test_and_set();
+    }
+
+    void operator()(T && t) noexcept
+    {
+        auto prev = done.test_and_set();
+        assert(!prev);
+        std::promise<T> promise;
+        promise.set_value(std::move(t));
+        fun(promise.get_future());
+    }
+
+    void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
+    {
+        auto prev = done.test_and_set();
+        assert(!prev);
+        std::promise<T> promise;
+        promise.set_exception(exc);
+        fun(promise.get_future());
+    }
+};
+
+}

src/libutil/config.cc

@@ -1,6 +1,8 @@
 #include "config.hh"
 #include "args.hh"
-#include "json.hh"
+#include "abstract-setting-to-json.hh"
+
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -131,15 +133,14 @@ void Config::resetOverriden()
         s.second.setting->overriden = false;
 }
 
-void Config::toJSON(JSONObject & out)
+nlohmann::json Config::toJSON()
 {
+    auto res = nlohmann::json::object();
     for (auto & s : _settings)
         if (!s.second.isAlias) {
-            JSONObject out2(out.object(s.first));
-            out2.attr("description", s.second.setting->description);
-            JSONPlaceholder out3(out2.placeholder("value"));
-            s.second.setting->toJSON(out3);
+            res.emplace(s.first, s.second.setting->toJSON());
         }
+    return res;
 }
 
 void Config::convertToArgs(Args & args, const std::string & category)
@@ -153,7 +154,7 @@ AbstractSetting::AbstractSetting(
     const std::string & name,
     const std::string & description,
     const std::set<std::string> & aliases)
-    : name(name), description(description), aliases(aliases)
+    : name(name), description(stripIndentation(description)), aliases(aliases)
 {
 }
 
@@ -162,21 +163,23 @@ void AbstractSetting::setDefault(const std::string & str)
     if (!overriden) set(str);
 }
 
-void AbstractSetting::toJSON(JSONPlaceholder & out)
+nlohmann::json AbstractSetting::toJSON()
 {
-    out.write(to_string());
+    return nlohmann::json(toJSONObject());
+}
+
+std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject()
+{
+    std::map<std::string, nlohmann::json> obj;
+    obj.emplace("description", description);
+    obj.emplace("aliases", aliases);
+    return obj;
 }
 
 void AbstractSetting::convertToArg(Args & args, const std::string & category)
 {
 }
 
-template<typename T>
-void BaseSetting<T>::toJSON(JSONPlaceholder & out)
-{
-    out.write(value);
-}
-
 template<typename T>
 void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 {
@@ -255,13 +258,6 @@ template<> std::string BaseSetting<Strings>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out)
-{
-    JSONList list(out.list());
-    for (auto & s : value)
-        list.elem(s);
-}
-
 template<> void BaseSetting<StringSet>::set(const std::string & str)
 {
     value = tokenizeString<StringSet>(str);
@@ -272,13 +268,6 @@ template<> std::string BaseSetting<StringSet>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out)
-{
-    JSONList list(out.list());
-    for (auto & s : value)
-        list.elem(s);
-}
-
 template class BaseSetting<int>;
 template class BaseSetting<unsigned int>;
 template class BaseSetting<long>;
@@ -323,10 +312,12 @@ void GlobalConfig::resetOverriden()
         config->resetOverriden();
 }
 
-void GlobalConfig::toJSON(JSONObject & out)
+nlohmann::json GlobalConfig::toJSON()
 {
+    auto res = nlohmann::json::object();
     for (auto & config : *configRegistrations)
-        config->toJSON(out);
+        res.update(config->toJSON());
+    return res;
 }
 
 void GlobalConfig::convertToArgs(Args & args, const std::string & category)

src/libutil/config.hh

@@ -4,6 +4,8 @@
 
 #include "types.hh"
 
+#include <nlohmann/json_fwd.hpp>
+
 #pragma once
 
 namespace nix {
@@ -42,8 +44,6 @@ namespace nix {
 
 class Args;
 class AbstractSetting;
-class JSONPlaceholder;
-class JSONObject;
 
 class AbstractConfig
 {
@@ -97,7 +97,7 @@ public:
      * Outputs all settings to JSON
      * - out: JSONObject to write the configuration to
      */
-    virtual void toJSON(JSONObject & out) = 0;
+    virtual nlohmann::json toJSON() = 0;
 
     /**
      * Converts settings to `Args` to be used on the command line interface
@@ -167,7 +167,7 @@ public:
 
     void resetOverriden() override;
 
-    void toJSON(JSONObject & out) override;
+    nlohmann::json toJSON() override;
 
     void convertToArgs(Args & args, const std::string & category) override;
 };
@@ -206,7 +206,9 @@ protected:
 
     virtual std::string to_string() const = 0;
 
-    virtual void toJSON(JSONPlaceholder & out);
+    nlohmann::json toJSON();
+
+    virtual std::map<std::string, nlohmann::json> toJSONObject();
 
     virtual void convertToArg(Args & args, const std::string & category);
 
@@ -220,6 +222,7 @@ class BaseSetting : public AbstractSetting
 protected:
 
     T value;
+    const T defaultValue;
 
 public:
 
@@ -229,6 +232,7 @@ public:
         const std::set<std::string> & aliases = {})
         : AbstractSetting(name, description, aliases)
         , value(def)
+        , defaultValue(def)
     { }
 
     operator const T &() const { return value; }
@@ -251,7 +255,7 @@ public:
 
     void convertToArg(Args & args, const std::string & category) override;
 
-    void toJSON(JSONPlaceholder & out) override;
+    std::map<std::string, nlohmann::json> toJSONObject() override;
 };
 
 template<typename T>
@@ -319,7 +323,7 @@ struct GlobalConfig : public AbstractConfig
 
     void resetOverriden() override;
 
-    void toJSON(JSONObject & out) override;
+    nlohmann::json toJSON() override;
 
     void convertToArgs(Args & args, const std::string & category) override;
 

src/libutil/fmt.hh

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <boost/format.hpp>
-#include <boost/algorithm/string/replace.hpp>
 #include <string>
 #include "ansicolor.hh"
 

src/libutil/logging.cc

@@ -184,6 +184,33 @@ struct JSONLogger : Logger {
         json["action"] = "msg";
         json["level"] = ei.level;
         json["msg"] = oss.str();
+        json["raw_msg"] = ei.hint->str();
+
+        if (ei.errPos.has_value() && (*ei.errPos)) {
+            json["line"] = ei.errPos->line;
+            json["column"] = ei.errPos->column;
+            json["file"] = ei.errPos->file;
+        } else {
+            json["line"] = nullptr;
+            json["column"] = nullptr;
+            json["file"] = nullptr;
+        }
+
+        if (loggerSettings.showTrace.get() && !ei.traces.empty()) {
+            nlohmann::json traces = nlohmann::json::array();
+            for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) {
+                nlohmann::json stackFrame;
+                stackFrame["raw_msg"] = iter->hint.str();
+                if (iter->pos.has_value() && (*iter->pos)) {
+                    stackFrame["line"] = iter->pos->line;
+                    stackFrame["column"] = iter->pos->column;
+                    stackFrame["file"] = iter->pos->file;
+                }
+                traces.push_back(stackFrame);
+            }
+
+            json["trace"] = traces;
+        }
 
         write(json);
     }

src/libutil/logging.hh

@@ -37,10 +37,12 @@ typedef uint64_t ActivityId;
 
 struct LoggerSettings : Config
 {
-    Setting<bool> showTrace{this,
-        false,
-        "show-trace",
-        "Whether to show a stack trace on evaluation errors."};
+    Setting<bool> showTrace{
+        this, false, "show-trace",
+        R"(
+          Where Nix should print out a stack trace in case of Nix
+          expression evaluation errors.
+        )"};
 };
 
 extern LoggerSettings loggerSettings;

src/libutil/serialise.hh

@@ -22,8 +22,15 @@ struct Sink
     }
 };
 
+/* Just throws away data. */
+struct NullSink : Sink
+{
+    void operator () (const unsigned char * data, size_t len) override
+    { }
+};
 
-/* A buffered abstract sink. */
+/* A buffered abstract sink. Warning: a BufferedSink should not be
+   used from multiple threads concurrently. */
 struct BufferedSink : virtual Sink
 {
     size_t bufSize, bufPos;
@@ -66,7 +73,8 @@ struct Source
 };
 
 
-/* A buffered abstract source. */
+/* A buffered abstract source. Warning: a BufferedSource should not be
+   used from multiple threads concurrently. */
 struct BufferedSource : Source
 {
     size_t bufSize, bufPosIn, bufPosOut;

src/libutil/tests/config.cc

@@ -1,9 +1,9 @@
-#include "json.hh"
 #include "config.hh"
 #include "args.hh"
 
 #include <sstream>
 #include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -33,7 +33,7 @@ namespace nix {
         const auto iter = settings.find("name-of-the-setting");
         ASSERT_NE(iter, settings.end());
         ASSERT_EQ(iter->second.value, "");
-        ASSERT_EQ(iter->second.description, "description");
+        ASSERT_EQ(iter->second.description, "description\n");
     }
 
     TEST(Config, getDefinedOverridenSettingNotSet) {
@@ -59,7 +59,7 @@ namespace nix {
         const auto iter = settings.find("name-of-the-setting");
         ASSERT_NE(iter, settings.end());
         ASSERT_EQ(iter->second.value, "value");
-        ASSERT_EQ(iter->second.description, "description");
+        ASSERT_EQ(iter->second.description, "description\n");
     }
 
     TEST(Config, getDefinedSettingSet2) {
@@ -73,7 +73,7 @@ namespace nix {
         const auto e = settings.find("name-of-the-setting");
         ASSERT_NE(e, settings.end());
         ASSERT_EQ(e->second.value, "value");
-        ASSERT_EQ(e->second.description, "description");
+        ASSERT_EQ(e->second.description, "description\n");
     }
 
     TEST(Config, addSetting) {
@@ -152,29 +152,16 @@ namespace nix {
     }
 
     TEST(Config, toJSONOnEmptyConfig) {
-        std::stringstream out;
-        { // Scoped to force the destructor of JSONObject to write the final `}`
-            JSONObject obj(out);
-            Config config;
-            config.toJSON(obj);
-        }
-
-        ASSERT_EQ(out.str(), "{}");
+        ASSERT_EQ(Config().toJSON().dump(), "{}");
     }
 
     TEST(Config, toJSONOnNonEmptyConfig) {
-        std::stringstream out;
-        { // Scoped to force the destructor of JSONObject to write the final `}`
-            JSONObject obj(out);
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+        setting.assign("value");
 
-            Config config;
-            std::map<std::string, Config::SettingInfo> settings;
-            Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
-            setting.assign("value");
-
-            config.toJSON(obj);
-        }
-        ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#");
+        ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#");
     }
 
     TEST(Config, setSettingAlias) {

src/libutil/tests/logging.cc

@@ -34,6 +34,24 @@ namespace nix {
         }
     }
 
+    TEST(logEI, jsonOutput) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create("random.nix");
+        testing::internal::CaptureStderr();
+
+        makeJSONLogger(*logger)->logEI({
+                .name = "error name",
+                .description = "error without any code lines.",
+                .hint = hintfmt("this hint has %1% templated %2%!!",
+                    "yellow",
+                    "values"),
+                .errPos = Pos(foFile, problem_file, 02, 13)
+            });
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
+    }
+
     TEST(logEI, appendingHintsToPreviousError) {
 
         MakeError(TestError, Error);

src/libutil/url-parts.hh

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <string>
+#include <regex>
+
+namespace nix {
+
+// URI stuff.
+const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
+const static std::string schemeRegex = "(?:[a-z+.-]+)";
+const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
+const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
+const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
+const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
+const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
+const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
+const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
+const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
+const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
+const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
+const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
+const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
+
+// A Git ref (i.e. branch or tag name).
+const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
+extern std::regex refRegex;
+
+// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
+// This is because of the definition of a ref in refs.c in https://github.com/git/git
+// See tests/fetchGitRefs.sh for the full definition
+const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
+extern std::regex badGitRefRegex;
+
+// A Git revision (a SHA-1 commit hash).
+const static std::string revRegexS = "[0-9a-fA-F]{40}";
+extern std::regex revRegex;
+
+// A ref or revision, or a ref followed by a revision.
+const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
+
+const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
+extern std::regex flakeIdRegex;
+
+}

src/libutil/url.cc

@@ -1,4 +1,5 @@
 #include "url.hh"
+#include "url-parts.hh"
 #include "util.hh"
 
 namespace nix {

src/libutil/url.hh

@@ -2,8 +2,6 @@
 
 #include "error.hh"
 
-#include <regex>
-
 namespace nix {
 
 struct ParsedURL
@@ -29,40 +27,4 @@ std::map<std::string, std::string> decodeQuery(const std::string & query);
 
 ParsedURL parseURL(const std::string & url);
 
-// URI stuff.
-const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
-const static std::string schemeRegex = "(?:[a-z+]+)";
-const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
-const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
-const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
-const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
-const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
-const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
-const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
-const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
-const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
-const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
-const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
-const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
-
-// A Git ref (i.e. branch or tag name).
-const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
-extern std::regex refRegex;
-
-// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
-// This is because of the definition of a ref in refs.c in https://github.com/git/git
-// See tests/fetchGitRefs.sh for the full definition
-const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
-extern std::regex badGitRefRegex;
-
-// A Git revision (a SHA-1 commit hash).
-const static std::string revRegexS = "[0-9a-fA-F]{40}";
-extern std::regex revRegex;
-
-// A ref or revision, or a ref followed by a revision.
-const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
-
-const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
-extern std::regex flakeIdRegex;
-
 }

src/libutil/util.cc

@@ -1464,6 +1464,47 @@ string base64Decode(std::string_view s)
 }
 
 
+std::string stripIndentation(std::string_view s)
+{
+    size_t minIndent = 10000;
+    size_t curIndent = 0;
+    bool atStartOfLine = true;
+
+    for (auto & c : s) {
+        if (atStartOfLine && c == ' ')
+            curIndent++;
+        else if (c == '\n') {
+            if (atStartOfLine)
+                minIndent = std::max(minIndent, curIndent);
+            curIndent = 0;
+            atStartOfLine = true;
+        } else {
+            if (atStartOfLine) {
+                minIndent = std::min(minIndent, curIndent);
+                atStartOfLine = false;
+            }
+        }
+    }
+
+    std::string res;
+
+    size_t pos = 0;
+    while (pos < s.size()) {
+        auto eol = s.find('\n', pos);
+        if (eol == s.npos) eol = s.size();
+        if (eol - pos > minIndent)
+            res.append(s.substr(pos + minIndent, eol - pos - minIndent));
+        res.push_back('\n');
+        pos = eol + 1;
+    }
+
+    return res;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 
 

src/libutil/util.hh

@@ -12,13 +12,9 @@
 #include <signal.h>
 
 #include <functional>
-#include <limits>
-#include <cstdio>
 #include <map>
 #include <sstream>
 #include <optional>
-#include <future>
-#include <iterator>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0
@@ -464,6 +460,12 @@ string base64Encode(std::string_view s);
 string base64Decode(std::string_view s);
 
 
+/* Remove common leading whitespace from the lines in the string
+   's'. For example, if every line is indented by at least 3 spaces,
+   then we remove 3 spaces from the start of every line. */
+std::string stripIndentation(std::string_view s);
+
+
 /* Get a value for the specified key from an associate container. */
 template <class T>
 std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
@@ -474,43 +476,8 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_
 }
 
 
-/* A callback is a wrapper around a lambda that accepts a valid of
-   type T or an exception. (We abuse std::future<T> to pass the value or
-   exception.) */
 template<typename T>
-class Callback
-{
-    std::function<void(std::future<T>)> fun;
-    std::atomic_flag done = ATOMIC_FLAG_INIT;
-
-public:
-
-    Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
-
-    Callback(Callback && callback) : fun(std::move(callback.fun))
-    {
-        auto prev = callback.done.test_and_set();
-        if (prev) done.test_and_set();
-    }
-
-    void operator()(T && t) noexcept
-    {
-        auto prev = done.test_and_set();
-        assert(!prev);
-        std::promise<T> promise;
-        promise.set_value(std::move(t));
-        fun(promise.get_future());
-    }
-
-    void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
-    {
-        auto prev = done.test_and_set();
-        assert(!prev);
-        std::promise<T> promise;
-        promise.set_exception(exc);
-        fun(promise.get_future());
-    }
-};
+class Callback;
 
 
 /* Start a thread that handles various signals. Also block those signals

src/nix-build/nix-build.cc

@@ -98,8 +98,8 @@ static void _main(int argc, char * * argv)
 
     // List of environment variables kept for --pure
     std::set<string> keepVars{
-        "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM",
-        "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL",
+        "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL",
+        "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL",
         "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"
     };
 
@@ -446,7 +446,7 @@ static void _main(int argc, char * * argv)
                 "PATH=%4%:\"$PATH\"; "
                 "SHELL=%5%; "
                 "set +e; "
-                R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
+                R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
                 "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                 "unset NIX_ENFORCE_PURITY; "
                 "shopt -u nullglob; "
@@ -487,50 +487,56 @@ static void _main(int argc, char * * argv)
 
         std::vector<StorePathWithOutputs> pathsToBuild;
 
-        std::map<Path, Path> drvPrefixes;
-        std::map<Path, Path> resultSymlinks;
-        std::vector<Path> outPaths;
+        std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
 
         for (auto & drvInfo : drvs) {
-            auto drvPath = drvInfo.queryDrvPath();
-            auto outPath = drvInfo.queryOutPath();
+            auto drvPath = store->parseStorePath(drvInfo.queryDrvPath());
 
             auto outputName = drvInfo.queryOutputName();
             if (outputName == "")
-                throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath);
+                throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
-            pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}});
+            pathsToBuild.push_back({drvPath, {outputName}});
 
-            std::string drvPrefix;
-            auto i = drvPrefixes.find(drvPath);
-            if (i != drvPrefixes.end())
-                drvPrefix = i->second;
+            auto i = drvMap.find(drvPath);
+            if (i != drvMap.end())
+                i->second.second.insert(outputName);
             else {
-                drvPrefix = outLink;
-                if (drvPrefixes.size())
-                    drvPrefix += fmt("-%d", drvPrefixes.size() + 1);
-                drvPrefixes[drvPath] = drvPrefix;
+                drvMap[drvPath] = {drvMap.size(), {outputName}};
             }
-
-            std::string symlink = drvPrefix;
-            if (outputName != "out") symlink += "-" + outputName;
-
-            resultSymlinks[symlink] = outPath;
-            outPaths.push_back(outPath);
         }
 
         buildPaths(pathsToBuild);
 
         if (dryRun) return;
 
-        for (auto & symlink : resultSymlinks)
-            if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true);
+        std::vector<StorePath> outPaths;
+
+        for (auto & [drvPath, info] : drvMap) {
+            auto & [counter, wantedOutputs] = info;
+            std::string drvPrefix = outLink;
+            if (counter)
+                drvPrefix += fmt("-%d", counter + 1);
+
+            auto builtOutputs = store->queryDerivationOutputMap(drvPath);
+
+            for (auto & outputName : wantedOutputs) {
+                auto outputPath = builtOutputs.at(outputName);
+
+                if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
+                    std::string symlink = drvPrefix;
+                    if (outputName != "out") symlink += "-" + outputName;
+                    store2->addPermRoot(outputPath, absPath(symlink));
+                }
+
+                outPaths.push_back(outputPath);
+            }
+        }
 
         logger->stop();
 
         for (auto & path : outPaths)
-            std::cout << path << '\n';
+            std::cout << store->printStorePath(path) << '\n';
     }
 }
 

src/nix-channel/nix-channel.cc

@@ -76,6 +76,13 @@ static void update(const StringSet & channelNames)
 
     auto store = openStore();
 
+    auto [fd, unpackChannelPath] = createTempFile();
+    writeFull(fd.get(),
+        #include "unpack-channel.nix.gen.hh"
+        );
+    fd = -1;
+    AutoDelete del(unpackChannelPath, false);
+
     // Download each channel.
     Strings exprs;
     for (const auto & channel : channels) {
@@ -104,7 +111,7 @@ static void update(const StringSet & channelNames)
 
         bool unpacked = false;
         if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
-            runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> "
+            runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath +
                         "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
             unpacked = true;
         }
@@ -125,7 +132,7 @@ static void update(const StringSet & channelNames)
     // Unpack the channel tarballs into the Nix store and install them
     // into the channels profile.
     std::cerr << "unpacking channels...\n";
-    Strings envArgs{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" };
+    Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--from-expression" };
     for (auto & expr : exprs)
         envArgs.push_back(std::move(expr));
     envArgs.push_back("--quiet");

src/nix-channel/unpack-channel.nix

@@ -0,0 +1,12 @@
+{ name, channelName, src }:
+
+derivation {
+  builder = "builtin:unpack-channel";
+
+  system = "builtin";
+
+  inherit name channelName src;
+
+  # No point in doing this remotely.
+  preferLocalBuild = true;
+}

src/nix-daemon/nix-daemon.cc

@@ -1,5 +1,6 @@
 #include "shared.hh"
 #include "local-store.hh"
+#include "remote-store.hh"
 #include "util.hh"
 #include "serialise.hh"
 #include "archive.hh"
@@ -285,44 +286,28 @@ static int _main(int argc, char * * argv)
         initPlugins();
 
         if (stdio) {
-            if (getStoreType() == tDaemon) {
-                //  Forward on this connection to the real daemon
-                auto socketPath = settings.nixDaemonSocketFile;
-                auto s = socket(PF_UNIX, SOCK_STREAM, 0);
-                if (s == -1)
-                    throw SysError("creating Unix domain socket");
+            if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) {
+                auto conn = store->openConnectionWrapper();
+                int from = conn->from.fd;
+                int to = conn->to.fd;
 
-                auto socketDir = dirOf(socketPath);
-                if (chdir(socketDir.c_str()) == -1)
-                    throw SysError("changing to socket directory '%1%'", socketDir);
-
-                auto socketName = std::string(baseNameOf(socketPath));
-                auto addr = sockaddr_un{};
-                addr.sun_family = AF_UNIX;
-                if (socketName.size() + 1 >= sizeof(addr.sun_path))
-                    throw Error("socket name %1% is too long", socketName);
-                strcpy(addr.sun_path, socketName.c_str());
-
-                if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1)
-                    throw SysError("cannot connect to daemon at %1%", socketPath);
-
-                auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1;
+                auto nfds = std::max(from, STDIN_FILENO) + 1;
                 while (true) {
                     fd_set fds;
                     FD_ZERO(&fds);
-                    FD_SET(s, &fds);
+                    FD_SET(from, &fds);
                     FD_SET(STDIN_FILENO, &fds);
                     if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
                         throw SysError("waiting for data from client or server");
-                    if (FD_ISSET(s, &fds)) {
-                        auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
+                    if (FD_ISSET(from, &fds)) {
+                        auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
                         if (res == -1)
                             throw SysError("splicing data from daemon socket to stdout");
                         else if (res == 0)
                             throw EndOfFile("unexpected EOF from daemon socket");
                     }
                     if (FD_ISSET(STDIN_FILENO, &fds)) {
-                        auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
+                        auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
                         if (res == -1)
                             throw SysError("splicing data from stdin to daemon socket");
                         else if (res == 0)

src/nix-env/nix-env.cc

@@ -230,7 +230,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
 {
     DrvNames selectors = drvNamesFromArgs(args);
     if (selectors.empty())
-        selectors.push_back(DrvName("*"));
+        selectors.emplace_back("*");
 
     DrvInfos elems;
     set<unsigned int> done;
@@ -708,7 +708,9 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     }
 
     debug(format("switching to new user environment"));
-    Path generation = createGeneration(ref<LocalFSStore>(store2), globals.profile, drv.queryOutPath());
+    Path generation = createGeneration(
+        ref<LocalFSStore>(store2), globals.profile,
+        store2->parseStorePath(drv.queryOutPath()));
     switchLink(globals.profile, generation);
 }
 

src/nix-env/user-env.cc

@@ -151,7 +151,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
         }
 
         debug(format("switching to new user environment"));
-        Path generation = createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut);
+        Path generation = createGeneration(ref<LocalFSStore>(store2), profile,
+            store2->parseStorePath(topLevelOut));
         switchLink(profile, generation);
     }
 

src/nix-instantiate/nix-instantiate.cc

@@ -20,7 +20,6 @@ using namespace nix;
 
 static Path gcRoot;
 static int rootNr = 0;
-static bool indirectRoot = false;
 
 
 enum OutputKind { okPlain, okXML, okJSON };
@@ -71,11 +70,11 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 if (gcRoot == "")
                     printGCWarning();
                 else {
-                    Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot;
+                    Path rootName = absPath(gcRoot);
                     if (++rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
                     if (store2)
-                        drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, indirectRoot);
+                        drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName);
                 }
                 std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : ""));
             }
@@ -127,7 +126,7 @@ static int _main(int argc, char * * argv)
             else if (*arg == "--add-root")
                 gcRoot = getArg(*arg, arg, end);
             else if (*arg == "--indirect")
-                indirectRoot = true;
+                ;
             else if (*arg == "--xml")
                 outputKind = okXML;
             else if (*arg == "--json")

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -57,6 +57,7 @@ static int _main(int argc, char * * argv)
         bool fromExpr = false;
         string attrPath;
         bool unpack = false;
+        bool executable = false;
         string name;
 
         struct MyArgs : LegacyArgs, MixEvalArgs
@@ -81,6 +82,8 @@ static int _main(int argc, char * * argv)
             }
             else if (*arg == "--unpack")
                 unpack = true;
+            else if (*arg == "--executable")
+                executable = true;
             else if (*arg == "--name")
                 name = getArg(*arg, arg, end);
             else if (*arg != "" && arg->at(0) == '-')
@@ -175,7 +178,11 @@ static int _main(int argc, char * * argv)
 
             /* Download the file. */
             {
-                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
+                auto mode = 0600;
+                if (executable)
+                    mode = 0700;
+
+                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
                 if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
 
                 FdSink sink(fd.get());
@@ -201,7 +208,7 @@ static int _main(int argc, char * * argv)
                     tmpFile = unpacked;
             }
 
-            const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
 
             auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
             storePath = info.path;

src/nix-store/nix-store.cc

@@ -34,7 +34,6 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs);
 
 static Path gcRoot;
 static int rootNr = 0;
-static bool indirectRoot = false;
 static bool noOutput = false;
 static std::shared_ptr<Store> store;
 
@@ -65,6 +64,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
 
     if (path.path.isDerivation()) {
         if (build) store->buildPaths({path});
+        auto outputPaths = store->queryDerivationOutputMap(path.path);
         Derivation drv = store->derivationFromPath(path.path);
         rootNr++;
 
@@ -77,7 +77,8 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
             if (i == drv.outputs.end())
                 throw Error("derivation '%s' does not have an output named '%s'",
                     store2->printStorePath(path.path), j);
-            auto outPath = store2->printStorePath(i->second.path(*store, drv.name));
+            auto outPath = outputPaths.at(i->first);
+            auto retPath = store->printStorePath(outPath);
             if (store2) {
                 if (gcRoot == "")
                     printGCWarning();
@@ -85,10 +86,10 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
                     Path rootName = gcRoot;
                     if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     if (i->first != "out") rootName += "-" + i->first;
-                    outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot);
+                    retPath = store2->addPermRoot(outPath, rootName);
                 }
             }
-            outputs.insert(outPath);
+            outputs.insert(retPath);
         }
         return outputs;
     }
@@ -104,7 +105,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
                 Path rootName = gcRoot;
                 rootNr++;
                 if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
-                return {store2->addPermRoot(path.path, rootName, indirectRoot)};
+                return {store2->addPermRoot(path.path, rootName)};
             }
         }
         return {store->printStorePath(path.path)};
@@ -218,8 +219,13 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
     if (useOutput && storePath.isDerivation()) {
         auto drv = store->derivationFromPath(storePath);
         StorePathSet outputs;
-        for (auto & i : drv.outputsAndPaths(*store))
-            outputs.insert(i.second.second);
+        if (forceRealise)
+            return store->queryDerivationOutputs(storePath);
+        for (auto & i : drv.outputsAndOptPaths(*store)) {
+            if (!i.second.second)
+                throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
+            outputs.insert(*i.second.second);
+        }
         return outputs;
     }
     else return {storePath};
@@ -309,11 +315,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
 
         case qOutputs: {
             for (auto & i : opArgs) {
-                auto i2 = store->followLinksToStorePath(i);
-                if (forceRealise) realisePath({i2});
-                Derivation drv = store->derivationFromPath(i2);
-                for (auto & j : drv.outputsAndPaths(*store))
-                    cout << fmt("%1%\n", store->printStorePath(j.second.second));
+                auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise);
+                for (auto & outputPath : outputs)
+                    cout << fmt("%1%\n", store->printStorePath(outputPath));
             }
             break;
         }
@@ -1085,7 +1089,7 @@ static int _main(int argc, char * * argv)
             else if (*arg == "--add-root")
                 gcRoot = absPath(getArg(*arg, arg, end));
             else if (*arg == "--indirect")
-                indirectRoot = true;
+                ;
             else if (*arg == "--no-output")
                 noOutput = true;
             else if (*arg != "" && arg->at(0) == '-') {

src/nix/add-to-store.cc

@@ -36,6 +36,14 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         return "add a path to the Nix store";
     }
 
+    std::string doc() override
+    {
+        return R"(
+          Copy the file or directory *path* to the Nix store, and
+          print the resulting store path on standard output.
+        )";
+    }
+
     Examples examples() override
     {
         return {

src/nix/build.cc

@@ -71,14 +71,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
                         [&](BuildableOpaque bo) {
                             std::string symlink = outLink;
                             if (i) symlink += fmt("-%d", i);
-                            store2->addPermRoot(bo.path, absPath(symlink), true);
+                            store2->addPermRoot(bo.path, absPath(symlink));
                         },
                         [&](BuildableFromDrv bfd) {
-                            for (auto & output : bfd.outputs) {
+                            auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
+                            for (auto & output : builtOutputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
                                 if (output.first != "out") symlink += fmt("-%s", output.first);
-                                store2->addPermRoot(output.second, absPath(symlink), true);
+                                store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
                     }, buildables[i]);

src/nix/bundle.cc

@@ -98,14 +98,14 @@ struct CmdBundle : InstallableCommand
         if (!evalState->isDerivation(*vRes))
             throw Error("the bundler '%s' does not produce a derivation", bundler.what());
 
-        auto attr1 = vRes->attrs->find(evalState->sDrvPath);
+        auto attr1 = vRes->attrs->get(evalState->sDrvPath);
         if (!attr1)
             throw Error("the bundler '%s' does not produce a derivation", bundler.what());
 
         PathSet context2;
         StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2));
 
-        auto attr2 = vRes->attrs->find(evalState->sOutPath);
+        auto attr2 = vRes->attrs->get(evalState->sOutPath);
         if (!attr2)
             throw Error("the bundler '%s' does not produce a derivation", bundler.what());
 
@@ -122,7 +122,7 @@ struct CmdBundle : InstallableCommand
         if (!outLink)
             outLink = baseNameOf(app.program);
 
-        store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink), true);
+        store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink));
     }
 };
 

src/nix/command.cc

@@ -4,12 +4,25 @@
 #include "nixexpr.hh"
 #include "profiles.hh"
 
+#include <nlohmann/json.hpp>
+
 extern char * * environ __attribute__((weak));
 
 namespace nix {
 
 Commands * RegisterCommand::commands = nullptr;
 
+void NixMultiCommand::printHelp(const string & programName, std::ostream & out)
+{
+    MultiCommand::printHelp(programName, out);
+}
+
+nlohmann::json NixMultiCommand::toJSON()
+{
+    // FIXME: use Command::toJSON() as well.
+    return MultiCommand::toJSON();
+}
+
 StoreCommand::StoreCommand()
 {
 }
@@ -121,7 +134,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
     switchLink(profile2,
         createGeneration(
             ref<LocalFSStore>(store),
-            profile2, store->printStorePath(storePath)));
+            profile2, storePath));
 }
 
 void MixProfile::updateProfile(const Buildables & buildables)
@@ -137,7 +150,10 @@ void MixProfile::updateProfile(const Buildables & buildables)
             },
             [&](BuildableFromDrv bfd) {
                 for (auto & output : bfd.outputs) {
-                    result.push_back(output.second);
+                    /* Output path should be known because we just tried to
+                       build it. */
+                    assert(!output.second);
+                    result.push_back(*output.second);
                 }
             },
         }, buildable);

src/nix/command.hh

@@ -21,6 +21,13 @@ static constexpr Command::Category catSecondary = 100;
 static constexpr Command::Category catUtility = 101;
 static constexpr Command::Category catNixInstallation = 102;
 
+struct NixMultiCommand : virtual MultiCommand, virtual Command
+{
+    void printHelp(const string & programName, std::ostream & out) override;
+
+    nlohmann::json toJSON() override;
+};
+
 /* A command that requires a Nix store. */
 struct StoreCommand : virtual Command
 {

src/nix/describe-stores.cc

@@ -0,0 +1,44 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+using namespace nix;
+
+struct CmdDescribeStores : Command, MixJSON
+{
+    std::string description() override
+    {
+        return "show registered store types and their available options";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+        auto res = nlohmann::json::object();
+        for (auto & implem : *Implementations::registered) {
+            auto storeConfig = implem.getConfig();
+            auto storeName = storeConfig->name();
+            res[storeName] = storeConfig->toJSON();
+        }
+        if (json) {
+            std::cout << res;
+        } else {
+            for (auto & [storeName, storeConfig] : res.items()) {
+                std::cout << "## " << storeName << std::endl << std::endl;
+                for (auto & [optionName, optionDesc] : storeConfig.items()) {
+                    std::cout << "### " << optionName << std::endl << std::endl;
+                    std::cout << optionDesc["description"].get<std::string>() << std::endl;
+                    std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl;
+                    if (!optionDesc["aliases"].empty())
+                        std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl;
+                }
+            }
+        }
+    }
+};
+
+static auto r1 = registerCommand<CmdDescribeStores>("describe-stores");

src/nix/develop.cc

@@ -15,7 +15,7 @@ struct Var
 {
     bool exported = true;
     bool associative = false;
-    std::string value; // quoted string or array
+    std::string quoted; // quoted string or array
 };
 
 struct BuildEnvironment
@@ -75,12 +75,12 @@ BuildEnvironment readEnvironment(const Path & path)
 
         else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
             pos = match[0].second;
-            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
+            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
         }
 
         else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
             pos = match[0].second;
-            res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
+            res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
         }
 
         else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
@@ -92,6 +92,8 @@ BuildEnvironment readEnvironment(const Path & path)
             path, file.substr(pos - file.cbegin(), 60));
     }
 
+    res.env.erase("__output");
+
     return res;
 }
 
@@ -125,27 +127,35 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
-    for (auto & output : drv.outputs)
-        drv.env.erase(output.first);
-    drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}};
-    drv.env["out"] = "";
-    drv.env["_outputs_saved"] = drv.env["outputs"];
-    drv.env["outputs"] = "out";
+    for (auto & output : drv.outputs) {
+        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+        drv.env[output.first] = "";
+    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
-    auto shellOutPath = store->makeOutputPath("out", h, drv.name);
-    drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
-                .path = shellOutPath
-            } });
-    drv.env["out"] = store->printStorePath(shellOutPath);
-    auto shellDrvPath2 = writeDerivation(store, drv);
+
+    for (auto & output : drv.outputs) {
+        auto outPath = store->makeOutputPath(output.first, h, drv.name);
+        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+        drv.env[output.first] = store->printStorePath(outPath);
+    }
+
+    auto shellDrvPath = writeDerivation(*store, drv);
 
     /* Build the derivation. */
-    store->buildPaths({{shellDrvPath2}});
+    store->buildPaths({{shellDrvPath}});
 
-    assert(store->isValidPath(shellOutPath));
+    for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
+        auto & [_1, optPath] = outputAndOptPath;
+        assert(optPath);
+        auto & outPath = *optPath;
+        assert(store->isValidPath(outPath));
+        auto outPathS = store->toRealPath(outPath);
+        if (lstat(outPathS).st_size)
+            return outPath;
+    }
 
-    return shellOutPath;
+    throw Error("get-env.sh failed to produce an environment");
 }
 
 struct Common : InstallableCommand, MixProfile
@@ -171,8 +181,12 @@ struct Common : InstallableCommand, MixProfile
         "UID",
     };
 
-    void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
+    std::string makeRcScript(
+        const BuildEnvironment & buildEnvironment,
+        const Path & outputsDir = absPath(".") + "/outputs")
     {
+        std::ostringstream out;
+
         out << "unset shellHook\n";
 
         out << "nix_saved_PATH=\"$PATH\"\n";
@@ -180,9 +194,9 @@ struct Common : InstallableCommand, MixProfile
         for (auto & i : buildEnvironment.env) {
             if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
                 if (i.second.associative)
-                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.value);
+                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted);
                 else {
-                    out << fmt("%s=%s\n", i.first, i.second.value);
+                    out << fmt("%s=%s\n", i.first, i.second.quoted);
                     if (i.second.exported)
                         out << fmt("export %s\n", i.first);
                 }
@@ -193,13 +207,26 @@ struct Common : InstallableCommand, MixProfile
 
         out << buildEnvironment.bashFunctions << "\n";
 
-        // FIXME: set outputs
-
         out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
 
         out << "eval \"$shellHook\"\n";
+
+        /* Substitute occurrences of output paths. */
+        auto outputs = buildEnvironment.env.find("outputs");
+        assert(outputs != buildEnvironment.env.end());
+
+        // FIXME: properly unquote 'outputs'.
+        StringMap rewrites;
+        for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) {
+            auto from = buildEnvironment.env.find(outputName);
+            assert(from != buildEnvironment.env.end());
+            // FIXME: unquote
+            rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
+        }
+
+        return rewriteStrings(out.str(), rewrites);
     }
 
     Strings getDefaultFlakeAttrPaths() override
@@ -240,6 +267,7 @@ struct Common : InstallableCommand, MixProfile
 struct CmdDevelop : Common, MixEnvironment
 {
     std::vector<std::string> command;
+    std::optional<std::string> phase;
 
     CmdDevelop()
     {
@@ -253,6 +281,43 @@ struct CmdDevelop : Common, MixEnvironment
                 command = ss;
             }}
         });
+
+        addFlag({
+            .longName = "phase",
+            .description = "phase to run (e.g. `build` or `configure`)",
+            .labels = {"phase-name"},
+            .handler = {&phase},
+        });
+
+        addFlag({
+            .longName = "configure",
+            .description = "run the configure phase",
+            .handler = {&phase, {"configure"}},
+        });
+
+        addFlag({
+            .longName = "build",
+            .description = "run the build phase",
+            .handler = {&phase, {"build"}},
+        });
+
+        addFlag({
+            .longName = "check",
+            .description = "run the check phase",
+            .handler = {&phase, {"check"}},
+        });
+
+        addFlag({
+            .longName = "install",
+            .description = "run the install phase",
+            .handler = {&phase, {"install"}},
+        });
+
+        addFlag({
+            .longName = "installcheck",
+            .description = "run the installcheck phase",
+            .handler = {&phase, {"installCheck"}},
+        });
     }
 
     std::string description() override
@@ -288,19 +353,31 @@ struct CmdDevelop : Common, MixEnvironment
 
         auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
 
-        std::ostringstream ss;
-        makeRcScript(buildEnvironment, ss);
+        auto script = makeRcScript(buildEnvironment);
 
-        ss << fmt("rm -f '%s'\n", rcFilePath);
+        if (verbosity >= lvlDebug)
+            script += "set -x\n";
 
-        if (!command.empty()) {
+        script += fmt("rm -f '%s'\n", rcFilePath);
+
+        if (phase) {
+            if (!command.empty())
+                throw UsageError("you cannot use both '--command' and '--phase'");
+            // FIXME: foundMakefile is set by buildPhase, need to get
+            // rid of that.
+            script += fmt("foundMakefile=1\n");
+            script += fmt("runHook %1%Phase\n", *phase);
+            script += fmt("exit 0\n", *phase);
+        }
+
+        else if (!command.empty()) {
             std::vector<std::string> args;
             for (auto s : command)
                 args.push_back(shellEscape(s));
-            ss << fmt("exec %s\n", concatStringsSep(" ", args));
+            script += fmt("exec %s\n", concatStringsSep(" ", args));
         }
 
-        writeFull(rcFileFd.get(), ss.str());
+        writeFull(rcFileFd.get(), script);
 
         stopProgressBar();
 
@@ -315,7 +392,7 @@ struct CmdDevelop : Common, MixEnvironment
 
             auto bashInstallable = std::make_shared<InstallableFlake>(
                 state,
-                std::move(installable->nixpkgsFlakeRef()),
+                installable->nixpkgsFlakeRef(),
                 Strings{"bashInteractive"},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
                 lockFlags);
@@ -362,7 +439,7 @@ struct CmdPrintDevEnv : Common
 
         stopProgressBar();
 
-        makeRcScript(buildEnvironment, std::cout);
+        std::cout << makeRcScript(buildEnvironment);
     }
 };
 

src/nix/diff-closures.cc

@@ -81,7 +81,7 @@ void printClosureDiff(
         auto beforeSize = totalSize(beforeVersions);
         auto afterSize = totalSize(afterVersions);
         auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
-        auto showDelta = abs(sizeDelta) >= 8 * 1024;
+        auto showDelta = std::abs(sizeDelta) >= 8 * 1024;
 
         std::set<std::string> removed, unchanged;
         for (auto & [version, _] : beforeVersions)

src/nix/doctor.cc

@@ -49,9 +49,7 @@ struct CmdDoctor : StoreCommand
     {
         logger->log("Running checks against store uri: " + store->getUri());
 
-        auto type = getStoreType();
-
-        if (type < tOther) {
+        if (store.dynamic_pointer_cast<LocalFSStore>()) {
             success &= checkNixInPath();
             success &= checkProfileRoots(store);
         }

src/nix/flake.cc

@@ -933,7 +933,7 @@ struct CmdFlakeShow : FlakeCommand
     }
 };
 
-struct CmdFlake : virtual MultiCommand, virtual Command
+struct CmdFlake : NixMultiCommand
 {
     CmdFlake()
         : MultiCommand({
@@ -963,11 +963,6 @@ struct CmdFlake : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdFlake>("flake");

src/nix/get-env.sh

@@ -1,12 +1,6 @@
 set -e
 if [ -e .attrs.sh ]; then source .attrs.sh; fi
 
-outputs=$_outputs_saved
-for __output in $_outputs_saved; do
-    declare "$__output"="$out"
-done
-unset _outputs_saved __output
-
 export IN_NIX_SHELL=impure
 export dontAddDisableDepTrack=1
 
@@ -14,5 +8,12 @@ if [[ -n $stdenv ]]; then
     source $stdenv/setup
 fi
 
-export > $out
-set >> $out
+for __output in $outputs; do
+    if [[ -z $__done ]]; then
+        export > ${!__output}
+        set >> ${!__output}
+        __done=1
+    else
+        echo -n >> ${!__output}
+    fi
+done

src/nix/installables.cc

@@ -76,7 +76,7 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "override-input",
-        .description = "override a specific flake input (e.g. 'dwarffs/nixpkgs')",
+        .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)",
         .labels = {"input-path", "flake-url"},
         .handler = {[&](std::string inputPath, std::string flakeRef) {
             lockFlags.inputOverrides.insert_or_assign(
@@ -116,7 +116,7 @@ SourceExprCommand::SourceExprCommand()
     addFlag({
         .longName = "file",
         .shortName = 'f',
-        .description = "evaluate FILE rather than the default",
+        .description = "evaluate *file* rather than the default",
         .labels = {"file"},
         .handler = {&file},
         .completer = completePath
@@ -124,7 +124,7 @@ SourceExprCommand::SourceExprCommand()
 
     addFlag({
         .longName ="expr",
-        .description = "evaluate attributes from EXPR",
+        .description = "evaluate attributes from *expr*",
         .labels = {"expr"},
         .handler = {&expr}
     });
@@ -302,10 +302,10 @@ struct InstallableStorePath : Installable
     Buildables toBuildables() override
     {
         if (storePath.isDerivation()) {
-            std::map<std::string, StorePath> outputs;
+            std::map<std::string, std::optional<StorePath>> outputs;
             auto drv = store->readDerivation(storePath);
-            for (auto & i : drv.outputsAndPaths(*store))
-                outputs.emplace(i.first, i.second.second);
+            for (auto & [name, output] : drv.outputsAndOptPaths(*store))
+                outputs.emplace(name, output.second);
             return {
                 BuildableFromDrv {
                     .drvPath = storePath,
@@ -331,7 +331,7 @@ Buildables InstallableValue::toBuildables()
 {
     Buildables res;
 
-    std::map<StorePath, OutputPathMap> drvsToOutputs;
+    std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
 
     // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
@@ -674,8 +674,11 @@ StorePathSet toStorePaths(ref<Store> store,
                     outPaths.insert(bo.path);
                 },
                 [&](BuildableFromDrv bfd) {
-                    for (auto & output : bfd.outputs)
-                        outPaths.insert(output.second);
+                    for (auto & output : bfd.outputs) {
+                        if (!output.second)
+                            throw Error("Cannot operate on output of unbuilt CA drv");
+                        outPaths.insert(*output.second);
+                    }
                 },
             }, b);
     } else {

src/nix/installables.hh

@@ -20,7 +20,7 @@ struct BuildableOpaque {
 
 struct BuildableFromDrv {
     StorePath drvPath;
-    std::map<std::string, StorePath> outputs;
+    std::map<std::string, std::optional<StorePath>> outputs;
 };
 
 typedef std::variant<
@@ -69,7 +69,7 @@ struct Installable
 
     virtual FlakeRef nixpkgsFlakeRef() const
     {
-        return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}));
+        return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
     }
 };
 
@@ -82,7 +82,7 @@ struct InstallableValue : Installable
     struct DerivationInfo
     {
         StorePath drvPath;
-        StorePath outPath;
+        std::optional<StorePath> outPath;
         std::string outputName;
     };
 

src/nix/local.mk

@@ -19,7 +19,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -llowdown
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
@@ -29,3 +29,5 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
 src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
 
 src/nix/develop.cc: src/nix/get-env.sh.gen.hh
+
+src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh

src/nix/main.cc

@@ -17,6 +17,8 @@
 #include <netdb.h>
 #include <netinet/in.h>
 
+#include <nlohmann/json.hpp>
+
 extern std::string chrootHelperName;
 
 void chrootHelper(int argc, char * * argv);
@@ -140,6 +142,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         printHelp(programName, std::cout);
         throw Exit();
     }
+
+    std::string description() override
+    {
+        return "a tool for reproducible and declarative configuration management";
+    }
 };
 
 void mainWrapped(int argc, char * * argv)
@@ -172,6 +179,30 @@ void mainWrapped(int argc, char * * argv)
 
     NixArgs args;
 
+    if (argc == 2 && std::string(argv[1]) == "__dump-args") {
+        std::cout << args.toJSON().dump() << "\n";
+        return;
+    }
+
+    if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
+        evalSettings.pureEval = false;
+        EvalState state({}, openStore("dummy://"));
+        auto res = nlohmann::json::object();
+        auto builtins = state.baseEnv.values[0]->attrs;
+        for (auto & builtin : *builtins) {
+            auto b = nlohmann::json::object();
+            if (builtin.value->type != tPrimOp) continue;
+            auto primOp = builtin.value->primOp;
+            if (!primOp->doc) continue;
+            b["arity"] = primOp->arity;
+            b["args"] = primOp->args;
+            b["doc"] = trim(stripIndentation(primOp->doc));
+            res[(std::string) builtin.name] = std::move(b);
+        }
+        std::cout << res.dump() << "\n";
+        return;
+    }
+
     Finally printCompletions([&]()
     {
         if (completions) {

src/nix/markdown.cc

@@ -0,0 +1,50 @@
+#include "markdown.hh"
+#include "util.hh"
+#include "finally.hh"
+
+#include <sys/queue.h>
+extern "C" {
+#include <lowdown.h>
+}
+
+namespace nix {
+
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    struct lowdown_opts opts {
+        .type = LOWDOWN_TERM,
+        .maxdepth = 20,
+        .cols = std::min(getWindowSize().second, (unsigned short) 80),
+        .hmargin = 0,
+        .vmargin = 0,
+        .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
+        .oflags = 0,
+    };
+
+    auto doc = lowdown_doc_new(&opts);
+    if (!doc)
+        throw Error("cannot allocate Markdown document");
+    Finally freeDoc([&]() { lowdown_doc_free(doc); });
+
+    size_t maxn = 0;
+    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
+    if (!node)
+        throw Error("cannot parse Markdown document");
+    Finally freeNode([&]() { lowdown_node_free(node); });
+
+    auto renderer = lowdown_term_new(&opts);
+    if (!renderer)
+        throw Error("cannot allocate Markdown renderer");
+    Finally freeRenderer([&]() { lowdown_term_free(renderer); });
+
+    auto buf = lowdown_buf_new(16384);
+    if (!buf)
+        throw Error("cannot allocate Markdown output buffer");
+    Finally freeBuffer([&]() { lowdown_buf_free(buf); });
+
+    lowdown_term_rndr(buf, nullptr, renderer, node);
+
+    return std::string(buf->data, buf->size);
+}
+
+}

src/nix/markdown.hh

@@ -0,0 +1,7 @@
+#include "types.hh"
+
+namespace nix {
+
+std::string renderMarkdownToTerminal(std::string_view markdown);
+
+}

src/nix/profile.cc

@@ -180,7 +180,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                 auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
 
                 ProfileElement element;
-                element.storePaths = {drv.outPath}; // FIXME
+                if (!drv.outPath)
+                    throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
+                element.storePaths = {*drv.outPath}; // FIXME
                 element.source = ProfileElementSource{
                     installable2->flakeRef,
                     resolvedRef,
@@ -191,7 +193,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 
                 manifest.elements.emplace_back(std::move(element));
             } else
-                throw Error("'nix profile install' does not support argument '%s'", installable->what());
+                throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what());
         }
 
         store->buildPaths(pathsToBuild);
@@ -349,7 +351,9 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 printInfo("upgrading '%s' from flake '%s' to '%s'",
                     element.source->attrPath, element.source->resolvedRef, resolvedRef);
 
-                element.storePaths = {drv.outPath}; // FIXME
+                if (!drv.outPath)
+                    throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
+                element.storePaths = {*drv.outPath}; // FIXME
                 element.source = ProfileElementSource{
                     installable.flakeRef,
                     resolvedRef,
@@ -437,7 +441,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
     }
 };
 
-struct CmdProfile : virtual MultiCommand, virtual Command
+struct CmdProfile : NixMultiCommand
 {
     CmdProfile()
         : MultiCommand({
@@ -461,11 +465,6 @@ struct CmdProfile : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdProfile>("profile");

src/nix/registry.cc

@@ -115,7 +115,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand
     }
 };
 
-struct CmdRegistry : virtual MultiCommand, virtual Command
+struct CmdRegistry : virtual NixMultiCommand
 {
     CmdRegistry()
         : MultiCommand({
@@ -141,11 +141,6 @@ struct CmdRegistry : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdRegistry>("registry");

src/nix/repl.cc

@@ -32,6 +32,7 @@ extern "C" {
 #include "globals.hh"
 #include "command.hh"
 #include "finally.hh"
+#include "markdown.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -64,7 +65,7 @@ struct NixRepl
     void mainLoop(const std::vector<std::string> & files);
     StringSet completePrefix(string prefix);
     bool getLine(string & input, const std::string &prompt);
-    Path getDerivationPath(Value & v);
+    StorePath getDerivationPath(Value & v);
     bool processLine(string line);
     void loadFile(const Path & path);
     void initEnv();
@@ -375,13 +376,16 @@ bool isVarName(const string & s)
 }
 
 
-Path NixRepl::getDerivationPath(Value & v) {
+StorePath NixRepl::getDerivationPath(Value & v) {
     auto drvInfo = getDerivation(*state, v, false);
     if (!drvInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
-    Path drvPath = drvInfo->queryDrvPath();
-    if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath)))
-        throw Error("expression did not evaluate to a valid derivation");
+    Path drvPathRaw = drvInfo->queryDrvPath();
+    if (drvPathRaw == "")
+        throw Error("expression did not evaluate to a valid derivation (no drv path)");
+    StorePath drvPath = state->store->parseStorePath(drvPathRaw);
+    if (!state->store->isValidPath(drvPath))
+        throw Error("expression did not evaluate to a valid derivation (invalid drv path)");
     return drvPath;
 }
 
@@ -416,7 +420,8 @@ bool NixRepl::processLine(string line)
              << "  :r            Reload all files\n"
              << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
              << "  :t <expr>     Describe result of evaluation\n"
-             << "  :u <expr>     Build derivation, then start nix-shell\n";
+             << "  :u <expr>     Build derivation, then start nix-shell\n"
+             << "  :doc <expr>   Show documentation of a builtin function\n";
     }
 
     else if (command == ":a" || command == ":add") {
@@ -474,29 +479,30 @@ bool NixRepl::processLine(string line)
         evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);
         state->callFunction(f, v, result, Pos());
 
-        Path drvPath = getDerivationPath(result);
-        runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
+        StorePath drvPath = getDerivationPath(result);
+        runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)});
     }
 
     else if (command == ":b" || command == ":i" || command == ":s") {
         Value v;
         evalString(arg, v);
-        Path drvPath = getDerivationPath(v);
+        StorePath drvPath = getDerivationPath(v);
+        Path drvPathRaw = state->store->printStorePath(drvPath);
 
         if (command == ":b") {
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
-                auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
+            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) {
+                auto drv = state->store->readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
-                for (auto & i : drv.outputsAndPaths(*state->store))
-                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second.second));
+                for (auto & i : drv.outputsAndOptPaths(*state->store))
+                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(*i.second.second));
             }
         } else if (command == ":i") {
-            runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
+            runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw});
         } else {
-            runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
+            runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw});
         }
     }
 
@@ -509,6 +515,29 @@ bool NixRepl::processLine(string line)
     else if (command == ":q" || command == ":quit")
         return false;
 
+    else if (command == ":doc") {
+        Value v;
+        evalString(arg, v);
+        if (auto doc = state->getDoc(v)) {
+            std::string markdown;
+
+            if (!doc->args.empty() && doc->name) {
+                auto args = doc->args;
+                for (auto & arg : args)
+                    arg = "*" + arg + "*";
+
+                markdown +=
+                    "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` "
+                    + concatStringsSep(" ", args) + "\n\n";
+            }
+
+            markdown += trim(stripIndentation(doc->doc));
+
+            std::cout << renderMarkdownToTerminal(markdown);
+        } else
+            throw Error("value does not have documentation");
+    }
+
     else if (command != "")
         throw Error("unknown command '%1%'", command);
 
@@ -782,7 +811,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs
         return {
           Example{
             "Display all special commands within the REPL:",
-              "nix repl\n  nix-repl> :?"
+            "nix repl\nnix-repl> :?"
           }
         };
     }