Merge branch 'master' into tomberek.absolute.attrpath.notation

src/build-remote/build-remote.cc

@@ -314,7 +314,7 @@ connected:
             //
             // 2. Changing the `inputSrcs` set changes the associated
             //    output ids, which break CA derivations
-            if (!drv.inputDrvs.empty())
+            if (!drv.inputDrvs.map.empty())
                 drv.inputSrcs = store->parseStorePathSet(inputs);
             optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
             auto & result = *optResult;

src/libcmd/built-path.cc

@@ -58,6 +58,28 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
+SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const
+{
+    return SingleDerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(drvPath->discardOutputPath()),
+        .output = output.first,
+    };
+}
+
+SingleDerivedPath SingleBuiltPath::discardOutputPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath {
+                return p;
+            },
+            [](const SingleBuiltPath::Built & b) -> SingleDerivedPath {
+                return b.discardOutputPath();
+            },
+        }, raw()
+    );
+}
+
 nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
 {
     nlohmann::json res;

src/libcmd/built-path.hh

@@ -9,6 +9,8 @@ struct SingleBuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::pair<std::string, StorePath> output;
 
+    SingleDerivedPathBuilt discardOutputPath() const;
+
     std::string to_string(const Store & store) const;
     static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const Store & store) const;
@@ -34,6 +36,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
 
     StorePath outPath() const;
 
+    SingleDerivedPath discardOutputPath() const;
+
     static SingleBuiltPath parse(const Store & store, std::string_view);
     nlohmann::json toJSON(const Store & store) const;
 };

src/libcmd/repl.cc

@@ -488,35 +488,40 @@ bool NixRepl::processLine(std::string line)
         std::cout
              << "The following commands are available:\n"
              << "\n"
-             << "  <expr>        Evaluate and print expression\n"
-             << "  <x> = <expr>  Bind expression to variable\n"
-             << "  :a <expr>     Add attributes from resulting set to scope\n"
-             << "  :b <expr>     Build a derivation\n"
-             << "  :bl <expr>    Build a derivation, creating GC roots in the working directory\n"
-             << "  :e <expr>     Open package or function in $EDITOR\n"
-             << "  :i <expr>     Build derivation, then install result into current profile\n"
-             << "  :l <path>     Load Nix expression and add it to scope\n"
-             << "  :lf <ref>     Load Nix flake and add it to scope\n"
-             << "  :p <expr>     Evaluate and print expression recursively\n"
-             << "  :q            Exit nix-repl\n"
-             << "  :r            Reload all files\n"
-             << "  :sh <expr>    Build dependencies of derivation, then start nix-shell\n"
-             << "  :t <expr>     Describe result of evaluation\n"
-             << "  :u <expr>     Build derivation, then start nix-shell\n"
-             << "  :doc <expr>   Show documentation of a builtin function\n"
-             << "  :log <expr>   Show logs for a derivation\n"
-             << "  :te [bool]    Enable, disable or toggle showing traces for errors\n"
+             << "  <expr>                       Evaluate and print expression\n"
+             << "  <x> = <expr>                 Bind expression to variable\n"
+             << "  :a, :add <expr>              Add attributes from resulting set to scope\n"
+             << "  :b <expr>                    Build a derivation\n"
+             << "  :bl <expr>                   Build a derivation, creating GC roots in the\n"
+             << "                               working directory\n"
+             << "  :e, :edit <expr>             Open package or function in $EDITOR\n"
+             << "  :i <expr>                    Build derivation, then install result into\n"
+             << "                               current profile\n"
+             << "  :l, :load <path>             Load Nix expression and add it to scope\n"
+             << "  :lf, :load-flake <ref>       Load Nix flake and add it to scope\n"
+             << "  :p, :print <expr>            Evaluate and print expression recursively\n"
+             << "  :q, :quit                    Exit nix-repl\n"
+             << "  :r, :reload                  Reload all files\n"
+             << "  :sh <expr>                   Build dependencies of derivation, then start\n"
+             << "                               nix-shell\n"
+             << "  :t <expr>                    Describe result of evaluation\n"
+             << "  :u <expr>                    Build derivation, then start nix-shell\n"
+             << "  :doc <expr>                  Show documentation of a builtin function\n"
+             << "  :log <expr>                  Show logs for a derivation\n"
+             << "  :te, :trace-enable [bool]    Enable, disable or toggle showing traces for\n"
+             << "                               errors\n"
+             << "  :?, :help                    Brings up this help menu\n"
              ;
         if (state->debugRepl) {
              std::cout
              << "\n"
              << "        Debug mode commands\n"
-             << "  :env          Show env stack\n"
-             << "  :bt           Show trace stack\n"
-             << "  :st           Show current trace\n"
-             << "  :st <idx>     Change to another trace in the stack\n"
-             << "  :c            Go until end of program, exception, or builtins.break\n"
-             << "  :s            Go one step\n"
+             << "  :env             Show env stack\n"
+             << "  :bt, :backtrace  Show trace stack\n"
+             << "  :st              Show current trace\n"
+             << "  :st <idx>        Change to another trace in the stack\n"
+             << "  :c, :continue    Go until end of program, exception, or builtins.break\n"
+             << "  :s, :step        Go one step\n"
              ;
         }
 
@@ -917,7 +922,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 
     case nString:
         str << ANSI_WARNING;
-        printLiteralString(str, v.string.s);
+        printLiteralString(str, v.string_view());
         str << ANSI_NORMAL;
         break;
 

src/libexpr/eval-cache.cc

@@ -440,8 +440,8 @@ Value & AttrCursor::forceValue()
 
     if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
         if (v.type() == nString)
-            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
-                           string_t{v.string.s, {}}};
+            cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()),
+                           string_t{v.c_str(), {}}};
         else if (v.type() == nPath) {
             auto path = v.path().path;
             cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}};
@@ -582,7 +582,7 @@ std::string AttrCursor::getString()
     if (v.type() != nString && v.type() != nPath)
         root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>();
 
-    return v.type() == nString ? v.string.s : v.path().to_string();
+    return v.type() == nString ? v.c_str() : v.path().to_string();
 }
 
 string_t AttrCursor::getStringWithContext()
@@ -624,7 +624,7 @@ string_t AttrCursor::getStringWithContext()
     if (v.type() == nString) {
         NixStringContext context;
         copyContext(v, context);
-        return {v.string.s, std::move(context)};
+        return {v.c_str(), std::move(context)};
     }
     else if (v.type() == nPath)
         return {v.path().to_string(), {}};

src/libexpr/eval-settings.hh

@@ -21,7 +21,7 @@ struct EvalSettings : Config
         R"(
           List of directories to be searched for `<...>` file references
 
-          In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
+          In particular, outside of [pure evaluation mode](#conf-pure-eval), this determines the value of
           [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
         )"};
 
@@ -47,11 +47,12 @@ struct EvalSettings : Config
     Setting<bool> enableImportFromDerivation{
         this, true, "allow-import-from-derivation",
         R"(
-          By default, Nix allows you to `import` from a derivation, allowing
-          building at evaluation time. With this option set to false, Nix will
-          throw an error when evaluating an expression that uses this feature,
-          allowing users to ensure their evaluation will not require any
-          builds to take place.
+          By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md).
+
+          With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature,
+          even when the required store object is readily available.
+          This ensures that evaluation will not require any builds to take place,
+          regardless of the state of the store.
         )"};
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",

src/libexpr/eval.cc

@@ -114,7 +114,7 @@ void Value::print(const SymbolTable &symbols, std::ostream &str,
         printLiteralBool(str, boolean);
         break;
     case tString:
-        printLiteralString(str, string.s);
+        printLiteralString(str, string_view());
         break;
     case tPath:
         str << path().to_string(); // !!! escaping?
@@ -339,7 +339,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
         Value nameValue;
         name.expr->eval(state, env, nameValue);
         state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name");
-        return state.symbols.create(nameValue.string.s);
+        return state.symbols.create(nameValue.string_view());
     }
 }
 
@@ -527,9 +527,9 @@ EvalState::EvalState(
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
         for (auto & i : _searchPath.elements)
-            addToSearchPath(SearchPath::Elem {i});
+            searchPath.elements.emplace_back(SearchPath::Elem {i});
         for (auto & i : evalSettings.nixPath.get())
-            addToSearchPath(SearchPath::Elem::parse(i));
+            searchPath.elements.emplace_back(SearchPath::Elem::parse(i));
     }
 
     if (evalSettings.restrictEval || evalSettings.pureEval) {
@@ -1343,7 +1343,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         if (nameVal.type() == nNull)
             continue;
         state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute");
-        auto nameSym = state.symbols.create(nameVal.string.s);
+        auto nameSym = state.symbols.create(nameVal.string_view());
         Bindings::iterator j = v.attrs->find(nameSym);
         if (j != v.attrs->end())
             state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow<EvalError>();
@@ -2155,7 +2155,7 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
         forceValue(v, pos);
         if (v.type() != nString)
             error("value is %1% while a string was expected", showType(v)).debugThrow<TypeError>();
-        return v.string.s;
+        return v.string_view();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
@@ -2182,8 +2182,8 @@ std::string_view EvalState::forceString(Value & v, NixStringContext & context, c
 std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     auto s = forceString(v, pos, errorCtx);
-    if (v.string.context) {
-        error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]).withTrace(pos, errorCtx).debugThrow<EvalError>();
+    if (v.context()) {
+        error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow<EvalError>();
     }
     return s;
 }
@@ -2196,7 +2196,7 @@ bool EvalState::isDerivation(Value & v)
     if (i == v.attrs->end()) return false;
     forceValue(*i->value, i->pos);
     if (i->value->type() != nString) return false;
-    return strcmp(i->value->string.s, "derivation") == 0;
+    return i->value->string_view().compare("derivation") == 0;
 }
 
 
@@ -2228,7 +2228,7 @@ BackedStringView EvalState::coerceToString(
 
     if (v.type() == nString) {
         copyContext(v, context);
-        return std::string_view(v.string.s);
+        return v.string_view();
     }
 
     if (v.type() == nPath) {
@@ -2426,7 +2426,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
             return v1.boolean == v2.boolean;
 
         case nString:
-            return strcmp(v1.string.s, v2.string.s) == 0;
+            return v1.string_view().compare(v2.string_view()) == 0;
 
         case nPath:
             return strcmp(v1._path, v2._path) == 0;
@@ -2506,6 +2506,7 @@ void EvalState::printStats()
             {"elements", nrValuesInEnvs},
             {"bytes", bEnvs},
         };
+        topObj["nrExprs"] = Expr::nrExprs;
         topObj["list"] = {
             {"elements", nrListElems},
             {"bytes", bLists},

src/libexpr/eval.hh

@@ -341,8 +341,6 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    void addToSearchPath(SearchPath::Elem && elem);
-
     SearchPath getSearchPath() { return searchPath; }
 
     /**

src/libexpr/flake/flake.cc

@@ -113,7 +113,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         try {
             if (attr.name == sUrl) {
                 expectType(state, nString, *attr.value, attr.pos);
-                url = attr.value->string.s;
+                url = attr.value->string_view();
                 attrs.emplace("url", *url);
             } else if (attr.name == sFlake) {
                 expectType(state, nBool, *attr.value, attr.pos);
@@ -122,7 +122,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath);
             } else if (attr.name == sFollows) {
                 expectType(state, nString, *attr.value, attr.pos);
-                auto follows(parseInputPath(attr.value->string.s));
+                auto follows(parseInputPath(attr.value->c_str()));
                 follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end());
                 input.follows = follows;
             } else {
@@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 #pragma GCC diagnostic ignored "-Wswitch-enum"
                 switch (attr.value->type()) {
                     case nString:
-                        attrs.emplace(state.symbols[attr.name], attr.value->string.s);
+                        attrs.emplace(state.symbols[attr.name], attr.value->c_str());
                         break;
                     case nBool:
                         attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean });
@@ -229,7 +229,7 @@ static Flake getFlake(
 
     if (auto description = vInfo.attrs->get(state.sDescription)) {
         expectType(state, nString, *description->value, description->pos);
-        flake.description = description->value->string.s;
+        flake.description = description->value->c_str();
     }
 
     auto sInputs = state.symbols.create("inputs");
@@ -520,11 +520,6 @@ LockedFlake lockFlake(
                             }
                         }
 
-                        auto localPath(parentPath);
-                        // If this input is a path, recurse it down.
-                        // This allows us to resolve path inputs relative to the current flake.
-                        if ((*input.ref).input.getType() == "path")
-                            localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
                         computeLocks(
                             mustRefetch
                             ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs
@@ -855,7 +850,7 @@ static void prim_flakeRefToString(
                           Explicit<bool> { attr.value->boolean });
         } else if (t == nString) {
             attrs.emplace(state.symbols[attr.name],
-                          std::string(attr.value->str()));
+                          std::string(attr.value->string_view()));
         } else {
             state.error(
                 "flake reference attribute sets may only contain integers, Booleans, "

src/libexpr/flake/flakeref.cc

@@ -69,32 +69,130 @@ std::optional<FlakeRef> maybeParseFlakeRef(
     }
 }
 
-std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
     const std::string & url,
     const std::optional<Path> & baseDir,
     bool allowMissing,
     bool isFlake)
 {
-    using namespace fetchers;
+    std::string path = url;
+    std::string fragment = "";
+    std::map<std::string, std::string> query;
+    auto pathEnd = url.find_first_of("#?");
+    auto fragmentStart = pathEnd;
+    if (pathEnd != std::string::npos && url[pathEnd] == '?') {
+        fragmentStart = url.find("#");
+    }
+    if (pathEnd != std::string::npos) {
+        path = url.substr(0, pathEnd);
+    }
+    if (fragmentStart != std::string::npos) {
+        fragment = percentDecode(url.substr(fragmentStart+1));
+    }
+    if (pathEnd != std::string::npos && fragmentStart != std::string::npos) {
+        query = decodeQuery(url.substr(pathEnd+1, fragmentStart));
+    }
 
-    static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+";
+    if (baseDir) {
+        /* Check if 'url' is a path (either absolute or relative
+            to 'baseDir'). If so, search upward to the root of the
+            repo (i.e. the directory containing .git). */
 
-    static std::regex pathUrlRegex(
-        "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)"
-        + "(?:\\?(" + queryRegex + "))?"
-        + "(?:#(" + queryRegex + "))?",
-        std::regex::ECMAScript);
+        path = absPath(path, baseDir);
+
+        if (isFlake) {
+
+            if (!allowMissing && !pathExists(path + "/flake.nix")){
+                notice("path '%s' does not contain a 'flake.nix', searching up",path);
+
+                // Save device to detect filesystem boundary
+                dev_t device = lstat(path).st_dev;
+                bool found = false;
+                while (path != "/") {
+                    if (pathExists(path + "/flake.nix")) {
+                        found = true;
+                        break;
+                    } else if (pathExists(path + "/.git"))
+                        throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
+                    else {
+                        if (lstat(path).st_dev != device)
+                            throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
+                    }
+                    path = dirOf(path);
+                }
+                if (!found)
+                    throw BadURL("could not find a flake.nix file");
+            }
+
+            if (!S_ISDIR(lstat(path).st_mode))
+                throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+
+            if (!allowMissing && !pathExists(path + "/flake.nix"))
+                throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+
+            auto flakeRoot = path;
+            std::string subdir;
+
+            while (flakeRoot != "/") {
+                if (pathExists(flakeRoot + "/.git")) {
+                    auto base = std::string("git+file://") + flakeRoot;
+
+                    auto parsedURL = ParsedURL{
+                        .url = base, // FIXME
+                        .base = base,
+                        .scheme = "git+file",
+                        .authority = "",
+                        .path = flakeRoot,
+                        .query = query,
+                    };
+
+                    if (subdir != "") {
+                        if (parsedURL.query.count("dir"))
+                            throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                        parsedURL.query.insert_or_assign("dir", subdir);
+                    }
+
+                    if (pathExists(flakeRoot + "/.git/shallow"))
+                        parsedURL.query.insert_or_assign("shallow", "1");
+
+                    return std::make_pair(
+                        FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")),
+                        fragment);
+                }
+
+                subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
+                flakeRoot = dirOf(flakeRoot);
+            }
+        }
+
+    } else {
+        if (!hasPrefix(path, "/"))
+            throw BadURL("flake reference '%s' is not an absolute path", url);
+        path = canonPath(path + "/" + getOr(query, "dir", ""));
+    }
+
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "path");
+    attrs.insert_or_assign("path", path);
+
+    return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment);
+};
+
+
+/* Check if 'url' is a flake ID. This is an abbreviated syntax for
+    'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
+std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
+    const std::string & url,
+    bool isFlake
+)
+{
+    std::smatch match;
 
     static std::regex flakeRegex(
         "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
         + "(?:#(" + queryRegex + "))?",
         std::regex::ECMAScript);
 
-    std::smatch match;
-
-    /* Check if 'url' is a flake ID. This is an abbreviated syntax for
-       'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
-
     if (std::regex_match(url, match, flakeRegex)) {
         auto parsedURL = ParsedURL{
             .url = url,
@@ -105,111 +203,53 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         };
 
         return std::make_pair(
-            FlakeRef(Input::fromURL(parsedURL, isFlake), ""),
+            FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""),
             percentDecode(match.str(6)));
     }
 
-    else if (std::regex_match(url, match, pathUrlRegex)) {
-        std::string path = match[1];
-        std::string fragment = percentDecode(match.str(3));
+    return {};
+}
 
-        if (baseDir) {
-            /* Check if 'url' is a path (either absolute or relative
-               to 'baseDir'). If so, search upward to the root of the
-               repo (i.e. the directory containing .git). */
-
-            path = absPath(path, baseDir);
-
-            if (isFlake) {
-
-                if (!allowMissing && !pathExists(path + "/flake.nix")){
-                    notice("path '%s' does not contain a 'flake.nix', searching up",path);
-
-                    // Save device to detect filesystem boundary
-                    dev_t device = lstat(path).st_dev;
-                    bool found = false;
-                    while (path != "/") {
-                        if (pathExists(path + "/flake.nix")) {
-                            found = true;
-                            break;
-                        } else if (pathExists(path + "/.git"))
-                            throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
-                        else {
-                            if (lstat(path).st_dev != device)
-                                throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
-                        }
-                        path = dirOf(path);
-                    }
-                    if (!found)
-                        throw BadURL("could not find a flake.nix file");
-                }
-
-                if (!S_ISDIR(lstat(path).st_mode))
-                    throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
-
-                if (!allowMissing && !pathExists(path + "/flake.nix"))
-                    throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
-
-                auto flakeRoot = path;
-                std::string subdir;
-
-                while (flakeRoot != "/") {
-                    if (pathExists(flakeRoot + "/.git")) {
-                        auto base = std::string("git+file://") + flakeRoot;
-
-                        auto parsedURL = ParsedURL{
-                            .url = base, // FIXME
-                            .base = base,
-                            .scheme = "git+file",
-                            .authority = "",
-                            .path = flakeRoot,
-                            .query = decodeQuery(match[2]),
-                        };
-
-                        if (subdir != "") {
-                            if (parsedURL.query.count("dir"))
-                                throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
-                            parsedURL.query.insert_or_assign("dir", subdir);
-                        }
-
-                        if (pathExists(flakeRoot + "/.git/shallow"))
-                            parsedURL.query.insert_or_assign("shallow", "1");
-
-                        return std::make_pair(
-                            FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")),
-                            fragment);
-                    }
-
-                    subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
-                    flakeRoot = dirOf(flakeRoot);
-                }
-            }
-
-        } else {
-            if (!hasPrefix(path, "/"))
-                throw BadURL("flake reference '%s' is not an absolute path", url);
-            auto query = decodeQuery(match[2]);
-            path = canonPath(path + "/" + getOr(query, "dir", ""));
-        }
-
-        fetchers::Attrs attrs;
-        attrs.insert_or_assign("type", "path");
-        attrs.insert_or_assign("path", path);
-
-        return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment);
+std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool isFlake
+)
+{
+    ParsedURL parsedURL;
+    try {
+        parsedURL = parseURL(url);
+    } catch (BadURL &) {
+        return std::nullopt;
     }
 
-    else {
-        auto parsedURL = parseURL(url);
-        std::string fragment;
-        std::swap(fragment, parsedURL.fragment);
+    std::string fragment;
+    std::swap(fragment, parsedURL.fragment);
 
-        auto input = Input::fromURL(parsedURL, isFlake);
-        input.parent = baseDir;
+    auto input = fetchers::Input::fromURL(parsedURL, isFlake);
+    input.parent = baseDir;
 
-        return std::make_pair(
-            FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
-            fragment);
+    return std::make_pair(
+        FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
+        fragment);
+}
+
+std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
+{
+    using namespace fetchers;
+
+    std::smatch match;
+
+    if (auto res = parseFlakeIdRef(url, isFlake)) {
+        return *res;
+    } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) {
+        return *res;
+    } else {
+        return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
     }
 }
 

src/libexpr/flake/lockfile.cc

@@ -345,7 +345,7 @@ void LockFile::check()
 
     for (auto & [inputPath, input] : inputs) {
         if (auto follows = std::get_if<1>(&input)) {
-            if (!follows->empty() && !get(inputs, *follows))
+            if (!follows->empty() && !findInput(*follows))
                 throw Error("input '%s' follows a non-existent input '%s'",
                     printInputPath(inputPath),
                     printInputPath(*follows));

src/libexpr/get-drvs.cc

@@ -156,7 +156,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
         Outputs result;
         for (auto elem : outTI->listItems()) {
             if (elem->type() != nString) throw errMsg;
-            auto out = outputs.find(elem->string.s);
+            auto out = outputs.find(elem->c_str());
             if (out == outputs.end()) throw errMsg;
             result.insert(*out);
         }
@@ -230,7 +230,7 @@ std::string DrvInfo::queryMetaString(const std::string & name)
 {
     Value * v = queryMeta(name);
     if (!v || v->type() != nString) return "";
-    return v->string.s;
+    return v->c_str();
 }
 
 
@@ -242,7 +242,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
-        if (auto n = string2Int<NixInt>(v->string.s))
+        if (auto n = string2Int<NixInt>(v->c_str()))
             return *n;
     }
     return def;
@@ -256,7 +256,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def)
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
-        if (auto n = string2Float<NixFloat>(v->string.s))
+        if (auto n = string2Float<NixFloat>(v->c_str()))
             return *n;
     }
     return def;
@@ -271,8 +271,8 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def)
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
-        if (strcmp(v->string.s, "true") == 0) return true;
-        if (strcmp(v->string.s, "false") == 0) return false;
+        if (v->string_view() == "true") return true;
+        if (v->string_view() == "false") return false;
     }
     return def;
 }

src/libexpr/nixexpr.cc

@@ -76,12 +76,12 @@ void Expr::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << n;
+    str << v.integer;
 }
 
 void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << nf;
+    str << v.fpoint;
 }
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const

src/libexpr/nixexpr.hh

@@ -155,6 +155,10 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 struct Expr
 {
+    static unsigned long nrExprs;
+    Expr() {
+        nrExprs++;
+    }
     virtual ~Expr() { };
     virtual void show(const SymbolTable & symbols, std::ostream & str) const;
     virtual void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
@@ -171,18 +175,16 @@ struct Expr
 
 struct ExprInt : Expr
 {
-    NixInt n;
     Value v;
-    ExprInt(NixInt n) : n(n) { v.mkInt(n); };
+    ExprInt(NixInt n) { v.mkInt(n); };
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS
 };
 
 struct ExprFloat : Expr
 {
-    NixFloat nf;
     Value v;
-    ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); };
+    ExprFloat(NixFloat nf) { v.mkFloat(nf); };
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS
 };
@@ -238,7 +240,7 @@ struct ExprSelect : Expr
     PosIdx pos;
     Expr * e, * def;
     AttrPath attrPath;
-    ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
+    ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
     ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
@@ -248,7 +250,7 @@ struct ExprOpHasAttr : Expr
 {
     Expr * e;
     AttrPath attrPath;
-    ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { };
+    ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { };
     PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };

src/libexpr/parser.y

@@ -520,7 +520,7 @@ path_start
     /* add back in the trailing '/' to the first segment */
     if ($1.p[$1.l-1] == '/' && $1.l > 1)
       path += "/";
-    $$ = new ExprPath(path);
+    $$ = new ExprPath(std::move(path));
   }
   | HPATH {
     if (evalSettings.pureEval) {
@@ -530,7 +530,7 @@ path_start
         );
     }
     Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(path);
+    $$ = new ExprPath(std::move(path));
   }
   ;
 
@@ -653,6 +653,7 @@ formal
 
 namespace nix {
 
+unsigned long Expr::nrExprs = 0;
 
 Expr * EvalState::parse(
     char * text,
@@ -736,12 +737,6 @@ Expr * EvalState::parseStdin()
 }
 
 
-void EvalState::addToSearchPath(SearchPath::Elem && elem)
-{
-    searchPath.elements.emplace_back(std::move(elem));
-}
-
-
 SourcePath EvalState::findFile(const std::string_view path)
 {
     return findFile(searchPath, path);

src/libexpr/primops.cc

@@ -590,7 +590,7 @@ struct CompareValues
                 case nFloat:
                     return v1->fpoint < v2->fpoint;
                 case nString:
-                    return strcmp(v1->string.s, v2->string.s) < 0;
+                    return v1->string_view().compare(v2->string_view()) < 0;
                 case nPath:
                     return strcmp(v1->_path, v2->_path) < 0;
                 case nList:
@@ -982,7 +982,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu
 {
     state.forceValue(*args[0], pos);
     if (args[0]->type() == nString)
-        printError("trace: %1%", args[0]->string.s);
+        printError("trace: %1%", args[0]->string_view());
     else
         printError("trace: %1%", printValue(state, *args[0]));
     state.forceValue(*args[1], pos);
@@ -1252,15 +1252,13 @@ drvName, Bindings * attrs, Value & v)
                 state.store->computeFSClosure(d.drvPath, refs);
                 for (auto & j : refs) {
                     drv.inputSrcs.insert(j);
-                    if (j.isDerivation())
-                        drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
+                    if (j.isDerivation()) {
+                        drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames();
+                    }
                 }
             },
             [&](const NixStringContextElem::Built & b) {
-                if (auto * p = std::get_if<DerivedPath::Opaque>(&*b.drvPath))
-                    drv.inputDrvs[p->path].insert(b.output);
-                else
-                    throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported");
+                drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
             },
             [&](const NixStringContextElem::Opaque & o) {
                 drv.inputSrcs.insert(o.path);
@@ -1520,15 +1518,25 @@ static RegisterPrimOp primop_storePath({
 
 static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
+    auto & arg = *args[0];
+
     /* We don’t check the path right now, because we don’t want to
        throw if the path isn’t allowed, but just return false (and we
        can’t just catch the exception here because we still want to
-       throw if something in the evaluation of `*args[0]` tries to
+       throw if something in the evaluation of `arg` tries to
        access an unauthorized path). */
-    auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false });
+    auto path = realisePath(state, pos, arg, { .checkForPureEval = false });
+
+    /* SourcePath doesn't know about trailing slash. */
+    auto mustBeDir = arg.type() == nString && arg.string_view().ends_with("/");
 
     try {
-        v.mkBool(state.checkSourcePath(path).pathExists());
+        auto checked = state.checkSourcePath(path);
+        auto exists = checked.pathExists();
+        if (exists && mustBeDir) {
+            exists = checked.lstat().type == InputAccessor::tDirectory;
+        }
+        v.mkBool(exists);
     } catch (SysError & e) {
         /* Don't give away info from errors while canonicalising
            ‘path’ in restricted mode. */
@@ -1843,7 +1851,7 @@ static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, V
 {
     SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
 
-    std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
+    OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
 
     state.mkSingleDerivedPathString(
         SingleDerivedPath::Built {
@@ -2392,7 +2400,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args,
         (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]);
 
     std::sort(v.listElems(), v.listElems() + n,
-              [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
+              [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; });
 }
 
 static RegisterPrimOp primop_attrNames({
@@ -2533,7 +2541,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args
     names.reserve(args[1]->listSize());
     for (auto elem : args[1]->listItems()) {
         state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs");
-        names.emplace_back(state.symbols.create(elem->string.s), nullptr);
+        names.emplace_back(state.symbols.create(elem->string_view()), nullptr);
     }
     std::sort(names.begin(), names.end());
 
@@ -2983,7 +2991,7 @@ static RegisterPrimOp primop_tail({
     .name = "__tail",
     .args = {"list"},
     .doc = R"(
-      Return the second to last elements of a list; abort evaluation if
+      Return the list without its first item; abort evaluation if
       the argument isn’t a list or is an empty list.
 
       > **Warning**

src/libexpr/primops/fetchClosure.cc

@@ -133,7 +133,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
 
         else if (attrName == "toPath") {
             state.forceValue(*attr.value, attr.pos);
-            bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string("");
+            bool isEmptyString = attr.value->type() == nString && attr.value->string_view() == "";
             if (isEmptyString) {
                 toPath = StorePathOrGap {};
             }

src/libexpr/tests/derived-path.cc

@@ -18,6 +18,8 @@ TEST_F(DerivedPathExpressionTest, force_init)
 {
 }
 
+#ifndef COVERAGE
+
 RC_GTEST_FIXTURE_PROP(
     DerivedPathExpressionTest,
     prop_opaque_path_round_trip,
@@ -61,4 +63,6 @@ RC_GTEST_FIXTURE_PROP(
     RC_ASSERT(SingleDerivedPath { b } == d);
 }
 
+#endif
+
 } /* namespace nix */

src/libexpr/tests/libexpr.hh

@@ -71,7 +71,7 @@ namespace nix {
         if (arg.type() != nString) {
             return false;
         }
-        return std::string_view(arg.string.s) == s;
+        return std::string_view(arg.c_str()) == s;
     }
 
     MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) {
@@ -106,8 +106,8 @@ namespace nix {
             if (arg.type() != nPath) {
                 *result_listener << "Expected a path got " << arg.type();
                 return false;
-            } else if (std::string_view(arg.string.s) != p) {
-                *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.string.s;
+            } else if (std::string_view(arg._path) != p) {
+                *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str();
                 return false;
             }
             return true;

src/libexpr/tests/primops.cc

@@ -711,14 +711,14 @@ namespace nix {
         // FIXME: add a test that verifies the string context is as expected
         auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\"");
         ASSERT_EQ(v.type(), nString);
-        ASSERT_EQ(v.string.s, std::string_view("fabir"));
+        ASSERT_EQ(v.string_view(), "fabir");
     }
 
     TEST_F(PrimOpTest, concatStringsSep) {
         // FIXME: add a test that verifies the string context is as expected
         auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]");
         ASSERT_EQ(v.type(), nString);
-        ASSERT_EQ(std::string_view(v.string.s), "foo%bar%baz");
+        ASSERT_EQ(v.string_view(), "foo%bar%baz");
     }
 
     TEST_F(PrimOpTest, split1) {

src/libexpr/tests/value/context.cc

@@ -147,6 +147,8 @@ Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
 
 namespace nix {
 
+#ifndef COVERAGE
+
 RC_GTEST_PROP(
     NixStringContextElemTest,
     prop_round_rip,
@@ -155,4 +157,6 @@ RC_GTEST_PROP(
     RC_ASSERT(o == NixStringContextElem::parse(o.to_string()));
 }
 
+#endif
+
 }

src/libexpr/value-to-json.cc

@@ -31,7 +31,7 @@ json printValueAsJSON(EvalState & state, bool strict,
 
         case nString:
             copyContext(v, context);
-            out = v.string.s;
+            out = v.c_str();
             break;
 
         case nPath:

src/libexpr/value-to-xml.cc

@@ -74,7 +74,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
         case nString:
             /* !!! show the context? */
             copyContext(v, context);
-            doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
+            doc.writeEmptyElement("string", singletonAttrs("value", v.c_str()));
             break;
 
         case nPath:
@@ -96,14 +96,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value, a->pos);
                     if (a->value->type() == nString)
-                        xmlAttrs["drvPath"] = drvPath = a->value->string.s;
+                        xmlAttrs["drvPath"] = drvPath = a->value->c_str();
                 }
 
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value, a->pos);
                     if (a->value->type() == nString)
-                        xmlAttrs["outPath"] = a->value->string.s;
+                        xmlAttrs["outPath"] = a->value->c_str();
                 }
 
                 XMLOpenElement _(doc, "derivation", xmlAttrs);

src/libexpr/value.hh

@@ -186,10 +186,9 @@ public:
          * For canonicity, the store paths should be in sorted order.
          */
         struct {
-            const char * s;
+            const char * c_str;
             const char * * context; // must be in sorted order
         } string;
-
         const char * _path;
         Bindings * attrs;
         struct {
@@ -270,7 +269,7 @@ public:
     inline void mkString(const char * s, const char * * context = 0)
     {
         internalType = tString;
-        string.s = s;
+        string.c_str = s;
         string.context = context;
     }
 
@@ -441,10 +440,21 @@ public:
         return SourcePath{CanonPath(_path)};
     }
 
-    std::string_view str() const
+    std::string_view string_view() const
     {
         assert(internalType == tString);
-        return std::string_view(string.s);
+        return std::string_view(string.c_str);
+    }
+
+    const char * const c_str() const
+    {
+        assert(internalType == tString);
+        return string.c_str;
+    }
+
+    const char * * context() const
+    {
+        return string.context;
     }
 };
 

src/libfetchers/tarball.cc

@@ -232,7 +232,7 @@ struct CurlInputScheme : InputScheme
         if (type != inputType()) return {};
 
         // FIXME: some of these only apply to TarballInputScheme.
-        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"};
+        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"};
         for (auto & [name, value] : attrs)
             if (!allowedNames.count(name))
                 throw Error("unsupported %s input attribute '%s'", *type, name);
@@ -310,6 +310,9 @@ struct TarballInputScheme : CurlInputScheme
             input = immutableInput;
         }
 
+        if (result.lastModified && !input.attrs.contains("lastModified"))
+            input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
+
         return {result.tree.storePath, std::move(input)};
     }
 };

src/libmain/shared.cc

@@ -379,9 +379,9 @@ RunPager::RunPager()
     });
 
     pid.setKillSignal(SIGINT);
-    stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0);
+    std_out = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0);
     if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1)
-        throw SysError("dupping stdout");
+        throw SysError("dupping standard output");
 }
 
 
@@ -390,7 +390,7 @@ RunPager::~RunPager()
     try {
         if (pid != -1) {
             std::cout.flush();
-            dup2(stdout, STDOUT_FILENO);
+            dup2(std_out, STDOUT_FILENO);
             pid.wait();
         }
     } catch (...) {

src/libmain/shared.hh

@@ -85,8 +85,9 @@ struct LegacyArgs : public MixCommonArgs
 void showManPage(const std::string & name);
 
 /**
- * The constructor of this class starts a pager if stdout is a
- * terminal and $PAGER is set. Stdout is redirected to the pager.
+ * The constructor of this class starts a pager if standard output is a
+ * terminal and $PAGER is set. Standard output is redirected to the
+ * pager.
  */
 class RunPager
 {
@@ -96,7 +97,7 @@ public:
 
 private:
     Pid pid;
-    int stdout;
+    int std_out;
 };
 
 extern volatile ::sig_atomic_t blockInt;

src/libstore/build/create-derivation-and-realise-goal.cc

@@ -0,0 +1,157 @@
+#include "create-derivation-and-realise-goal.hh"
+#include "worker.hh"
+
+namespace nix {
+
+CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
+    const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
+    : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
+    , drvReq(drvReq)
+    , wantedOutputs(wantedOutputs)
+    , buildMode(buildMode)
+{
+    state = &CreateDerivationAndRealiseGoal::getDerivation;
+    name = fmt(
+        "outer obtaining drv from '%s' and then building outputs %s",
+        drvReq->to_string(worker.store),
+        std::visit(overloaded {
+            [&](const OutputsSpec::All) -> std::string {
+                return "* (all of them)";
+            },
+            [&](const OutputsSpec::Names os) {
+                return concatStringsSep(", ", quoteStrings(os));
+            },
+        }, wantedOutputs.raw));
+    trace("created outer");
+
+    worker.updateProgress();
+}
+
+
+CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
+{
+}
+
+
+static StorePath pathPartOfReq(const SingleDerivedPath & req)
+{
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & bo) {
+            return bo.path;
+        },
+        [&](const SingleDerivedPath::Built & bfd) {
+            return pathPartOfReq(*bfd.drvPath);
+        },
+    }, req.raw());
+}
+
+
+std::string CreateDerivationAndRealiseGoal::key()
+{
+    /* Ensure that derivations get built in order of their name,
+       i.e. a derivation named "aardvark" always comes before "baboon". And
+       substitution goals and inner derivation goals always happen before
+       derivation goals (due to "b$"). */
+    return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
+}
+
+
+void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
+{
+}
+
+
+void CreateDerivationAndRealiseGoal::work()
+{
+    (this->*state)();
+}
+
+
+void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
+{
+    /* If we already want all outputs, there is nothing to do. */
+    auto newWanted = wantedOutputs.union_(outputs);
+    bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
+    wantedOutputs = newWanted;
+
+    if (!needRestart) return;
+
+    if (!optDrvPath)
+        // haven't started steps where the outputs matter yet
+        return;
+    worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
+}
+
+
+void CreateDerivationAndRealiseGoal::getDerivation()
+{
+    trace("outer init");
+
+    /* The first thing to do is to make sure that the derivation
+       exists.  If it doesn't, it may be created through a
+       substitute. */
+    if (auto optDrvPath = [this]() -> std::optional<StorePath> {
+        if (buildMode != bmNormal) return std::nullopt;
+
+        auto drvPath = StorePath::dummy;
+        try {
+            drvPath = resolveDerivedPath(worker.store, *drvReq);
+        } catch (MissingRealisation &) {
+            return std::nullopt;
+        }
+        return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
+            ? std::optional { drvPath }
+            : std::nullopt;
+    }()) {
+        trace(fmt("already have drv '%s' for '%s', can go straight to building",
+            worker.store.printStorePath(*optDrvPath),
+            drvReq->to_string(worker.store)));
+
+        loadAndBuildDerivation();
+    } else {
+        trace("need to obtain drv we want to build");
+
+        addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
+
+        state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
+        if (waitees.empty()) work();
+    }
+}
+
+
+void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
+{
+    trace("outer load and build derivation");
+
+    if (nrFailed != 0) {
+        amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
+        return;
+    }
+
+    StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
+    /* Build this step! */
+    concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
+    addWaitee(upcast_goal(concreteDrvGoal));
+    state = &CreateDerivationAndRealiseGoal::buildDone;
+    optDrvPath = std::move(drvPath);
+    if (waitees.empty()) work();
+}
+
+
+void CreateDerivationAndRealiseGoal::buildDone()
+{
+    trace("outer build done");
+
+    buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
+        .drvPath = drvReq,
+        .outputs = wantedOutputs,
+    });
+
+    if (buildResult.success())
+        amDone(ecSuccess);
+    else
+        amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
+}
+
+
+}

src/libstore/build/create-derivation-and-realise-goal.hh

@@ -0,0 +1,96 @@
+#pragma once
+
+#include "parsed-derivations.hh"
+#include "lock.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
+#include "goal.hh"
+
+namespace nix {
+
+struct DerivationGoal;
+
+/**
+ * This goal type is essentially the serial composition (like function
+ * composition) of a goal for getting a derivation, and then a
+ * `DerivationGoal` using the newly-obtained derivation.
+ *
+ * In the (currently experimental) general inductive case of derivations
+ * that are themselves build outputs, that first goal will be *another*
+ * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
+ * where the derivation has no provence and is just referred to by
+ * (content-addressed) store path, that first goal is a
+ * `SubstitutionGoal`.
+ *
+ * If we already have the derivation (e.g. if the evalutator has created
+ * the derivation locally and then instructured the store to build it),
+ * we can skip the first goal entirely as a small optimization.
+ */
+struct CreateDerivationAndRealiseGoal : public Goal
+{
+    /**
+     * How to obtain a store path of the derivation to build.
+     */
+    ref<SingleDerivedPath> drvReq;
+
+    /**
+     * The path of the derivation, once obtained.
+     **/
+    std::optional<StorePath> optDrvPath;
+
+    /**
+     * The goal for the corresponding concrete derivation.
+     **/
+    std::shared_ptr<DerivationGoal> concreteDrvGoal;
+
+    /**
+     * The specific outputs that we need to build.
+     */
+    OutputsSpec wantedOutputs;
+
+    typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
+    GoalState state;
+
+    /**
+     * The final output paths of the build.
+     *
+     * - For input-addressed derivations, always the precomputed paths
+     *
+     * - For content-addressed derivations, calcuated from whatever the
+     *   hash ends up being. (Note that fixed outputs derivations that
+     *   produce the "wrong" output still install that data under its
+     *   true content-address.)
+     */
+    OutputPathMap finalOutputs;
+
+    BuildMode buildMode;
+
+    CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
+        const OutputsSpec & wantedOutputs, Worker & worker,
+        BuildMode buildMode = bmNormal);
+    virtual ~CreateDerivationAndRealiseGoal();
+
+    void timedOut(Error && ex) override;
+
+    std::string key() override;
+
+    void work() override;
+
+    /**
+     * Add wanted outputs to an already existing derivation goal.
+     */
+    void addWantedOutputs(const OutputsSpec & outputs);
+
+    /**
+     * The states.
+     */
+    void getDerivation();
+    void loadAndBuildDerivation();
+    void buildDone();
+
+    JobCategory jobCategory() const override {
+        return JobCategory::Administration;
+    };
+};
+
+}

src/libstore/build/derivation-goal.cc

@@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
-    state = &DerivationGoal::getDerivation;
+    state = &DerivationGoal::loadDerivation;
     name = fmt(
         "building of '%s' from .drv file",
         DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@@ -164,24 +164,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
 }
 
 
-void DerivationGoal::getDerivation()
-{
-    trace("init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
-        loadDerivation();
-        return;
-    }
-
-    addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
-
-    state = &DerivationGoal::loadDerivation;
-}
-
-
 void DerivationGoal::loadDerivation()
 {
     trace("loading derivation");
@@ -368,20 +350,37 @@ void DerivationGoal::gaveUpOnSubstitution()
 
     /* The inputs must be built before we can build this goal. */
     inputDrvOutputs.clear();
-    if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+    if (useDerivation) {
+        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> addWaiteeDerivedPath;
+
+        addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            if (!inputNode.value.empty())
+                addWaitee(worker.makeGoal(
+                    DerivedPath::Built {
+                        .drvPath = inputDrv,
+                        .outputs = inputNode.value,
+                    },
+                    buildMode == bmRepair ? bmRepair : bmNormal));
+            for (const auto & [outputName, childNode] : inputNode.childMap)
+                addWaiteeDerivedPath(
+                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                    childNode);
+        };
+
+        for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs.map) {
             /* Ensure that pure, non-fixed-output derivations don't
                depend on impure derivations. */
             if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
-                auto inputDrv = worker.evalStore.readDerivation(i.first);
+                auto inputDrv = worker.evalStore.readDerivation(inputDrvPath);
                 if (!inputDrv.type().isPure())
                     throw Error("pure derivation '%s' depends on impure derivation '%s'",
                         worker.store.printStorePath(drvPath),
-                        worker.store.printStorePath(i.first));
+                        worker.store.printStorePath(inputDrvPath));
             }
 
-            addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
+            addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode);
         }
+    }
 
     /* Copy the input sources from the eval store to the build
        store. */
@@ -452,7 +451,12 @@ void DerivationGoal::repairClosure()
         if (drvPath2 == outputsToDrv.end())
             addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
         else
-            addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair));
+            addWaitee(worker.makeGoal(
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(drvPath2->second),
+                    .outputs = OutputsSpec::All { },
+                },
+                bmRepair));
     }
 
     if (waitees.empty()) {
@@ -509,7 +513,7 @@ void DerivationGoal::inputsRealised()
                 return ia.deferred;
             },
             [&](const DerivationType::ContentAddressed & ca) {
-                return !fullDrv.inputDrvs.empty() && (
+                return !fullDrv.inputDrvs.map.empty() && (
                     ca.fixed
                     /* Can optionally resolve if fixed, which is good
                        for avoiding unnecessary rebuilds. */
@@ -523,7 +527,7 @@ void DerivationGoal::inputsRealised()
             }
         }, drvType.raw);
 
-        if (resolveDrv && !fullDrv.inputDrvs.empty()) {
+        if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
             experimentalFeatureSettings.require(Xp::CaDerivations);
 
             /* We are be able to resolve this derivation based on the
@@ -560,11 +564,13 @@ void DerivationGoal::inputsRealised()
             return;
         }
 
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
+        std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
+
+        accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            for (auto & j : wantedDepOutputs) {
+            auto getOutput = [&](const std::string & outputName) {
                 /* TODO (impure derivations-induced tech debt):
                    Tracking input derivation outputs statefully through the
                    goals is error prone and has led to bugs.
@@ -576,21 +582,30 @@ void DerivationGoal::inputsRealised()
                    a representation in the store, which is a usability problem
                    in itself. When implementing this logic entirely with lookups
                    make sure that they're cached. */
-                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
-                    worker.store.computeFSClosure(*outPath, inputPaths);
+                if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) {
+                    return *outPath;
                 }
                 else {
                     auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
-                    auto outMapPath = outMap.find(j);
+                    auto outMapPath = outMap.find(outputName);
                     if (outMapPath == outMap.end()) {
                         throw Error(
                             "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                            worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath));
                     }
-                    worker.store.computeFSClosure(outMapPath->second, inputPaths);
+                    return outMapPath->second;
                 }
-            }
-        }
+            };
+
+            for (auto & outputName : inputNode.value)
+                worker.store.computeFSClosure(getOutput(outputName), inputPaths);
+
+            for (auto & [outputName, childNode] : inputNode.childMap)
+                accumInputPaths(getOutput(outputName), childNode);
+        };
+
+        for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map)
+            accumInputPaths(depDrvPath, depNode);
     }
 
     /* Second, the input sources. */
@@ -1483,22 +1498,24 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
     if (!useDerivation) return;
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
-    if (!dg) return;
+    std::optional info = tryGetConcreteDrvGoal(waitee);
+    if (!info) return;
+    const auto & [dg, drvReq] = *info;
 
-    auto outputs = fullDrv.inputDrvs.find(dg->drvPath);
-    if (outputs == fullDrv.inputDrvs.end()) return;
+    auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get());
+    if (!nodeP) return;
+    auto & outputs = nodeP->value;
 
-    for (auto & outputName : outputs->second) {
-        auto buildResult = dg->getBuildResult(DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(dg->drvPath),
+    for (auto & outputName : outputs) {
+        auto buildResult = dg.get().getBuildResult(DerivedPath::Built {
+            .drvPath = makeConstantStorePathRef(dg.get().drvPath),
             .outputs = OutputsSpec::Names { outputName },
         });
         if (buildResult.success()) {
             auto i = buildResult.builtOutputs.find(outputName);
             if (i != buildResult.builtOutputs.end())
                 inputDrvOutputs.insert_or_assign(
-                    { dg->drvPath, outputName },
+                    { dg.get().drvPath, outputName },
                     i->second.outPath);
         }
     }

src/libstore/build/derivation-goal.hh

@@ -50,6 +50,13 @@ struct InitialOutput {
     std::optional<InitialOutputStatus> known;
 };
 
+/**
+ * A goal for building some or all of the outputs of a derivation.
+ *
+ * The derivation must already be present, either in the store in a drv
+ * or in memory. If the derivation itself needs to be gotten first, a
+ * `CreateDerivationAndRealiseGoal` goal must be used instead.
+ */
 struct DerivationGoal : public Goal
 {
     /**
@@ -66,8 +73,7 @@ struct DerivationGoal : public Goal
     std::shared_ptr<DerivationGoal> resolvedDrvGoal;
 
     /**
-     * The specific outputs that we need to build.  Empty means all of
-     * them.
+     * The specific outputs that we need to build.
      */
     OutputsSpec wantedOutputs;
 
@@ -229,7 +235,6 @@ struct DerivationGoal : public Goal
     /**
      * The states.
      */
-    void getDerivation();
     void loadDerivation();
     void haveDerivation();
     void outputsSubstitutionTried();
@@ -334,7 +339,9 @@ struct DerivationGoal : public Goal
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
 
-    JobCategory jobCategory() override { return JobCategory::Build; };
+    JobCategory jobCategory() const override {
+        return JobCategory::Build;
+    };
 };
 
 MakeError(NotDeterministic, BuildError);

src/libstore/build/drv-output-substitution-goal.hh

@@ -73,7 +73,9 @@ public:
     void work() override;
     void handleEOF(int fd) override;
 
-    JobCategory jobCategory() override { return JobCategory::Substitution; };
+    JobCategory jobCategory() const override {
+        return JobCategory::Substitution;
+    };
 };
 
 }

src/libstore/build/entry-points.cc

@@ -1,5 +1,6 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
+#include "create-derivation-and-realise-goal.hh"
 #include "derivation-goal.hh"
 #include "local-store.hh"
 
@@ -15,7 +16,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
 
     worker.run(goals);
 
-    StorePathSet failed;
+    StringSet failed;
     std::optional<Error> ex;
     for (auto & i : goals) {
         if (i->ex) {
@@ -25,8 +26,10 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
                 ex = std::move(i->ex);
         }
         if (i->exitCode != Goal::ecSuccess) {
-            if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
-            else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
+            if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get()))
+                failed.insert(i2->drvReq->to_string(*this));
+            else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
+                failed.insert(printStorePath(i2->storePath));
         }
     }
 
@@ -35,7 +38,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         throw std::move(*ex);
     } else if (!failed.empty()) {
         if (ex) logError(ex->info());
-        throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
+        throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
     }
 }
 
@@ -124,8 +127,11 @@ void Store::repairPath(const StorePath & path)
         auto info = queryPathInfo(path);
         if (info->deriver && isValidPath(*info->deriver)) {
             goals.clear();
-            // FIXME: Should just build the specific output we need.
-            goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair));
+            goals.insert(worker.makeGoal(DerivedPath::Built {
+                .drvPath = makeConstantStorePathRef(*info->deriver),
+                // FIXME: Should just build the specific output we need.
+                .outputs = OutputsSpec::All { },
+            }, bmRepair));
             worker.run(goals);
         } else
             throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));

src/libstore/build/goal.cc

@@ -11,7 +11,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
 }
 
 
-BuildResult Goal::getBuildResult(const DerivedPath & req) {
+BuildResult Goal::getBuildResult(const DerivedPath & req) const {
     BuildResult res { buildResult };
 
     if (auto pbp = std::get_if<DerivedPath::Built>(&req)) {

src/libstore/build/goal.hh

@@ -41,8 +41,24 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
  * of each category in parallel.
  */
 enum struct JobCategory {
+    /**
+     * A build of a derivation; it will use CPU and disk resources.
+     */
     Build,
+    /**
+     * A substitution an arbitrary store object; it will use network resources.
+     */
     Substitution,
+    /**
+     * A goal that does no "real" work by itself, and just exists to depend on
+     * other goals which *do* do real work. These goals therefore are not
+     * limited.
+     *
+     * These goals cannot infinitely create themselves, so there is no risk of
+     * a "fork bomb" type situation (which would be a problem even though the
+     * goal do no real work) either.
+     */
+    Administration,
 };
 
 struct Goal : public std::enable_shared_from_this<Goal>
@@ -110,7 +126,7 @@ public:
      * sake of both privacy and determinism, and this "safe accessor"
      * ensures we don't.
      */
-    BuildResult getBuildResult(const DerivedPath &);
+    BuildResult getBuildResult(const DerivedPath &) const;
 
     /**
      * Exception containing an error message, if any.
@@ -144,7 +160,7 @@ public:
 
     void trace(std::string_view s);
 
-    std::string getName()
+    std::string getName() const
     {
         return name;
     }
@@ -166,7 +182,7 @@ public:
      * @brief Hint for the scheduler, which concurrency limit applies.
      * @see JobCategory
      */
-    virtual JobCategory jobCategory() = 0;
+    virtual JobCategory jobCategory() const = 0;
 };
 
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);

src/libstore/build/local-derivation-goal.cc

@@ -2955,7 +2955,7 @@ bool LocalDerivationGoal::isReadDesc(int fd)
 }
 
 
-StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName)
+StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName)
 {
     return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),

src/libstore/build/local-derivation-goal.hh

@@ -297,7 +297,7 @@ struct LocalDerivationGoal : public DerivationGoal
      * @todo Add option to randomize, so we can audit whether our
      * rewrites caught everything
      */
-    StorePath makeFallbackPath(std::string_view outputName);
+    StorePath makeFallbackPath(OutputNameView outputName);
 };
 
 }

src/libstore/build/substitution-goal.hh

@@ -117,7 +117,9 @@ public:
     /* Called by destructor, can't be overridden */
     void cleanup() override final;
 
-    JobCategory jobCategory() override { return JobCategory::Substitution; };
+    JobCategory jobCategory() const override {
+        return JobCategory::Substitution;
+    };
 };
 
 }

src/libstore/build/worker.cc

@@ -2,6 +2,7 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "drv-output-substitution-goal.hh"
+#include "create-derivation-and-realise-goal.hh"
 #include "local-derivation-goal.hh"
 #include "hook-instance.hh"
 
@@ -41,6 +42,24 @@ Worker::~Worker()
 }
 
 
+std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
+    ref<SingleDerivedPath> drvReq,
+    const OutputsSpec & wantedOutputs,
+    BuildMode buildMode)
+{
+    std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
+    std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
+    if (!goal) {
+        goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
+        goal_weak = goal;
+        wakeUp(goal);
+    } else {
+        goal->addWantedOutputs(wantedOutputs);
+    }
+    return goal;
+}
+
+
 std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
     const StorePath & drvPath,
     const OutputsSpec & wantedOutputs,
@@ -111,10 +130,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 {
     return std::visit(overloaded {
         [&](const DerivedPath::Built & bfd) -> GoalPtr {
-            if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
-                return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
-            else
-                throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
+            return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode);
         },
         [&](const DerivedPath::Opaque & bo) -> GoalPtr {
             return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@@ -123,24 +139,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 }
 
 
+template<typename K, typename V, typename F>
+static void cullMap(std::map<K, V> & goalMap, F f)
+{
+    for (auto i = goalMap.begin(); i != goalMap.end();)
+        if (!f(i->second))
+            i = goalMap.erase(i);
+        else ++i;
+}
+
+
 template<typename K, typename G>
 static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
 {
     /* !!! inefficient */
-    for (auto i = goalMap.begin();
-         i != goalMap.end(); )
-        if (i->second.lock() == goal) {
-            auto j = i; ++j;
-            goalMap.erase(i);
-            i = j;
-        }
-        else ++i;
+    cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
+        return gp.lock() != goal;
+    });
+}
+
+template<typename K>
+static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap);
+
+template<typename K>
+static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
+{
+    /* !!! inefficient */
+    cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
+        if (node.value.lock() == goal)
+            node.value.reset();
+        removeGoal(goal, node.childMap);
+        return !node.value.expired() || !node.childMap.empty();
+    });
 }
 
 
 void Worker::removeGoal(GoalPtr goal)
 {
-    if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
+    if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal))
+        nix::removeGoal(drvGoal, outerDerivationGoals.map);
+    else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
         nix::removeGoal(drvGoal, derivationGoals);
     else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
         nix::removeGoal(subGoal, substitutionGoals);
@@ -198,8 +236,19 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
     if (inBuildSlot) {
-        if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
-        else nrLocalBuilds++;
+        switch (goal->jobCategory()) {
+        case JobCategory::Substitution:
+            nrSubstitutions++;
+            break;
+        case JobCategory::Build:
+            nrLocalBuilds++;
+            break;
+        case JobCategory::Administration:
+            /* Intentionally not limited, see docs */
+            break;
+        default:
+            abort();
+        }
     }
 }
 
@@ -211,12 +260,20 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
     if (i == children.end()) return;
 
     if (i->inBuildSlot) {
-        if (goal->jobCategory() == JobCategory::Substitution) {
+        switch (goal->jobCategory()) {
+        case JobCategory::Substitution:
             assert(nrSubstitutions > 0);
             nrSubstitutions--;
-        } else {
+            break;
+        case JobCategory::Build:
             assert(nrLocalBuilds > 0);
             nrLocalBuilds--;
+            break;
+        case JobCategory::Administration:
+            /* Intentionally not limited, see docs */
+            break;
+        default:
+            abort();
         }
     }
 
@@ -267,9 +324,9 @@ void Worker::run(const Goals & _topGoals)
 
     for (auto & i : _topGoals) {
         topGoals.insert(i);
-        if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
+        if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) {
             topPaths.push_back(DerivedPath::Built {
-                .drvPath = makeConstantStorePathRef(goal->drvPath),
+                .drvPath = goal->drvReq,
                 .outputs = goal->wantedOutputs,
             });
         } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
@@ -522,11 +579,29 @@ void Worker::markContentsGood(const StorePath & path)
 }
 
 
-GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
-    return subGoal;
-}
-GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal)
+{
     return subGoal;
 }
 
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
+{
+    return subGoal;
+}
+
+GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
+{
+    return subGoal;
+}
+
+std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
+{
+    auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
+    if (!odg) return std::nullopt;
+    return {{
+        std::cref(*odg->concreteDrvGoal),
+        std::cref(*odg->drvReq),
+    }};
+}
+
 }

src/libstore/build/worker.hh

@@ -4,6 +4,7 @@
 #include "types.hh"
 #include "lock.hh"
 #include "store-api.hh"
+#include "derived-path-map.hh"
 #include "goal.hh"
 #include "realisation.hh"
 
@@ -13,6 +14,7 @@
 namespace nix {
 
 /* Forward definition. */
+struct CreateDerivationAndRealiseGoal;
 struct DerivationGoal;
 struct PathSubstitutionGoal;
 class DrvOutputSubstitutionGoal;
@@ -31,9 +33,24 @@ class DrvOutputSubstitutionGoal;
  */
 GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
 GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
+GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
+/**
+ * The current implementation of impure derivations has
+ * `DerivationGoal`s accumulate realisations from their waitees.
+ * Unfortunately, `DerivationGoal`s don't directly depend on other
+ * goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
+ *
+ * We try not to share any of the details of any goal type with any
+ * other, for sake of modularity and quicker rebuilds. This means we
+ * cannot "just" downcast and fish out the field. So as an escape hatch,
+ * we have made the function, written in `worker.cc` where all the goal
+ * types are visible, and use it instead.
+ */
+
+std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
 
 /**
  * A mapping used to remember for each child process to what goal it
@@ -102,6 +119,9 @@ private:
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
      */
+
+    DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
+
     std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
     std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
     std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@@ -189,6 +209,9 @@ public:
      * @ref DerivationGoal "derivation goal"
      */
 private:
+    std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
+        ref<SingleDerivedPath> drvPath,
+        const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
     std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs,
         std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);

src/libstore/derivations.cc

@@ -12,7 +12,7 @@
 
 namespace nix {
 
-std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
 {
     return std::visit(overloaded {
         [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
@@ -36,7 +36,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
 }
 
 
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
 {
     return store.makeFixedOutputPathFromCA(
         outputPathName(drvName, outputName),
@@ -136,7 +136,7 @@ StorePath writeDerivation(Store & store,
     const Derivation & drv, RepairFlag repair, bool readOnly)
 {
     auto references = drv.inputSrcs;
-    for (auto & i : drv.inputDrvs)
+    for (auto & i : drv.inputDrvs.map)
         references.insert(i.first);
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
@@ -154,8 +154,9 @@ static void expect(std::istream & str, std::string_view s)
 {
     char s2[s.size()];
     str.read(s2, s.size());
-    if (std::string(s2, s.size()) != s)
-        throw FormatError("expected string '%1%'", s);
+    std::string_view s2View { s2, s.size() };
+    if (s2View != s)
+        throw FormatError("expected string '%s', got '%s'", s, s2View);
 }
 
 
@@ -207,23 +208,27 @@ static bool endOfList(std::istream & str)
 static StringSet parseStrings(std::istream & str, bool arePaths)
 {
     StringSet res;
+    expect(str, "[");
     while (!endOfList(str))
         res.insert(arePaths ? parsePath(str) : parseString(str));
     return res;
 }
 
 
-static DerivationOutput parseDerivationOutput(const Store & store,
-    std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
+static DerivationOutput parseDerivationOutput(
+    const Store & store,
+    std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     if (hashAlgo != "") {
         ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
         if (method == TextIngestionMethod {})
-            experimentalFeatureSettings.require(Xp::DynamicDerivations);
+            xpSettings.require(Xp::DynamicDerivations);
         const auto hashType = parseHashType(hashAlgo);
         if (hashS == "impure") {
-            experimentalFeatureSettings.require(Xp::ImpureDerivations);
-            assert(pathS == "");
+            xpSettings.require(Xp::ImpureDerivations);
+            if (pathS != "")
+                throw FormatError("impure derivation output should not specify output path");
             return DerivationOutput::Impure {
                 .method = std::move(method),
                 .hashType = std::move(hashType),
@@ -238,8 +243,9 @@ static DerivationOutput parseDerivationOutput(const Store & store,
                 },
             };
         } else {
-            experimentalFeatureSettings.require(Xp::CaDerivations);
-            assert(pathS == "");
+            xpSettings.require(Xp::CaDerivations);
+            if (pathS != "")
+                throw FormatError("content-addressed derivation output should not specify output path");
             return DerivationOutput::CAFloating {
                 .method = std::move(method),
                 .hashType = std::move(hashType),
@@ -256,29 +262,116 @@ static DerivationOutput parseDerivationOutput(const Store & store,
     }
 }
 
-static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+static DerivationOutput parseDerivationOutput(
+    const Store & store, std::istringstream & str,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
     expect(str, ","); const auto pathS = parseString(str);
     expect(str, ","); const auto hashAlgo = parseString(str);
     expect(str, ","); const auto hash = parseString(str);
     expect(str, ")");
 
-    return parseDerivationOutput(store, pathS, hashAlgo, hash);
+    return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings);
+}
+
+/**
+ * All ATerm Derivation format versions currently known.
+ *
+ * Unknown versions are rejected at the parsing stage.
+ */
+enum struct DerivationATermVersion {
+    /**
+     * Older unversioned form
+     */
+    Traditional,
+
+    /**
+     * Newer versioned form; only this version so far.
+     */
+    DynamicDerivations,
+};
+
+static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
+    const Store & store,
+    std::istringstream & str,
+    DerivationATermVersion version)
+{
+    DerivedPathMap<StringSet>::ChildNode node;
+
+    auto parseNonDynamic = [&]() {
+        node.value = parseStrings(str, false);
+    };
+
+    // Older derivation should never use new form, but newer
+    // derivaiton can use old form.
+    switch (version) {
+    case DerivationATermVersion::Traditional:
+        parseNonDynamic();
+        break;
+    case DerivationATermVersion::DynamicDerivations:
+        switch (str.peek()) {
+        case '[':
+            parseNonDynamic();
+            break;
+        case '(':
+            expect(str, "(");
+            node.value = parseStrings(str, false);
+            expect(str, ",[");
+            while (!endOfList(str)) {
+                expect(str, "(");
+                auto outputName = parseString(str);
+                expect(str, ",");
+                node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version));
+                expect(str, ")");
+            }
+            expect(str, ")");
+            break;
+        default:
+            throw FormatError("invalid inputDrvs entry in derivation");
+        }
+        break;
+    default:
+        // invalid format, not a parse error but internal error
+        assert(false);
+    }
+    return node;
 }
 
 
-Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
+Derivation parseDerivation(
+    const Store & store, std::string && s, std::string_view name,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     Derivation drv;
     drv.name = name;
 
     std::istringstream str(std::move(s));
-    expect(str, "Derive([");
+    expect(str, "D");
+    DerivationATermVersion version;
+    switch (str.peek()) {
+    case 'e':
+        expect(str, "erive(");
+        version = DerivationATermVersion::Traditional;
+        break;
+    case 'r':
+        expect(str, "rvWithVersion(");
+        auto versionS = parseString(str);
+        if (versionS == "xp-dyn-drv") {
+            // Only verison we have so far
+            version = DerivationATermVersion::DynamicDerivations;
+            xpSettings.require(Xp::DynamicDerivations);
+        } else {
+            throw FormatError("Unknown derivation ATerm format version '%s'", versionS);
+        }
+        expect(str, ",");
+        break;
+    }
 
     /* Parse the list of outputs. */
+    expect(str, "[");
     while (!endOfList(str)) {
         expect(str, "("); std::string id = parseString(str);
-        auto output = parseDerivationOutput(store, str);
+        auto output = parseDerivationOutput(store, str, xpSettings);
         drv.outputs.emplace(std::move(id), std::move(output));
     }
 
@@ -287,12 +380,12 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
     while (!endOfList(str)) {
         expect(str, "(");
         Path drvPath = parsePath(str);
-        expect(str, ",[");
-        drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false));
+        expect(str, ",");
+        drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version));
         expect(str, ")");
     }
 
-    expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
+    expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
     expect(str, ","); drv.platform = parseString(str);
     expect(str, ","); drv.builder = parseString(str);
 
@@ -376,14 +469,67 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt
 }
 
 
+static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
+{
+    s += ',';
+    if (node.childMap.empty()) {
+        printUnquotedStrings(s, node.value.begin(), node.value.end());
+    } else {
+        s += "(";
+        printUnquotedStrings(s, node.value.begin(), node.value.end());
+        s += ",[";
+        bool first = true;
+        for (auto & [outputName, childNode] : node.childMap) {
+            if (first) first = false; else s += ',';
+            s += '('; printUnquotedString(s, outputName);
+            unparseDerivedPathMapNode(store, s, childNode);
+            s += ')';
+        }
+        s += "])";
+    }
+}
+
+
+/**
+ * Does the derivation have a dependency on the output of a dynamic
+ * derivation?
+ *
+ * In other words, does it on the output of derivation that is itself an
+ * ouput of a derivation? This corresponds to a dependency that is an
+ * inductive derived path with more than one layer of
+ * `DerivedPath::Built`.
+ */
+static bool hasDynamicDrvDep(const Derivation & drv)
+{
+    return
+        std::find_if(
+            drv.inputDrvs.map.begin(),
+            drv.inputDrvs.map.end(),
+            [](auto & kv) { return !kv.second.childMap.empty(); })
+        != drv.inputDrvs.map.end();
+}
+
+
 std::string Derivation::unparse(const Store & store, bool maskOutputs,
-    std::map<std::string, StringSet> * actualInputs) const
+    DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const
 {
     std::string s;
     s.reserve(65536);
-    s += "Derive([";
+
+    /* Use older unversioned form if possible, for wider compat. Use
+       newer form only if we need it, which we do for
+       `Xp::DynamicDerivations`. */
+    if (hasDynamicDrvDep(*this)) {
+        s += "DrvWithVersion(";
+        // Only version we have so far
+        printUnquotedString(s, "xp-dyn-drv");
+        s += ",";
+    } else {
+        s += "Derive(";
+    }
 
     bool first = true;
+    s += "[";
     for (auto & i : outputs) {
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
@@ -421,17 +567,17 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
     s += "],[";
     first = true;
     if (actualInputs) {
-        for (auto & i : *actualInputs) {
+        for (auto & [drvHashModulo, childMap] : *actualInputs) {
             if (first) first = false; else s += ',';
-            s += '('; printUnquotedString(s, i.first);
-            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
+            s += '('; printUnquotedString(s, drvHashModulo);
+            unparseDerivedPathMapNode(store, s, childMap);
             s += ')';
         }
     } else {
-        for (auto & i : inputDrvs) {
+        for (auto & [drvPath, childMap] : inputDrvs.map) {
             if (first) first = false; else s += ',';
-            s += '('; printUnquotedString(s, store.printStorePath(i.first));
-            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
+            s += '('; printUnquotedString(s, store.printStorePath(drvPath));
+            unparseDerivedPathMapNode(store, s, childMap);
             s += ')';
         }
     }
@@ -466,7 +612,7 @@ bool isDerivation(std::string_view fileName)
 }
 
 
-std::string outputPathName(std::string_view drvName, std::string_view outputName) {
+std::string outputPathName(std::string_view drvName, OutputNameView outputName) {
     std::string res { drvName };
     if (outputName != "out") {
         res += "-";
@@ -665,18 +811,16 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         }
     }, drv.type().raw);
 
-    std::map<std::string, StringSet> inputs2;
-    for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
-        // Avoid lambda capture restriction with standard / Clang
-        auto & inputOutputs = inputOutputs0;
+    DerivedPathMap<StringSet>::ChildNode::Map inputs2;
+    for (auto & [drvPath, node] : drv.inputDrvs.map) {
         const auto & res = pathDerivationModulo(store, drvPath);
         if (res.kind == DrvHash::Kind::Deferred)
             kind = DrvHash::Kind::Deferred;
-        for (auto & outputName : inputOutputs) {
+        for (auto & outputName : node.value) {
             const auto h = get(res.hashes, outputName);
             if (!h)
                 throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
-            inputs2[h->to_string(Base16, false)].insert(outputName);
+            inputs2[h->to_string(Base16, false)].value.insert(outputName);
         }
     }
 
@@ -706,7 +850,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
     const auto hashAlgo = readString(in);
     const auto hash = readString(in);
 
-    return parseDerivationOutput(store, pathS, hashAlgo, hash);
+    return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings);
 }
 
 StringSet BasicDerivation::outputNames() const
@@ -810,7 +954,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
 }
 
 
-std::string hashPlaceholder(const std::string_view outputName)
+std::string hashPlaceholder(const OutputNameView outputName)
 {
     // FIXME: memoize?
     return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
@@ -821,6 +965,8 @@ std::string hashPlaceholder(const std::string_view outputName)
 
 static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
 {
+    debug("Rewriting the derivation");
+
     for (auto & rewrite : rewrites) {
         debug("rewriting %s as %s", rewrite.first, rewrite.second);
     }
@@ -859,14 +1005,70 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
 {
     std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
 
-    for (auto & input : inputDrvs)
-        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
-            if (outputPath)
-                inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
+    std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
+    accum = [&](auto & inputDrv, auto & node) {
+        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) {
+            if (outputPath) {
+                inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
+                if (auto p = get(node.childMap, outputName))
+                    accum(*outputPath, *p);
+            }
+        }
+    };
+
+    for (auto & [inputDrv, node] : inputDrvs.map)
+        accum(inputDrv, node);
 
     return tryResolve(store, inputDrvOutputs);
 }
 
+static bool tryResolveInput(
+    Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
+    const DownstreamPlaceholder * placeholderOpt,
+    const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode,
+    const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs)
+{
+    auto getOutput = [&](const std::string & outputName) {
+        auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName });
+        if (!actualPathOpt)
+            warn("output %s of input %s missing, aborting the resolving",
+                outputName,
+                store.printStorePath(inputDrv)
+            );
+        return actualPathOpt;
+    };
+
+    auto getPlaceholder = [&](const std::string & outputName) {
+        return placeholderOpt
+            ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName)
+            : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName);
+    };
+
+    for (auto & outputName : inputNode.value) {
+        auto actualPathOpt = getOutput(outputName);
+        if (!actualPathOpt) return false;
+        auto actualPath = *actualPathOpt;
+        if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+            inputRewrites.emplace(
+                getPlaceholder(outputName).render(),
+                store.printStorePath(actualPath));
+        }
+        inputSrcs.insert(std::move(actualPath));
+    }
+
+    for (auto & [outputName, childNode] : inputNode.childMap) {
+        auto actualPathOpt = getOutput(outputName);
+        if (!actualPathOpt) return false;
+        auto actualPath = *actualPathOpt;
+        auto nextPlaceholder = getPlaceholder(outputName);
+        if (!tryResolveInput(store, inputSrcs, inputRewrites,
+            &nextPlaceholder, actualPath, childNode,
+            inputDrvOutputs))
+            return false;
+    }
+    return true;
+}
+
 std::optional<BasicDerivation> Derivation::tryResolve(
     Store & store,
     const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
@@ -876,23 +1078,10 @@ std::optional<BasicDerivation> Derivation::tryResolve(
     // Input paths that we'll want to rewrite in the derivation
     StringMap inputRewrites;
 
-    for (auto & [inputDrv, inputOutputs] : inputDrvs) {
-        for (auto & outputName : inputOutputs) {
-            if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
-                if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
-                    inputRewrites.emplace(
-                        DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
-                        store.printStorePath(*actualPath));
-                }
-                resolved.inputSrcs.insert(*actualPath);
-            } else {
-                warn("output '%s' of input '%s' missing, aborting the resolving",
-                    outputName,
-                    store.printStorePath(inputDrv));
-                return {};
-            }
-        }
-    }
+    for (auto & [inputDrv, inputNode] : inputDrvs.map)
+        if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites,
+            nullptr, inputDrv, inputNode, inputDrvOutputs))
+            return std::nullopt;
 
     rewriteDerivation(store, resolved, inputRewrites);
 
@@ -963,7 +1152,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
 const Hash impureOutputHash = hashString(htSHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
-    const Store & store, std::string_view drvName, std::string_view outputName) const
+    const Store & store, std::string_view drvName, OutputNameView outputName) const
 {
     nlohmann::json res = nlohmann::json::object();
     std::visit(overloaded {
@@ -990,7 +1179,7 @@ nlohmann::json DerivationOutput::toJSON(
 
 
 DerivationOutput DerivationOutput::fromJSON(
-    const Store & store, std::string_view drvName, std::string_view outputName,
+    const Store & store, std::string_view drvName, OutputNameView outputName,
     const nlohmann::json & _json,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -1081,10 +1270,25 @@ nlohmann::json Derivation::toJSON(const Store & store) const
     }
 
     {
-        auto& inputDrvsObj = res["inputDrvs"];
-        inputDrvsObj = nlohmann::json ::object();
-        for (auto & input : inputDrvs)
-            inputDrvsObj[store.printStorePath(input.first)] = input.second;
+        std::function<nlohmann::json(const DerivedPathMap<StringSet>::ChildNode &)> doInput;
+        doInput = [&](const auto & inputNode) {
+            auto value = nlohmann::json::object();
+            value["outputs"] = inputNode.value;
+            {
+                auto next = nlohmann::json::object();
+                for (auto & [outputId, childNode] : inputNode.childMap)
+                    next[outputId] = doInput(childNode);
+                value["dynamicOutputs"] = std::move(next);
+            }
+            return value;
+        };
+        {
+            auto& inputDrvsObj = res["inputDrvs"];
+            inputDrvsObj = nlohmann::json::object();
+            for (auto & [inputDrv, inputNode] : inputDrvs.map) {
+                inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode);
+            }
+        }
     }
 
     res["system"] = platform;
@@ -1098,7 +1302,8 @@ nlohmann::json Derivation::toJSON(const Store & store) const
 
 Derivation Derivation::fromJSON(
     const Store & store,
-    const nlohmann::json & json)
+    const nlohmann::json & json,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     using nlohmann::detail::value_t;
 
@@ -1130,12 +1335,21 @@ Derivation Derivation::fromJSON(
     }
 
     try {
+        std::function<DerivedPathMap<StringSet>::ChildNode(const nlohmann::json &)> doInput;
+        doInput = [&](const auto & json) {
+            DerivedPathMap<StringSet>::ChildNode node;
+            node.value = static_cast<const StringSet &>(
+                ensureType(valueAt(json, "outputs"), value_t::array));
+            for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) {
+                xpSettings.require(Xp::DynamicDerivations);
+                node.childMap[outputId] = doInput(childNode);
+            }
+            return node;
+        };
         auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object);
-        for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) {
-            ensureType(inputOutputs, value_t::array);
-            res.inputDrvs[store.parseStorePath(inputDrvPath)] =
-                static_cast<const StringSet &>(inputOutputs);
-        }
+        for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
+            res.inputDrvs.map[store.parseStorePath(inputDrvPath)] =
+                doInput(inputOutputs);
     } catch (Error & e) {
         e.addTrace({}, "while reading key 'inputDrvs'");
         throw;

src/libstore/derivations.hh

@@ -6,7 +6,7 @@
 #include "hash.hh"
 #include "content-address.hh"
 #include "repair-flag.hh"
-#include "derived-path.hh"
+#include "derived-path-map.hh"
 #include "sync.hh"
 #include "comparator.hh"
 #include "variant-wrapper.hh"
@@ -55,7 +55,7 @@ struct DerivationOutput
          * @param drvName The name of the derivation this is an output of, without the `.drv`.
          * @param outputName The name of this output.
          */
-        StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+        StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
 
         GENERATE_CMP(CAFixed, me->ca);
     };
@@ -132,19 +132,19 @@ struct DerivationOutput
      * the safer interface provided by
      * BasicDerivation::outputsAndOptPaths
      */
-    std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+    std::optional<StorePath> path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
 
     nlohmann::json toJSON(
         const Store & store,
         std::string_view drvName,
-        std::string_view outputName) const;
+        OutputNameView outputName) const;
     /**
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivationOutput fromJSON(
         const Store & store,
         std::string_view drvName,
-        std::string_view outputName,
+        OutputNameView outputName,
         const nlohmann::json & json,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 };
@@ -323,13 +323,13 @@ struct Derivation : BasicDerivation
     /**
      * inputs that are sub-derivations
      */
-    DerivationInputs inputDrvs;
+    DerivedPathMap<std::set<OutputName>> inputDrvs;
 
     /**
      * Print a derivation.
      */
     std::string unparse(const Store & store, bool maskOutputs,
-        std::map<std::string, StringSet> * actualInputs = nullptr) const;
+        DerivedPathMap<StringSet>::ChildNode::Map * actualInputs = nullptr) const;
 
     /**
      * Return the underlying basic derivation but with these changes:
@@ -368,7 +368,8 @@ struct Derivation : BasicDerivation
     nlohmann::json toJSON(const Store & store) const;
     static Derivation fromJSON(
         const Store & store,
-        const nlohmann::json & json);
+        const nlohmann::json & json,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     GENERATE_CMP(Derivation,
         static_cast<const BasicDerivation &>(*me),
@@ -389,7 +390,11 @@ StorePath writeDerivation(Store & store,
 /**
  * Read a derivation from a file.
  */
-Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
+Derivation parseDerivation(
+    const Store & store,
+    std::string && s,
+    std::string_view name,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
 /**
  * \todo Remove.
@@ -405,7 +410,7 @@ bool isDerivation(std::string_view fileName);
  * This is usually <drv-name>-<output-name>, but is just <drv-name> when
  * the output name is "out".
  */
-std::string outputPathName(std::string_view drvName, std::string_view outputName);
+std::string outputPathName(std::string_view drvName, OutputNameView outputName);
 
 
 /**
@@ -499,7 +504,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
  * own outputs without needing to use the hash of a derivation in
  * itself, making the hash near-impossible to calculate.
  */
-std::string hashPlaceholder(const std::string_view outputName);
+std::string hashPlaceholder(const OutputNameView outputName);
 
 extern const Hash impureOutputHash;
 

src/libstore/derived-path-map.cc

@@ -0,0 +1,72 @@
+#include "derived-path-map.hh"
+
+namespace nix {
+
+template<typename V>
+typename DerivedPathMap<V>::ChildNode & DerivedPathMap<V>::ensureSlot(const SingleDerivedPath & k)
+{
+    std::function<ChildNode &(const SingleDerivedPath & )> initIter;
+    initIter = [&](const auto & k) -> auto & {
+        return std::visit(overloaded {
+            [&](const SingleDerivedPath::Opaque & bo) -> auto & {
+                // will not overwrite if already there
+                return map[bo.path];
+            },
+            [&](const SingleDerivedPath::Built & bfd) -> auto & {
+                auto & n = initIter(*bfd.drvPath);
+                return n.childMap[bfd.output];
+            },
+        }, k.raw());
+    };
+    return initIter(k);
+}
+
+template<typename V>
+typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const SingleDerivedPath & k)
+{
+    std::function<ChildNode *(const SingleDerivedPath & )> initIter;
+    initIter = [&](const auto & k) {
+        return std::visit(overloaded {
+            [&](const SingleDerivedPath::Opaque & bo) {
+                auto it = map.find(bo.path);
+                return it != map.end()
+                    ? &it->second
+                    : nullptr;
+            },
+            [&](const SingleDerivedPath::Built & bfd) {
+                auto * n = initIter(*bfd.drvPath);
+                if (!n) return (ChildNode *)nullptr;
+
+                auto it = n->childMap.find(bfd.output);
+                return it != n->childMap.end()
+                    ? &it->second
+                    : nullptr;
+            },
+        }, k.raw());
+    };
+    return initIter(k);
+}
+
+}
+
+// instantiations
+
+#include "create-derivation-and-realise-goal.hh"
+namespace nix {
+
+template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
+
+GENERATE_CMP_EXT(
+    template<>,
+    DerivedPathMap<std::set<std::string>>::ChildNode,
+    me->value,
+    me->childMap);
+
+GENERATE_CMP_EXT(
+    template<>,
+    DerivedPathMap<std::set<std::string>>,
+    me->map);
+
+template struct DerivedPathMap<std::set<std::string>>;
+
+};

src/libstore/derived-path-map.hh

@@ -0,0 +1,98 @@
+#pragma once
+
+#include "types.hh"
+#include "derived-path.hh"
+
+namespace nix {
+
+/**
+ * A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to
+ * values.
+ *
+ * Concretely, an n-ary tree, as described below. A
+ * `SingleDerivedPath::Opaque` maps to the value of an immediate child
+ * of the root node. A `SingleDerivedPath::Built` maps to a deeper child
+ * node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a
+ * child node (inductively), and then the
+ * `SingleDerivedPath::Built::output` is used to look up that child's
+ * child via its map. In this manner, every `SingleDerivedPath` is
+ * mapped to a child node.
+ *
+ * @param V A type to instantiate for each output. It should probably
+ * should be an "optional" type so not every interior node has to have a
+ * value. For example, the scheduler uses
+ * `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to
+ * remember which goals correspond to which outputs. `* const Something`
+ * or `std::optional<Something>` would also be good choices for
+ * "optional" types.
+ */
+template<typename V>
+struct DerivedPathMap {
+    /**
+     * A child node (non-root node).
+     */
+    struct ChildNode {
+        /**
+         * Value of this child node.
+         *
+         * @see DerivedPathMap for what `V` should be.
+         */
+        V value;
+
+        /**
+         * The map type for the root node.
+         */
+        using Map = std::map<OutputName, ChildNode>;
+
+        /**
+         * The map of the root node.
+         */
+        Map childMap;
+
+        DECLARE_CMP(ChildNode);
+    };
+
+    /**
+     * The map type for the root node.
+     */
+    using Map = std::map<StorePath, ChildNode>;
+
+    /**
+     * The map of root node.
+     */
+    Map map;
+
+    DECLARE_CMP(DerivedPathMap);
+
+    /**
+     * Find the node for `k`, creating it if needed.
+     *
+     * The node is referred to as a "slot" on the assumption that `V` is
+     * some sort of optional type, so the given key can be set or unset
+     * by changing this node.
+     */
+    ChildNode & ensureSlot(const SingleDerivedPath & k);
+
+    /**
+     * Like `ensureSlot` but does not create the slot if it doesn't exist.
+     *
+     * Read the entire description of `ensureSlot` to understand an
+     * important caveat here that "have slot" does *not* imply "key is
+     * set in map". To ensure a key is set one would need to get the
+     * child node (with `findSlot` or `ensureSlot`) *and* check the
+     * `ChildNode::value`.
+     */
+    ChildNode * findSlot(const SingleDerivedPath & k);
+};
+
+
+DECLARE_CMP_EXT(
+    template<>,
+    DerivedPathMap<std::set<std::string>>::,
+    DerivedPathMap<std::set<std::string>>);
+DECLARE_CMP_EXT(
+    template<>,
+    DerivedPathMap<std::set<std::string>>::ChildNode::,
+    DerivedPathMap<std::set<std::string>>::ChildNode);
+
+}

src/libstore/derived-path.cc

@@ -167,7 +167,7 @@ void drvRequireExperiment(
 
 SingleDerivedPath::Built SingleDerivedPath::Built::parse(
     const Store & store, ref<SingleDerivedPath> drv,
-    std::string_view output,
+    OutputNameView output,
     const ExperimentalFeatureSettings & xpSettings)
 {
     drvRequireExperiment(*drv, xpSettings);
@@ -179,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
 
 DerivedPath::Built DerivedPath::Built::parse(
     const Store & store, ref<SingleDerivedPath> drv,
-    std::string_view outputsS,
+    OutputNameView outputsS,
     const ExperimentalFeatureSettings & xpSettings)
 {
     drvRequireExperiment(*drv, xpSettings);

src/libstore/derived-path.hh

@@ -42,7 +42,7 @@ struct SingleDerivedPath;
  */
 struct SingleDerivedPathBuilt {
     ref<SingleDerivedPath> drvPath;
-    std::string output;
+    OutputName output;
 
     /**
      * Get the store path this is ultimately derived from (by realising
@@ -71,7 +71,7 @@ struct SingleDerivedPathBuilt {
      */
     static SingleDerivedPathBuilt parse(
         const Store & store, ref<SingleDerivedPath> drvPath,
-        std::string_view outputs,
+        OutputNameView outputs,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
 

src/libstore/downstream-placeholder.cc

@@ -11,7 +11,7 @@ std::string DownstreamPlaceholder::render() const
 
 DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
     const StorePath & drvPath,
-    std::string_view outputName,
+    OutputNameView outputName,
     const ExperimentalFeatureSettings & xpSettings)
 {
     xpSettings.require(Xp::CaDerivations);
@@ -25,7 +25,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
 
 DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
     const DownstreamPlaceholder & placeholder,
-    std::string_view outputName,
+    OutputNameView outputName,
     const ExperimentalFeatureSettings & xpSettings)
 {
     xpSettings.require(Xp::DynamicDerivations);

src/libstore/downstream-placeholder.hh

@@ -58,7 +58,7 @@ public:
      */
     static DownstreamPlaceholder unknownCaOutput(
         const StorePath & drvPath,
-        std::string_view outputName,
+        OutputNameView outputName,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
@@ -72,7 +72,7 @@ public:
      */
     static DownstreamPlaceholder unknownDerivation(
         const DownstreamPlaceholder & drvPlaceholder,
-        std::string_view outputName,
+        OutputNameView outputName,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**

src/libstore/globals.hh

@@ -343,7 +343,7 @@ public:
           users in `build-users-group`.
 
           UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS.
-        )"};
+        )", {}, true, Xp::AutoAllocateUids};
 
     Setting<uint32_t> startId{this,
         #if __linux__
@@ -554,7 +554,7 @@ public:
         R"(
           This option determines the maximum size of the `tmpfs` filesystem
           mounted on `/dev/shm` in Linux sandboxes. For the format, see the
-          description of the `size` option of `tmpfs` in mount8. The default
+          description of the `size` option of `tmpfs` in mount(8). The default
           is `50%`.
         )"};
 
@@ -697,19 +697,40 @@ public:
         getDefaultSystemFeatures(),
         "system-features",
         R"(
-          A set of system “features” supported by this machine, e.g. `kvm`.
-          Derivations can express a dependency on such features through the
-          derivation attribute `requiredSystemFeatures`. For example, the
-          attribute
+          A set of system “features” supported by this machine.
 
-              requiredSystemFeatures = [ "kvm" ];
+          This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations.
 
-          ensures that the derivation can only be built on a machine with the
-          `kvm` feature.
+          A derivation can require system features in the [`requiredSystemFeatures` attribute](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures), and the machine to build the derivation must have them.
 
-          This setting by default includes `kvm` if `/dev/kvm` is accessible,
-          and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
-          that are used in Nixpkgs to route builds to specific machines.
+          System features are user-defined, but Nix sets the following defaults:
+
+          - `kvm`
+
+            Included by default if `/dev/kvm` is accessible.
+
+          - `nixos-test`, `benchmark`, `big-parallel`
+
+            These historical pseudo-features are always enabled for backwards compatibility, as they are used in Nixpkgs to route Hydra builds to specific machines.
+
+          - `ca-derivations`
+
+            Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled.
+
+            This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed).
+
+          - `recursive-nix`
+
+            Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled.
+
+          - `uid-range`
+
+            On Linux, Nix can run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available.
+            This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn].
+
+            [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix.
+
+            Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled.
         )", {}, false};
 
     Setting<Strings> substituters{

src/libstore/misc.cc

@@ -125,14 +125,26 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
 
     std::function<void(DerivedPath)> doPath;
 
+    std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> enqueueDerivedPaths;
+
+    enqueueDerivedPaths = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+        if (!inputNode.value.empty())
+            pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value }));
+        for (const auto & [outputName, childNode] : inputNode.childMap)
+            enqueueDerivedPaths(
+                make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                childNode);
+    };
+
     auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
         {
             auto state(state_.lock());
             state->willBuild.insert(drvPath);
         }
 
-        for (auto & i : drv.inputDrvs)
-            pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second }));
+        for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) {
+            enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode);
+        }
     };
 
     auto checkOutput = [&](
@@ -322,24 +334,41 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
 {
     std::set<Realisation> inputRealisations;
 
-    for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
-        const auto outputHashes =
-            staticOutputHashes(store, store.readDerivation(inputDrv));
-        for (const auto & outputName : outputNames) {
-            auto outputHash = get(outputHashes, outputName);
-            if (!outputHash)
-                throw Error(
-                    "output '%s' of derivation '%s' isn't realised", outputName,
-                    store.printStorePath(inputDrv));
-            auto thisRealisation = store.queryRealisation(
-                DrvOutput{*outputHash, outputName});
-            if (!thisRealisation)
-                throw Error(
-                    "output '%s' of derivation '%s' isn't built", outputName,
-                    store.printStorePath(inputDrv));
-            inputRealisations.insert(*thisRealisation);
+    std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
+
+    accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+        if (!inputNode.value.empty()) {
+            auto outputHashes =
+                staticOutputHashes(store, store.readDerivation(inputDrv));
+            for (const auto & outputName : inputNode.value) {
+                auto outputHash = get(outputHashes, outputName);
+                if (!outputHash)
+                    throw Error(
+                        "output '%s' of derivation '%s' isn't realised", outputName,
+                        store.printStorePath(inputDrv));
+                auto thisRealisation = store.queryRealisation(
+                    DrvOutput{*outputHash, outputName});
+                if (!thisRealisation)
+                    throw Error(
+                        "output '%s' of derivation '%s' isn’t built", outputName,
+                        store.printStorePath(inputDrv));
+                inputRealisations.insert(*thisRealisation);
+            }
         }
-    }
+        if (!inputNode.value.empty()) {
+            auto d = makeConstantStorePathRef(inputDrv);
+            for (const auto & [outputName, childNode] : inputNode.childMap) {
+                SingleDerivedPath next = SingleDerivedPath::Built { d, outputName };
+                accumRealisations(
+                    // TODO deep resolutions for dynamic derivations, issue #8947, would go here.
+                    resolveDerivedPath(store, next),
+                    childNode);
+            }
+        }
+    };
+
+    for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
+        accumRealisations(inputDrv, inputNode);
 
     auto info = store.queryPathInfo(outputPath);
 
@@ -399,8 +428,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store
                     store.printStorePath(drvPath), bfd.output);
             auto & optPath = outputPaths.at(bfd.output);
             if (!optPath)
-                throw Error("'%s' does not yet map to a known concrete store path",
-                    bfd.to_string(store));
+                throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output);
             return *optPath;
         },
     }, req.raw());

src/libstore/outputs-spec.hh

@@ -13,24 +13,36 @@
 
 namespace nix {
 
+/**
+ * An (owned) output name. Just a type alias used to make code more
+ * readible.
+ */
+typedef std::string OutputName;
+
+/**
+ * A borrowed output name. Just a type alias used to make code more
+ * readible.
+ */
+typedef std::string_view OutputNameView;
+
 struct OutputsSpec {
     /**
      * A non-empty set of outputs, specified by name
      */
-    struct Names : std::set<std::string> {
-        using std::set<std::string>::set;
+    struct Names : std::set<OutputName> {
+        using std::set<OutputName>::set;
 
         /* These need to be "inherited manually" */
 
-        Names(const std::set<std::string> & s)
-            : std::set<std::string>(s)
+        Names(const std::set<OutputName> & s)
+            : std::set<OutputName>(s)
         { assert(!empty()); }
 
         /**
          * Needs to be "inherited manually"
          */
-        Names(std::set<std::string> && s)
-            : std::set<std::string>(s)
+        Names(std::set<OutputName> && s)
+            : std::set<OutputName>(s)
         { assert(!empty()); }
 
         /* This set should always be non-empty, so we delete this
@@ -57,7 +69,7 @@ struct OutputsSpec {
      */
     OutputsSpec() = delete;
 
-    bool contains(const std::string & output) const;
+    bool contains(const OutputName & output) const;
 
     /**
      * Create a new OutputsSpec which is the union of this and that.

src/libstore/profiles.cc

@@ -183,7 +183,7 @@ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bo
     iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; });
 
     // Skip over `max` generations, preserving them
-    for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep);
+    for (GenerationNumber keep = 0; i != gens.rend() && keep < max; ++i, ++keep);
 
     // Delete the rest
     for (; i != gens.rend(); ++i)

src/libstore/realisation.hh

@@ -34,7 +34,7 @@ struct DrvOutput {
     /**
      * The name of the output.
      */
-    std::string outputName;
+    OutputName outputName;
 
     std::string to_string() const;
 
@@ -84,7 +84,7 @@ struct Realisation {
  * Since these are the outputs of a single derivation, we know the
  * output names are unique so we can use them as the map key.
  */
-typedef std::map<std::string, Realisation> SingleDrvOutputs;
+typedef std::map<OutputName, Realisation> SingleDrvOutputs;
 
 /**
  * Collection type for multiple derivations' outputs' `Realisation`s.
@@ -146,7 +146,7 @@ public:
     MissingRealisation(DrvOutput & outputId)
         : MissingRealisation(outputId.outputName, outputId.strHash())
     {}
-    MissingRealisation(std::string_view drv, std::string outputName)
+    MissingRealisation(std::string_view drv, OutputName outputName)
         : Error( "cannot operate on output '%s' of the "
                 "unbuilt derivation '%s'",
                 outputName,

src/libstore/remote-store.cc

@@ -172,7 +172,24 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source,
     auto ex = handle->processStderr(sink, source, flush);
     if (ex) {
         daemonException = true;
-        std::rethrow_exception(ex);
+        try {
+            std::rethrow_exception(ex);
+        } catch (const Error & e) {
+            // Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded.
+            // To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the
+            // old incomprehensible error here, so that we can explain to users what's going on when their daemon is
+            // older than #4628 (2023).
+            if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) &&
+                GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35)
+            {
+                auto m = e.msg();
+                if (m.find("parsing derivation") != std::string::npos &&
+                    m.find("expected string") != std::string::npos &&
+                    m.find("Derive([") != std::string::npos)
+                    throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
+            }
+            throw;
+        }
     }
 }
 

src/libstore/store-api.hh

@@ -153,19 +153,22 @@ struct StoreConfig : public Config
 
     Setting<int> priority{this, 0, "priority",
         R"(
-          Priority of this store when used as a substituter. A lower value means a higher priority.
+          Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
+          A lower value means a higher priority.
         )"};
 
     Setting<bool> wantMassQuery{this, false, "want-mass-query",
         R"(
-          Whether this store (when used as a substituter) can be
-          queried efficiently for path validity.
+          Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
         )"};
 
     Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
         "system-features",
-        "Optional features that the system this store builds on implements (like \"kvm\")."};
+        R"(
+          Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations.
 
+          Example: `"kvm"`
+        )" };
 };
 
 class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig

src/libstore/tests/derivation.cc

@@ -42,6 +42,26 @@ class ImpureDerivationTest : public DerivationTest
     }
 };
 
+TEST_F(DerivationTest, BadATerm_version) {
+    ASSERT_THROW(
+        parseDerivation(
+            *store,
+            R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+            "whatever",
+            mockXpSettings),
+        FormatError);
+}
+
+TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
+    ASSERT_THROW(
+        parseDerivation(
+            *store,
+            R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+            "dyn-dep-derivation",
+            mockXpSettings),
+        FormatError);
+}
+
 #define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
     TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) {    \
         using nlohmann::literals::operator "" _json;           \
@@ -143,34 +163,93 @@ TEST_JSON(ImpureDerivationTest, impure,
 
 #undef TEST_JSON
 
-#define TEST_JSON(NAME, STR, VAL, DRV_NAME)                     \
-    TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) {   \
-        using nlohmann::literals::operator "" _json;            \
-        ASSERT_EQ(                                              \
-            STR ## _json,                                       \
-            (Derivation { VAL }).toJSON(*store));               \
-    }                                                           \
-                                                                \
-    TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \
-        using nlohmann::literals::operator "" _json;            \
-        ASSERT_EQ(                                              \
-            Derivation { VAL },                                 \
-            Derivation::fromJSON(                               \
-                *store,                                         \
-                STR ## _json));                                 \
+#define TEST_JSON(FIXTURE, NAME, STR, VAL)                       \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) {           \
+        using nlohmann::literals::operator "" _json;             \
+        ASSERT_EQ(                                               \
+            STR ## _json,                                        \
+            (VAL).toJSON(*store));                               \
+    }                                                            \
+                                                                 \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) {         \
+        using nlohmann::literals::operator "" _json;             \
+        ASSERT_EQ(                                               \
+            (VAL),                                               \
+            Derivation::fromJSON(                                \
+                *store,                                          \
+                STR ## _json,                                    \
+                mockXpSettings));                                \
     }
 
-TEST_JSON(simple,
+#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME)            \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) {          \
+        ASSERT_EQ(                                               \
+            STR,                                                 \
+            (VAL).unparse(*store, false));                       \
+    }                                                            \
+                                                                 \
+    TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) {        \
+        auto parsed = parseDerivation(                           \
+            *store,                                              \
+            STR,                                                 \
+            DRV_NAME,                                            \
+            mockXpSettings);                                     \
+        ASSERT_EQ(                                               \
+            (VAL).toJSON(*store),                                \
+            parsed.toJSON(*store));                              \
+        ASSERT_EQ(                                               \
+            (VAL),                                               \
+            parsed);                                             \
+    }
+
+Derivation makeSimpleDrv(const Store & store) {
+    Derivation drv;
+    drv.name = "simple-derivation";
+    drv.inputSrcs = {
+        store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
+    };
+    drv.inputDrvs = {
+        .map = {
+            {
+                store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
+                {
+                    .value = {
+                        "cat",
+                        "dog",
+                    },
+                },
+            },
+        },
+    };
+    drv.platform = "wasm-sel4";
+    drv.builder = "foo";
+    drv.args = {
+        "bar",
+        "baz",
+    };
+    drv.env = {
+        {
+            "BIG_BAD",
+            "WOLF",
+        },
+    };
+    return drv;
+}
+
+TEST_JSON(DerivationTest, simple,
     R"({
-      "name": "my-derivation",
+      "name": "simple-derivation",
       "inputSrcs": [
         "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
       ],
       "inputDrvs": {
-        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [
-          "cat",
-          "dog"
-        ]
+        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
+          "dynamicOutputs": {},
+          "outputs": [
+            "cat",
+            "dog"
+          ]
+        }
       },
       "system": "wasm-sel4",
       "builder": "foo",
@@ -183,37 +262,108 @@ TEST_JSON(simple,
       },
       "outputs": {}
     })",
-    ({
-        Derivation drv;
-        drv.name = "my-derivation";
-        drv.inputSrcs = {
-            store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
-        };
-        drv.inputDrvs = {
+    makeSimpleDrv(*store))
+
+TEST_ATERM(DerivationTest, simple,
+    R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+    makeSimpleDrv(*store),
+    "simple-derivation")
+
+Derivation makeDynDepDerivation(const Store & store) {
+    Derivation drv;
+    drv.name = "dyn-dep-derivation";
+    drv.inputSrcs = {
+        store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
+    };
+    drv.inputDrvs = {
+        .map = {
             {
-                store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
-                {
-                    "cat",
-                    "dog",
+                store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"),
+                DerivedPathMap<StringSet>::ChildNode {
+                    .value = {
+                        "cat",
+                        "dog",
+                    },
+                    .childMap = {
+                        {
+                            "cat",
+                            DerivedPathMap<StringSet>::ChildNode {
+                                .value = {
+                                    "kitten",
+                                },
+                            },
+                        },
+                        {
+                            "goose",
+                            DerivedPathMap<StringSet>::ChildNode {
+                                .value = {
+                                    "gosling",
+                                },
+                            },
+                        },
+                    },
                 },
-            }
-        };
-        drv.platform = "wasm-sel4";
-        drv.builder = "foo";
-        drv.args = {
-            "bar",
-            "baz",
-        };
-        drv.env = {
-            {
-                "BIG_BAD",
-                "WOLF",
             },
-        };
-        drv;
-    }),
-    "drv-name")
+        },
+    };
+    drv.platform = "wasm-sel4";
+    drv.builder = "foo";
+    drv.args = {
+        "bar",
+        "baz",
+    };
+    drv.env = {
+        {
+            "BIG_BAD",
+            "WOLF",
+        },
+    };
+    return drv;
+}
+
+TEST_JSON(DynDerivationTest, dynDerivationDeps,
+    R"({
+      "name": "dyn-dep-derivation",
+      "inputSrcs": [
+        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
+      ],
+      "inputDrvs": {
+        "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
+          "dynamicOutputs": {
+            "cat": {
+              "dynamicOutputs": {},
+              "outputs": ["kitten"]
+            },
+            "goose": {
+              "dynamicOutputs": {},
+              "outputs": ["gosling"]
+            }
+          },
+          "outputs": [
+            "cat",
+            "dog"
+          ]
+        }
+      },
+      "system": "wasm-sel4",
+      "builder": "foo",
+      "args": [
+        "bar",
+        "baz"
+      ],
+      "env": {
+        "BIG_BAD": "WOLF"
+      },
+      "outputs": {}
+    })",
+    makeDynDepDerivation(*store))
+
+TEST_ATERM(DynDerivationTest, dynDerivationDeps,
+    R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
+    makeDynDepDerivation(*store),
+    "dyn-dep-derivation")
 
 #undef TEST_JSON
+#undef TEST_ATERM
 
 }

src/libstore/tests/derived-path.cc

@@ -130,6 +130,8 @@ TEST_F(DerivedPathTest, built_built_xp) {
         MissingExperimentalFeature);
 }
 
+#ifndef COVERAGE
+
 RC_GTEST_FIXTURE_PROP(
     DerivedPathTest,
     prop_legacy_round_rip,
@@ -146,4 +148,6 @@ RC_GTEST_FIXTURE_PROP(
     RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store)));
 }
 
+#endif
+
 }

src/libstore/tests/outputs-spec.cc

@@ -224,6 +224,8 @@ Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
 
 namespace nix {
 
+#ifndef COVERAGE
+
 RC_GTEST_PROP(
     OutputsSpec,
     prop_round_rip,
@@ -232,4 +234,6 @@ RC_GTEST_PROP(
     RC_ASSERT(o == OutputsSpec::parse(o.to_string()));
 }
 
+#endif
+
 }

src/libstore/tests/path.cc

@@ -134,6 +134,8 @@ Gen<StorePath> Arbitrary<StorePath>::arbitrary()
 
 namespace nix {
 
+#ifndef COVERAGE
+
 RC_GTEST_FIXTURE_PROP(
     StorePathTest,
     prop_regex_accept,
@@ -150,4 +152,6 @@ RC_GTEST_FIXTURE_PROP(
     RC_ASSERT(p == store->parseStorePath(store->printStorePath(p)));
 }
 
+#endif
+
 }

src/libstore/tests/worker-protocol.cc

@@ -0,0 +1,139 @@
+#include <regex>
+
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+
+#include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
+#include "derived-path.hh"
+#include "tests/libstore.hh"
+
+namespace nix {
+
+class WorkerProtoTest : public LibStoreTest
+{
+public:
+    Path unitTestData = getEnv("_NIX_TEST_UNIT_DATA").value() + "/libstore/worker-protocol";
+
+    bool testAccept() {
+        return getEnv("_NIX_TEST_ACCEPT") == "1";
+    }
+
+    Path goldenMaster(std::string_view testStem) {
+        return unitTestData + "/" + testStem + ".bin";
+    }
+
+    /**
+     * Golden test for `T` reading
+     */
+    template<typename T>
+    void readTest(PathView testStem, T value)
+    {
+        if (testAccept())
+        {
+            GTEST_SKIP() << "Cannot read golden master because another test is also updating it";
+        }
+        else
+        {
+            auto expected = readFile(goldenMaster(testStem));
+
+            T got = ({
+                StringSource from { expected };
+                WorkerProto::Serialise<T>::read(
+                    *store,
+                    WorkerProto::ReadConn { .from = from });
+            });
+
+            ASSERT_EQ(got, value);
+        }
+    }
+
+    /**
+     * Golden test for `T` write
+     */
+    template<typename T>
+    void writeTest(PathView testStem, const T & value)
+    {
+        auto file = goldenMaster(testStem);
+
+        StringSink to;
+        WorkerProto::write(
+            *store,
+            WorkerProto::WriteConn { .to = to },
+            value);
+
+        if (testAccept())
+        {
+            createDirs(dirOf(file));
+            writeFile(file, to.s);
+            GTEST_SKIP() << "Updating golden master";
+        }
+        else
+        {
+            auto expected = readFile(file);
+            ASSERT_EQ(to.s, expected);
+        }
+    }
+};
+
+#define CHARACTERIZATION_TEST(NAME, STEM, VALUE)  \
+    TEST_F(WorkerProtoTest, NAME ## _read) {   \
+        readTest(STEM, VALUE);                 \
+    }                                          \
+    TEST_F(WorkerProtoTest, NAME ## _write) {  \
+        writeTest(STEM, VALUE);                \
+    }
+
+CHARACTERIZATION_TEST(
+    string,
+    "string",
+    (std::tuple<std::string, std::string, std::string, std::string, std::string> {
+        "",
+        "hi",
+        "white rabbit",
+        "大白兔",
+        "oh no \0\0\0 what was that!",
+    }))
+
+CHARACTERIZATION_TEST(
+    storePath,
+    "store-path",
+    (std::tuple<StorePath, StorePath> {
+        StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+        StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" },
+    }))
+
+CHARACTERIZATION_TEST(
+    contentAddress,
+    "content-address",
+    (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
+        ContentAddress {
+            .method = TextIngestionMethod {},
+            .hash = hashString(HashType::htSHA256, "Derive(...)"),
+        },
+        ContentAddress {
+            .method = FileIngestionMethod::Flat,
+            .hash = hashString(HashType::htSHA1, "blob blob..."),
+        },
+        ContentAddress {
+            .method = FileIngestionMethod::Recursive,
+            .hash = hashString(HashType::htSHA256, "(...)"),
+        },
+    }))
+
+CHARACTERIZATION_TEST(
+    derivedPath,
+    "derived-path",
+    (std::tuple<DerivedPath, DerivedPath> {
+        DerivedPath::Opaque {
+            .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+        },
+        DerivedPath::Built {
+            .drvPath = makeConstantStorePathRef(StorePath {
+                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+            }),
+            .outputs = OutputsSpec::Names { "x", "y" },
+        },
+    }))
+
+}

src/libstore/worker-protocol-impl.hh

@@ -75,4 +75,20 @@ void WorkerProto::Serialise<std::map<K, V>>::write(const Store & store, WorkerPr
     }
 }
 
+template<typename... Ts>
+std::tuple<Ts...> WorkerProto::Serialise<std::tuple<Ts...>>::read(const Store & store, WorkerProto::ReadConn conn)
+{
+    return std::tuple<Ts...> {
+        WorkerProto::Serialise<Ts>::read(store, conn)...,
+    };
+}
+
+template<typename... Ts>
+void WorkerProto::Serialise<std::tuple<Ts...>>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple<Ts...> & res)
+{
+    std::apply([&]<typename... Us>(const Us &... args) {
+        (WorkerProto::Serialise<Us>::write(store, conn, args), ...);
+    }, res);
+}
+
 }

src/libstore/worker-protocol.hh

@@ -28,6 +28,8 @@ class Store;
 struct Source;
 
 // items being serialised
+class StorePath;
+struct ContentAddress;
 struct DerivedPath;
 struct DrvOutput;
 struct Realisation;
@@ -220,6 +222,8 @@ template<typename T>
 MAKE_WORKER_PROTO(std::vector<T>);
 template<typename T>
 MAKE_WORKER_PROTO(std::set<T>);
+template<typename... Ts>
+MAKE_WORKER_PROTO(std::tuple<Ts...>);
 
 template<typename K, typename V>
 #define X_ std::map<K, V>

src/libutil/archive.cc

@@ -27,8 +27,6 @@ struct ArchiveSettings : Config
         #endif
         "use-case-hack",
         "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
-    Setting<bool> preallocateContents{this, false, "preallocate-contents",
-        "Whether to preallocate files when writing objects with known size."};
 };
 
 static ArchiveSettings archiveSettings;
@@ -302,71 +300,6 @@ void parseDump(ParseSink & sink, Source & source)
 }
 
 
-struct RestoreSink : ParseSink
-{
-    Path dstPath;
-    AutoCloseFD fd;
-
-    void createDirectory(const Path & path) override
-    {
-        Path p = dstPath + path;
-        if (mkdir(p.c_str(), 0777) == -1)
-            throw SysError("creating directory '%1%'", p);
-    };
-
-    void createRegularFile(const Path & path) override
-    {
-        Path p = dstPath + path;
-        fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
-        if (!fd) throw SysError("creating file '%1%'", p);
-    }
-
-    void closeRegularFile() override
-    {
-        /* Call close explicitly to make sure the error is checked */
-        fd.close();
-    }
-
-    void isExecutable() override
-    {
-        struct stat st;
-        if (fstat(fd.get(), &st) == -1)
-            throw SysError("fstat");
-        if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
-            throw SysError("fchmod");
-    }
-
-    void preallocateContents(uint64_t len) override
-    {
-        if (!archiveSettings.preallocateContents)
-            return;
-
-#if HAVE_POSIX_FALLOCATE
-        if (len) {
-            errno = posix_fallocate(fd.get(), 0, len);
-            /* Note that EINVAL may indicate that the underlying
-               filesystem doesn't support preallocation (e.g. on
-               OpenSolaris).  Since preallocation is just an
-               optimisation, ignore it. */
-            if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
-                throw SysError("preallocating file of %1% bytes", len);
-        }
-#endif
-    }
-
-    void receiveContents(std::string_view data) override
-    {
-        writeFull(fd.get(), data);
-    }
-
-    void createSymlink(const Path & path, const std::string & target) override
-    {
-        Path p = dstPath + path;
-        nix::createSymlink(target, p);
-    }
-};
-
-
 void restorePath(const Path & path, Source & source)
 {
     RestoreSink sink;

src/libutil/archive.hh

@@ -3,6 +3,7 @@
 
 #include "types.hh"
 #include "serialise.hh"
+#include "fs-sink.hh"
 
 
 namespace nix {
@@ -72,22 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
  */
 void dumpString(std::string_view s, Sink & sink);
 
-/**
- * \todo Fix this API, it sucks.
- */
-struct ParseSink
-{
-    virtual void createDirectory(const Path & path) { };
-
-    virtual void createRegularFile(const Path & path) { };
-    virtual void closeRegularFile() { };
-    virtual void isExecutable() { };
-    virtual void preallocateContents(uint64_t size) { };
-    virtual void receiveContents(std::string_view data) { };
-
-    virtual void createSymlink(const Path & path, const std::string & target) { };
-};
-
 /**
  * If the NAR archive contains a single file at top-level, then save
  * the contents of the file to `s`.  Otherwise barf.

src/libutil/args.cc

@@ -236,6 +236,7 @@ nlohmann::json Args::toJSON()
 
     for (auto & [name, flag] : longFlags) {
         auto j = nlohmann::json::object();
+        j["hiddenCategory"] = hiddenCategories.count(flag->category) > 0;
         if (flag->aliases.count(name)) continue;
         if (flag->shortName)
             j["shortName"] = std::string(1, flag->shortName);

src/libutil/comparator.hh

@@ -1,21 +1,48 @@
 #pragma once
 ///@file
 
+#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \
+    PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const;
+#define DECLARE_EQUAL(prefix, qualification, my_type) \
+    DECLARE_ONE_CMP(prefix, qualification, ==, my_type)
+#define DECLARE_LEQ(prefix, qualification, my_type) \
+    DECLARE_ONE_CMP(prefix, qualification, <, my_type)
+#define DECLARE_NEQ(prefix, qualification, my_type) \
+    DECLARE_ONE_CMP(prefix, qualification, !=, my_type)
+
+#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \
+    PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \
+      __VA_OPT__(const MY_TYPE * me = this;) \
+      auto fields1 = std::make_tuple( __VA_ARGS__ ); \
+      __VA_OPT__(me = &other;) \
+      auto fields2 = std::make_tuple( __VA_ARGS__ ); \
+      return fields1 COMPARATOR fields2; \
+    }
+#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \
+    GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args)
+#define GENERATE_LEQ(prefix, qualification, my_type, args...) \
+    GENERATE_ONE_CMP(prefix, qualification, <, my_type, args)
+#define GENERATE_NEQ(prefix, qualification, my_type, args...) \
+    GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args)
+
 /**
  * Declare comparison methods without defining them.
  */
-#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \
-    bool operator COMPARATOR(const MY_TYPE & other) const;
-#define DECLARE_EQUAL(my_type) \
-    DECLARE_ONE_CMP(==, my_type)
-#define DECLARE_LEQ(my_type) \
-    DECLARE_ONE_CMP(<, my_type)
-#define DECLARE_NEQ(my_type) \
-    DECLARE_ONE_CMP(!=, my_type)
 #define DECLARE_CMP(my_type) \
-    DECLARE_EQUAL(my_type) \
-    DECLARE_LEQ(my_type) \
-    DECLARE_NEQ(my_type)
+    DECLARE_EQUAL(,,my_type) \
+    DECLARE_LEQ(,,my_type) \
+    DECLARE_NEQ(,,my_type)
+
+/**
+ * @param prefix This is for something before each declaration like
+ * `template<classname Foo>`.
+ *
+ * @param my_type the type are defining operators for.
+ */
+#define DECLARE_CMP_EXT(prefix, qualification, my_type) \
+    DECLARE_EQUAL(prefix, qualification, my_type) \
+    DECLARE_LEQ(prefix, qualification, my_type) \
+    DECLARE_NEQ(prefix, qualification, my_type)
 
 /**
  * Awful hacky generation of the comparison operators by doing a lexicographic
@@ -33,18 +60,18 @@
  * }
  * ```
  */
-#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \
-    bool operator COMPARATOR(const MY_TYPE& other) const { \
-      __VA_OPT__(const MY_TYPE* me = this;) \
-      auto fields1 = std::make_tuple( __VA_ARGS__ ); \
-      __VA_OPT__(me = &other;) \
-      auto fields2 = std::make_tuple( __VA_ARGS__ ); \
-      return fields1 COMPARATOR fields2; \
-    }
-#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
-#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
-#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
 #define GENERATE_CMP(args...) \
-    GENERATE_EQUAL(args) \
-    GENERATE_LEQ(args) \
-    GENERATE_NEQ(args)
+    GENERATE_EQUAL(,,args) \
+    GENERATE_LEQ(,,args) \
+    GENERATE_NEQ(,,args)
+
+/**
+ * @param prefix This is for something before each declaration like
+ * `template<classname Foo>`.
+ *
+ * @param my_type the type are defining operators for.
+ */
+#define GENERATE_CMP_EXT(prefix, my_type, args...) \
+    GENERATE_EQUAL(prefix, my_type ::, my_type, args) \
+    GENERATE_LEQ(prefix, my_type ::, my_type, args) \
+    GENERATE_NEQ(prefix, my_type ::, my_type, args)

src/libutil/config.hh

@@ -36,8 +36,8 @@ namespace nix {
  *
  *   std::map<std::string, Config::SettingInfo> settings;
  *   config.getSettings(settings);
- *   config["system"].description == "the current system"
- *   config["system"].value == "x86_64-linux"
+ *   settings["system"].description == "the current system"
+ *   settings["system"].value == "x86_64-linux"
  *
  *
  * The above retrieves all currently known settings from the `Config` object

src/libutil/error.cc

@@ -14,6 +14,11 @@ void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, boo
     err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
 }
 
+void throwExceptionSelfCheck(){
+    // This is meant to be caught in initLibUtil()
+    throw SysError("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded.");
+}
+
 // c++ std::exception descendants must have a 'const char* what()' function.
 // This stringifies the error and caches it for use by what(), or similarly by msg().
 const std::string & BaseError::calcWhat() const

src/libutil/error.hh

@@ -214,4 +214,8 @@ public:
     }
 };
 
+/** Throw an exception for the purpose of checking that exception handling works; see 'initLibUtil()'.
+ */
+void throwExceptionSelfCheck();
+
 }

src/libutil/experimental-features.cc

@@ -163,6 +163,8 @@ constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
         .tag = Xp::ReplFlake,
         .name = "repl-flake",
         .description = R"(
+            *Enabled with [`flakes`](#xp-feature-flakes) since 2.19*
+
             Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands.
         )",
     },
@@ -171,7 +173,7 @@ constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
         .name = "auto-allocate-uids",
         .description = R"(
             Allows Nix to automatically pick UIDs for builds, rather than creating
-            `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details.
+            `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details.
         )",
     },
     {
@@ -179,7 +181,7 @@ constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
         .name = "cgroups",
         .description = R"(
             Allows Nix to execute builds inside cgroups. See
-            the [`use-cgroups`](#conf-use-cgroups) setting for details.
+            the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details.
         )",
     },
     {
@@ -272,7 +274,7 @@ std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFea
 }
 
 MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature)
-    : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature))
+    : Error("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature))
     , missingFeature(feature)
 {}
 

src/libutil/fs-sink.cc

@@ -0,0 +1,77 @@
+#include <fcntl.h>
+
+#include "config.hh"
+#include "fs-sink.hh"
+
+namespace nix {
+
+struct RestoreSinkSettings : Config
+{
+    Setting<bool> preallocateContents{this, false, "preallocate-contents",
+        "Whether to preallocate files when writing objects with known size."};
+};
+
+static RestoreSinkSettings restoreSinkSettings;
+
+static GlobalConfig::Register r1(&restoreSinkSettings);
+
+
+void RestoreSink::createDirectory(const Path & path)
+{
+    Path p = dstPath + path;
+    if (mkdir(p.c_str(), 0777) == -1)
+        throw SysError("creating directory '%1%'", p);
+};
+
+void RestoreSink::createRegularFile(const Path & path)
+{
+    Path p = dstPath + path;
+    fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
+    if (!fd) throw SysError("creating file '%1%'", p);
+}
+
+void RestoreSink::closeRegularFile()
+{
+    /* Call close explicitly to make sure the error is checked */
+    fd.close();
+}
+
+void RestoreSink::isExecutable()
+{
+    struct stat st;
+    if (fstat(fd.get(), &st) == -1)
+        throw SysError("fstat");
+    if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
+        throw SysError("fchmod");
+}
+
+void RestoreSink::preallocateContents(uint64_t len)
+{
+    if (!restoreSinkSettings.preallocateContents)
+        return;
+
+#if HAVE_POSIX_FALLOCATE
+    if (len) {
+        errno = posix_fallocate(fd.get(), 0, len);
+        /* Note that EINVAL may indicate that the underlying
+           filesystem doesn't support preallocation (e.g. on
+           OpenSolaris).  Since preallocation is just an
+           optimisation, ignore it. */
+        if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
+            throw SysError("preallocating file of %1% bytes", len);
+    }
+#endif
+}
+
+void RestoreSink::receiveContents(std::string_view data)
+{
+    writeFull(fd.get(), data);
+}
+
+void RestoreSink::createSymlink(const Path & path, const std::string & target)
+{
+    Path p = dstPath + path;
+    nix::createSymlink(target, p);
+}
+
+}

src/libutil/fs-sink.hh

@@ -0,0 +1,42 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "serialise.hh"
+
+namespace nix {
+
+/**
+ * \todo Fix this API, it sucks.
+ */
+struct ParseSink
+{
+    virtual void createDirectory(const Path & path) { };
+
+    virtual void createRegularFile(const Path & path) { };
+    virtual void closeRegularFile() { };
+    virtual void isExecutable() { };
+    virtual void preallocateContents(uint64_t size) { };
+    virtual void receiveContents(std::string_view data) { };
+
+    virtual void createSymlink(const Path & path, const std::string & target) { };
+};
+
+struct RestoreSink : ParseSink
+{
+    Path dstPath;
+    AutoCloseFD fd;
+
+
+    void createDirectory(const Path & path) override;
+
+    void createRegularFile(const Path & path) override;
+    void closeRegularFile() override;
+    void isExecutable() override;
+    void preallocateContents(uint64_t size) override;
+    void receiveContents(std::string_view data) override;
+
+    void createSymlink(const Path & path, const std::string & target) override;
+};
+
+}

src/libutil/hash.hh

@@ -145,13 +145,17 @@ std::string printHash16or32(const Hash & hash);
 Hash hashString(HashType ht, std::string_view s);
 
 /**
- * Compute the hash of the given file.
+ * Compute the hash of the given file, hashing its contents directly.
+ *
+ * (Metadata, such as the executable permission bit, is ignored.)
  */
 Hash hashFile(HashType ht, const Path & path);
 
 /**
- * Compute the hash of the given path.  The hash is defined as
- * (essentially) hashString(ht, dumpPath(path)).
+ * Compute the hash of the given path, serializing as a Nix Archive and
+ * then hashing that.
+ *
+ * The hash is defined as (essentially) hashString(ht, dumpPath(path)).
  */
 typedef std::pair<Hash, uint64_t> HashResult;
 HashResult hashPath(HashType ht, const Path & path,

src/libutil/logging.cc

@@ -220,8 +220,8 @@ struct JSONLogger : Logger {
         json["level"] = lvl;
         json["type"] = type;
         json["text"] = s;
+        json["parent"] = parent;
         addFields(json, fields);
-        // FIXME: handle parent
         write(json);
     }
 

src/libutil/tests/url.cc

@@ -335,4 +335,13 @@ namespace nix {
         ASSERT_EQ(d, s);
     }
 
+    TEST(percentEncode, yen) {
+        // https://en.wikipedia.org/wiki/Percent-encoding#Character_data
+        std::string s = reinterpret_cast<const char*>(u8"円");
+        std::string e = "%E5%86%86";
+
+        ASSERT_EQ(percentEncode(s), e);
+        ASSERT_EQ(percentDecode(e), s);
+    }
+
 }

src/libutil/url.cc

@@ -103,7 +103,7 @@ std::string percentEncode(std::string_view s, std::string_view keep)
             || keep.find(c) != std::string::npos)
             res += c;
         else
-            res += fmt("%%%02X", (unsigned int) c);
+            res += fmt("%%%02X", c & 0xFF);
     return res;
 }
 

src/libutil/util.cc

@@ -48,6 +48,23 @@ extern char * * environ __attribute__((weak));
 namespace nix {
 
 void initLibUtil() {
+    // Check that exception handling works. Exception handling has been observed
+    // not to work on darwin when the linker flags aren't quite right.
+    // In this case we don't want to expose the user to some unrelated uncaught
+    // exception, but rather tell them exactly that exception handling is
+    // broken.
+    // When exception handling fails, the message tends to be printed by the
+    // C++ runtime, followed by an abort.
+    // For example on macOS we might see an error such as
+    // libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded.
+    bool caught = false;
+    try {
+        throwExceptionSelfCheck();
+    } catch (const nix::Error & _e) {
+        caught = true;
+    }
+    // This is not actually the main point of this check, but let's make sure anyway:
+    assert(caught);
 }
 
 std::optional<std::string> getEnv(const std::string & key)
@@ -1498,7 +1515,7 @@ bool shouldANSI()
 {
     return isatty(STDERR_FILENO)
         && getEnv("TERM").value_or("dumb") != "dumb"
-        && !getEnv("NO_COLOR").has_value();
+        && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value());
 }
 
 std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width)

src/nix-build/nix-build.cc

@@ -406,8 +406,22 @@ static void main_nix_build(int argc, char * * argv)
             }
         }
 
+        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumDerivedPath;
+
+        accumDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            if (!inputNode.value.empty())
+                pathsToBuild.push_back(DerivedPath::Built {
+                    .drvPath = inputDrv,
+                    .outputs = OutputsSpec::Names { inputNode.value },
+                });
+            for (const auto & [outputName, childNode] : inputNode.childMap)
+                accumDerivedPath(
+                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                    childNode);
+        };
+
         // Build or fetch all dependencies of the derivation.
-        for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) {
+        for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) {
             // To get around lambda capturing restrictions in the
             // standard.
             const auto & inputDrv = inputDrv0;
@@ -416,10 +430,7 @@ static void main_nix_build(int argc, char * * argv)
                         return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude));
                     }))
             {
-                pathsToBuild.push_back(DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(inputDrv),
-                    .outputs = OutputsSpec::Names { inputOutputs },
-                });
+                accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode);
                 pathsToCopy.insert(inputDrv);
             }
         }
@@ -482,13 +493,21 @@ static void main_nix_build(int argc, char * * argv)
 
         if (env.count("__json")) {
             StorePathSet inputs;
-            for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) {
-                auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath);
-                for (auto & i : wantedDepOutputs) {
+
+            std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
+
+            accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+                auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv);
+                for (auto & i : inputNode.value) {
                     auto o = outputs.at(i);
                     store->computeFSClosure(*o, inputs);
                 }
-            }
+                for (const auto & [outputName, childNode] : inputNode.childMap)
+                    accumInputClosure(*outputs.at(outputName), childNode);
+            };
+
+            for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
+                accumInputClosure(inputDrv, inputNode);
 
             ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv);
 

src/nix-env/nix-env.cc

@@ -1228,7 +1228,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                         else {
                             if (v->type() == nString) {
                                 attrs2["type"] = "string";
-                                attrs2["value"] = v->string.s;
+                                attrs2["value"] = v->c_str();
                                 xml.writeEmptyElement("meta", attrs2);
                             } else if (v->type() == nInt) {
                                 attrs2["type"] = "int";
@@ -1248,7 +1248,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                                 for (auto elem : v->listItems()) {
                                     if (elem->type() != nString) continue;
                                     XMLAttrs attrs3;
-                                    attrs3["value"] = elem->string.s;
+                                    attrs3["value"] = elem->c_str();
                                     xml.writeEmptyElement("string", attrs3);
                                 }
                             } else if (v->type() == nAttrs) {
@@ -1260,7 +1260,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                                     if(a.value->type() != nString) continue;
                                     XMLAttrs attrs3;
                                     attrs3["type"] = globals.state->symbols[i.name];
-                                    attrs3["value"] = a.value->string.s;
+                                    attrs3["value"] = a.value->c_str();
                                     xml.writeEmptyElement("string", attrs3);
                             }
                             }

src/nix-store/nix-store.cc

@@ -283,7 +283,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
 {
     enum QueryType
         { qOutputs, qRequisites, qReferences, qReferrers
-        , qReferrersClosure, qDeriver, qBinding, qHash, qSize
+        , qReferrersClosure, qDeriver, qValidDerivers, qBinding, qHash, qSize
         , qTree, qGraph, qGraphML, qResolve, qRoots };
     std::optional<QueryType> query;
     bool useOutput = false;
@@ -299,6 +299,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (i == "--referrers" || i == "--referers") query = qReferrers;
         else if (i == "--referrers-closure" || i == "--referers-closure") query = qReferrersClosure;
         else if (i == "--deriver" || i == "-d") query = qDeriver;
+        else if (i == "--valid-derivers") query = qValidDerivers;
         else if (i == "--binding" || i == "-b") {
             if (opArgs.size() == 0)
                 throw UsageError("expected binding name");
@@ -372,6 +373,21 @@ static void opQuery(Strings opFlags, Strings opArgs)
             }
             break;
 
+        case qValidDerivers: {
+            StorePathSet result;
+            for (auto & i : opArgs) {
+                auto derivers = store->queryValidDerivers(store->followLinksToStorePath(i));
+                for (const auto &i: derivers) {
+                    result.insert(i);
+                }
+            }
+            auto sorted = store->topoSortPaths(result);
+            for (StorePaths::reverse_iterator i = sorted.rbegin();
+                 i != sorted.rend(); ++i)
+                cout << fmt("%s\n", store->printStorePath(*i));
+            break;
+        }
+
         case qBinding:
             for (auto & i : opArgs) {
                 auto path = useDeriver(store->followLinksToStorePath(i));

src/nix/app.cc

@@ -20,15 +20,22 @@ StringPairs resolveRewrites(
     const std::vector<BuiltPathWithResult> & dependencies)
 {
     StringPairs res;
-    for (auto & dep : dependencies)
-        if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path))
-            if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
-                for (auto & [ outputName, outputPath ] : drvDep->outputs)
+    if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+        for (auto & dep : dependencies) {
+            if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path)) {
+                for (auto & [ outputName, outputPath ] : drvDep->outputs) {
                     res.emplace(
-                        DownstreamPlaceholder::unknownCaOutput(
-                            drvDep->drvPath->outPath(), outputName).render(),
+                        DownstreamPlaceholder::fromSingleDerivedPathBuilt(
+                            SingleDerivedPath::Built {
+                                .drvPath = make_ref<SingleDerivedPath>(drvDep->drvPath->discardOutputPath()),
+                                .output = outputName,
+                            }).render(),
                         store.printStorePath(outputPath)
                     );
+                }
+            }
+        }
+    }
     return res;
 }
 

src/nix/daemon.cc

@@ -500,6 +500,45 @@ static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon);
 
 struct CmdDaemon : StoreCommand
 {
+    bool stdio = false;
+    std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
+
+    CmdDaemon()
+    {
+        addFlag({
+            .longName = "stdio",
+            .description = "Attach to standard I/O, instead of trying to bind to a UNIX socket.",
+            .handler = {&stdio, true},
+        });
+
+        addFlag({
+            .longName = "force-trusted",
+            .description = "Force the daemon to trust connecting clients.",
+            .handler = {[&]() {
+                isTrustedOpt = Trusted;
+            }},
+            .experimentalFeature = Xp::DaemonTrustOverride,
+        });
+
+        addFlag({
+            .longName = "force-untrusted",
+            .description = "Force the daemon to not trust connecting clients. The connection will be processed by the receiving daemon before forwarding commands.",
+            .handler = {[&]() {
+                isTrustedOpt = NotTrusted;
+            }},
+            .experimentalFeature = Xp::DaemonTrustOverride,
+        });
+
+        addFlag({
+            .longName = "default-trust",
+            .description = "Use Nix's default trust.",
+            .handler = {[&]() {
+                isTrustedOpt = std::nullopt;
+            }},
+            .experimentalFeature = Xp::DaemonTrustOverride,
+        });
+    }
+
     std::string description() override
     {
         return "daemon to perform store operations on behalf of non-root clients";
@@ -516,7 +555,7 @@ struct CmdDaemon : StoreCommand
 
     void run(ref<Store> store) override
     {
-        runDaemon(false, std::nullopt);
+        runDaemon(stdio, isTrustedOpt);
     }
 };
 

src/nix/daemon.md

@@ -1,20 +1,44 @@
 R""(
 
-# Example
+# Examples
 
-* Run the daemon in the foreground:
+* Run the daemon:
 
   ```console
   # nix daemon
   ```
 
+* Run the daemon and listen on standard I/O instead of binding to a UNIX socket:
+
+  ```console
+  # nix daemon --stdio
+  ```
+
+* Run the daemon and force all connections to be trusted:
+
+  ```console
+  # nix daemon --force-trusted
+  ```
+
+* Run the daemon and force all connections to be untrusted:
+
+  ```console
+  # nix daemon --force-untrusted
+  ```
+
+* Run the daemon, listen on standard I/O, and force all connections to use Nix's default trust:
+
+  ```console
+  # nix daemon --stdio --default-trust
+  ```
+
 # Description
 
 This command runs the Nix daemon, which is a required component in
 multi-user Nix installations. It runs build tasks and other
 operations on the Nix store on behalf of non-root users. Usually you
 don't run the daemon directly; instead it's managed by a service
-management framework such as `systemd`.
+management framework such as `systemd` on Linux, or `launchctl` on Darwin.
 
 Note that this daemon does not fork into the background.
 

src/nix/develop.md

@@ -69,7 +69,7 @@ R""(
 * Run a series of script commands:
 
   ```console
-  # nix develop --command bash --command "mkdir build && cmake .. && make"
+  # nix develop --command bash -c "mkdir build && cmake .. && make"
   ```
 
 # Description

src/nix/eval.cc

@@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
                 state->forceValue(v, pos);
                 if (v.type() == nString)
                     // FIXME: disallow strings with contexts?
-                    writeFile(path, v.string.s);
+                    writeFile(path, v.string_view());
                 else if (v.type() == nAttrs) {
                     if (mkdir(path.c_str(), 0777) == -1)
                         throw SysError("creating directory '%s'", path);

src/nix/flake.md

@@ -67,6 +67,11 @@ inputs.nixpkgs = {
 };
 ```
 
+Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1),
+characters outside of the allowed range (i.e. neither [reserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2)
+nor [unreserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3))
+must be percent-encoded.
+
 ### Examples
 
 Here are some examples of flake references in their URL-like representation:
@@ -103,10 +108,14 @@ The semantic of such a path is as follows:
     2. The filesystem root (/), or
     3. A folder on a different mount point.
 
+Contrary to URL-like references, path-like flake references can contain arbitrary unicode characters (except `#` and `?`).
+
 ### Examples
 
 * `.`: The flake to which the current directory belongs to.
 * `/home/alice/src/patchelf`: A flake in some other directory.
+* `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that
+  has Unicode characters in its name.
 
 ## Flake reference attributes
 

src/nix/repl.cc

@@ -47,7 +47,7 @@ struct CmdRepl : RawInstallablesCommand
 
     void applyDefaultInstallables(std::vector<std::string> & rawInstallables) override
     {
-        if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) {
+        if (!experimentalFeatureSettings.isEnabled(Xp::Flakes) && !(file) && rawInstallables.size() >= 1) {
             warn("future versions of Nix will require using `--file` to load a file");
             if (rawInstallables.size() > 1)
                 warn("more than one input file is not currently supported");

src/nix/repl.md

@@ -36,16 +36,13 @@ R""(
   Loading Installable ''...
   Added 1 variables.
 
-  # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs
+  # nix repl --extra-experimental-features 'flakes' nixpkgs
   Loading Installable 'flake:nixpkgs#'...
   Added 5 variables.
 
   nix-repl> legacyPackages.x86_64-linux.emacs.name
   "emacs-27.1"
 
-  nix-repl> legacyPackages.x86_64-linux.emacs.name
-  "emacs-27.1"
-
   nix-repl> :q
 
   # nix repl --expr 'import <nixpkgs>{}'

src/nix/shell.md

@@ -26,7 +26,7 @@ R""(
 * Run multiple commands in a shell environment:
 
   ```console
-  # nix shell nixpkgs#gnumake --command sh --command "cd src && make"
+  # nix shell nixpkgs#gnumake --command sh -c "cd src && make"
   ```
 
 * Run GNU Hello in a chroot store: