Merge branch 'master' into angerman/mac-fix-recursive-nix

src/build-remote/build-remote.cc

@@ -258,6 +258,8 @@ static int main_build_remote(int argc, char * * argv)
 connected:
         close(5);
 
+        assert(sshStore);
+
         std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
@@ -286,23 +288,48 @@ connected:
         uploadLock = -1;
 
         auto drv = store->readDerivation(*drvPath);
+
+        std::optional<BuildResult> optResult;
+
+        // If we don't know whether we are trusted (e.g. `ssh://`
+        // stores), we assume we are. This is necessary for backwards
+        // compat.
+        bool trustedOrLegacy = ({
+            std::optional trusted = sshStore->isTrustedClient();
+            !trusted || *trusted;
+        });
+
+        // See the very large comment in `case wopBuildDerivation:` in
+        // `src/libstore/daemon.cc` that explains the trust model here.
+        //
+        // This condition mirrors that: that code enforces the "rules" outlined there;
+        // we do the best we can given those "rules".
+        if (trustedOrLegacy || drv.type().isCA())  {
+            // Hijack the inputs paths of the derivation to include all
+            // the paths that come from the `inputDrvs` set. We don’t do
+            // that for the derivations whose `inputDrvs` is empty
+            // because:
+            //
+            // 1. It’s not needed
+            //
+            // 2. Changing the `inputSrcs` set changes the associated
+            //    output ids, which break CA derivations
+            if (!drv.inputDrvs.empty())
+                drv.inputSrcs = store->parseStorePathSet(inputs);
+            optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
+            auto & result = *optResult;
+            if (!result.success())
+                throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
+        } else {
+            copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
+            auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } });
+            // One path to build should produce exactly one build result
+            assert(res.size() == 1);
+            optResult = std::move(res[0]);
+        }
+
+
         auto outputHashes = staticOutputHashes(*store, drv);
-
-        // Hijack the inputs paths of the derivation to include all the paths
-        // that come from the `inputDrvs` set.
-        // We don’t do that for the derivations whose `inputDrvs` is empty
-        // because
-        // 1. It’s not needed
-        // 2. Changing the `inputSrcs` set changes the associated output ids,
-        //  which break CA derivations
-        if (!drv.inputDrvs.empty())
-            drv.inputSrcs = store->parseStorePathSet(inputs);
-
-        auto result = sshStore->buildDerivation(*drvPath, drv);
-
-        if (!result.success())
-            throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
-
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
         if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
@@ -311,6 +338,8 @@ connected:
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
                 if (!store->queryRealisation(thisOutputId)) {
                     debug("missing output %s", outputName);
+                    assert(optResult);
+                    auto & result = *optResult;
                     auto i = result.builtOutputs.find(outputName);
                     assert(i != result.builtOutputs.end());
                     auto & newRealisation = i->second;

src/libcmd/command.cc

@@ -121,6 +121,8 @@ ref<EvalState> EvalCommand::getEvalState()
             #endif
             ;
 
+        evalState->repair = repair;
+
         if (startReplOnEvalErrors) {
             evalState->debugRepl = &AbstractNixRepl::runSimple;
         };

src/libcmd/common-eval-args.cc

@@ -153,7 +153,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
     for (auto & i : autoArgs) {
         auto v = state.allocValue();
         if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), absPath(".")));
+            state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(CanonPath::fromCwd())));
         else
             v->mkString(((std::string_view) i.second).substr(1));
         res.insert(state.symbols.create(i.first), v);
@@ -161,19 +161,19 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
     return res.finish();
 }
 
-Path lookupFileArg(EvalState & state, std::string_view s)
+SourcePath lookupFileArg(EvalState & state, std::string_view s)
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto storePath = fetchers::downloadTarball(
             state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath;
-        return state.store->toRealPath(storePath);
+        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
         auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath;
-        return state.store->toRealPath(storePath);
+        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 
     else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
@@ -182,7 +182,7 @@ Path lookupFileArg(EvalState & state, std::string_view s)
     }
 
     else
-        return absPath(std::string(s));
+        return state.rootPath(CanonPath::fromCwd(s));
 }
 
 }

src/libcmd/common-eval-args.hh

@@ -2,14 +2,16 @@
 ///@file
 
 #include "args.hh"
+#include "common-args.hh"
 
 namespace nix {
 
 class Store;
 class EvalState;
 class Bindings;
+struct SourcePath;
 
-struct MixEvalArgs : virtual Args
+struct MixEvalArgs : virtual Args, virtual MixRepair
 {
     static constexpr auto category = "Common evaluation options";
 
@@ -25,6 +27,6 @@ private:
     std::map<std::string, std::string> autoArgs;
 };
 
-Path lookupFileArg(EvalState & state, std::string_view s);
+SourcePath lookupFileArg(EvalState & state, std::string_view s);
 
 }

src/libcmd/editor-for.cc

@@ -3,8 +3,11 @@
 
 namespace nix {
 
-Strings editorFor(const Path & file, uint32_t line)
+Strings editorFor(const SourcePath & file, uint32_t line)
 {
+    auto path = file.getPhysicalPath();
+    if (!path)
+        throw Error("cannot open '%s' in an editor because it has no physical path", file);
     auto editor = getEnv("EDITOR").value_or("cat");
     auto args = tokenizeString<Strings>(editor);
     if (line > 0 && (
@@ -13,7 +16,7 @@ Strings editorFor(const Path & file, uint32_t line)
         editor.find("vim") != std::string::npos ||
         editor.find("kak") != std::string::npos))
         args.push_back(fmt("+%d", line));
-    args.push_back(file);
+    args.push_back(path->abs());
     return args;
 }
 

src/libcmd/editor-for.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "types.hh"
+#include "input-accessor.hh"
 
 namespace nix {
 
@@ -9,6 +10,6 @@ namespace nix {
  * Helper function to generate args that invoke $EDITOR on
  * filename:lineno.
  */
-Strings editorFor(const Path & file, uint32_t line);
+Strings editorFor(const SourcePath & file, uint32_t line);
 
 }

src/libcmd/installable-attr-path.cc

@@ -46,7 +46,15 @@ std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state)
 
 DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
 {
-    auto v = toValue(*state).first;
+    auto [v, pos] = toValue(*state);
+
+    if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
+        *v,
+        pos,
+        fmt("while evaluating the attribute '%s'", attrPath)))
+    {
+        return { *derivedPathWithInfo };
+    }
 
     Bindings & autoArgs = *cmd.getAutoArgs(*state);
 

src/libcmd/installable-flake.cc

@@ -95,32 +95,13 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
         // FIXME: use eval cache?
         auto v = attr->forceValue();
 
-        if (v.type() == nPath) {
-            PathSet context;
-            auto storePath = state->copyPathToStore(context, Path(v.path));
-            return {{
-                .path = DerivedPath::Opaque {
-                    .path = std::move(storePath),
-                },
-                .info = make_ref<ExtraPathInfo>(),
-            }};
+        if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
+            v,
+            noPos,
+            fmt("while evaluating the flake output attribute '%s'", attrPath)))
+        {
+            return { *derivedPathWithInfo };
         }
-
-        else if (v.type() == nString) {
-            PathSet context;
-            auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath));
-            auto storePath = state->store->maybeParseStorePath(s);
-            if (storePath && context.count(std::string(s))) {
-                return {{
-                    .path = DerivedPath::Opaque {
-                        .path = std::move(*storePath),
-                    },
-                    .info = make_ref<ExtraPathInfo>(),
-                }};
-            } else
-                throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s);
-        }
-
         else
             throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
     }
@@ -235,7 +216,7 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
         }
     }
 
-    return InstallableValue::nixpkgsFlakeRef();
+    return defaultNixpkgsFlakeRef();
 }
 
 }

src/libcmd/installable-flake.hh

@@ -67,9 +67,22 @@ struct InstallableFlake : InstallableValue
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
-    FlakeRef nixpkgsFlakeRef() const override;
+    FlakeRef nixpkgsFlakeRef() const;
 };
 
+/**
+ * Default flake ref for referring to Nixpkgs. For flakes that don't
+ * have their own Nixpkgs input, or other installables.
+ *
+ * It is a layer violation for Nix to know about Nixpkgs; currently just
+ * `nix develop` does. Be wary of using this /
+ * `InstallableFlake::nixpkgsFlakeRef` more places.
+ */
+static inline FlakeRef defaultNixpkgsFlakeRef()
+{
+    return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
+}
+
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake);

src/libcmd/installable-value.cc

@@ -41,4 +41,26 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
     return ref { castedInstallable };
 }
 
+std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
+{
+    if (v.type() == nPath) {
+        auto storePath = v.path().fetchToStore(state->store);
+        return {{
+            .path = DerivedPath::Opaque {
+                .path = std::move(storePath),
+            },
+            .info = make_ref<ExtraPathInfo>(),
+        }};
+    }
+
+    else if (v.type() == nString) {
+        return {{
+            .path = state->coerceToDerivedPath(pos, v, errorCtx),
+            .info = make_ref<ExtraPathInfo>(),
+        }};
+    }
+
+    else return std::nullopt;
+}
+
 }

src/libcmd/installable-value.hh

@@ -96,13 +96,26 @@ struct InstallableValue : Installable
 
     UnresolvedApp toApp(EvalState & state);
 
-    virtual FlakeRef nixpkgsFlakeRef() const
-    {
-        return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
-    }
-
     static InstallableValue & require(Installable & installable);
     static ref<InstallableValue> require(ref<Installable> installable);
+
+protected:
+
+    /**
+     * Handles either a plain path, or a string with a single string
+     * context elem in the right format. The latter case is handled by
+     * `EvalState::coerceToDerivedPath()`; see it for details.
+     *
+     * @param v Value that is hopefully a string or path per the above.
+     *
+     * @param pos Position of value to aid with diagnostics.
+     *
+     * @param errorCtx Arbitrary message for use in potential error message when something is wrong with `v`.
+     *
+     * @result A derived path (with empty info, for now) if the value
+     * matched the above criteria.
+     */
+    std::optional<DerivedPathWithInfo> trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx);
 };
 
 }

src/libcmd/installables.cc

@@ -449,7 +449,7 @@ Installables SourceExprCommand::parseInstallables(
         else if (file)
             state->evalFile(lookupFileArg(*state, *file), *vFile);
         else {
-            auto e = state->parseExprFromString(*expr, absPath("."));
+            auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
             state->eval(e, *vFile);
         }
 

src/libcmd/repl.cc

@@ -55,8 +55,6 @@ struct NixRepl
     , gc
     #endif
 {
-    std::string curDir;
-
     size_t debugTraceIndex;
 
     Strings loadedFiles;
@@ -114,7 +112,6 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalStat
     , staticEnv(new StaticEnv(false, state->staticBaseEnv.get()))
     , historyFile(getDataDir() + "/nix/repl-history")
 {
-    curDir = absPath(".");
 }
 
 
@@ -594,14 +591,14 @@ bool NixRepl::processLine(std::string line)
         Value v;
         evalString(arg, v);
 
-        const auto [path, line] = [&] () -> std::pair<Path, uint32_t> {
+        const auto [path, line] = [&] () -> std::pair<SourcePath, uint32_t> {
             if (v.type() == nPath || v.type() == nString) {
-                PathSet context;
+                NixStringContext context;
                 auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit");
                 return {path, 0};
             } else if (v.isLambda()) {
                 auto pos = state->positions[v.lambda.fun->pos];
-                if (auto path = std::get_if<Path>(&pos.origin))
+                if (auto path = std::get_if<SourcePath>(&pos.origin))
                     return {*path, pos.line};
                 else
                     throw Error("'%s' cannot be shown in an editor", pos);
@@ -876,8 +873,7 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
 
 Expr * NixRepl::parseString(std::string s)
 {
-    Expr * e = state->parseExprFromString(std::move(s), curDir, staticEnv);
-    return e;
+    return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv);
 }
 
 
@@ -925,7 +921,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     case nPath:
-        str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping?
+        str << ANSI_GREEN << v.path().to_string() << ANSI_NORMAL; // !!! escaping?
         break;
 
     case nNull:
@@ -940,7 +936,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         if (isDrv) {
             str << "«derivation ";
             Bindings::iterator i = v.attrs->find(state->sDrvPath);
-            PathSet context;
+            NixStringContext context;
             if (i != v.attrs->end())
                 str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
             else

src/libexpr/attr-path.cc

@@ -106,7 +106,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
 }
 
 
-std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what)
+std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what)
 {
     Value * v2;
     try {
@@ -118,21 +118,25 @@ std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value &
 
     // FIXME: is it possible to extract the Pos object instead of doing this
     //        toString + parsing?
-    auto pos = state.forceString(*v2, noPos, "while evaluating the 'meta.position' attribute of a derivation");
+    NixStringContext context;
+    auto path = state.coerceToPath(noPos, *v2, context, "while evaluating the 'meta.position' attribute of a derivation");
 
-    auto colon = pos.rfind(':');
-    if (colon == std::string::npos)
-        throw ParseError("cannot parse meta.position attribute '%s'", pos);
+    auto fn = path.path.abs();
+
+    auto fail = [fn]() {
+        throw ParseError("cannot parse 'meta.position' attribute '%s'", fn);
+    };
 
-    std::string filename(pos, 0, colon);
-    unsigned int lineno;
     try {
-        lineno = std::stoi(std::string(pos, colon + 1, std::string::npos));
+        auto colon = fn.rfind(':');
+        if (colon == std::string::npos) fail();
+        std::string filename(fn, 0, colon);
+        auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
+        return {CanonPath(fn.substr(0, colon)), lineno};
     } catch (std::invalid_argument & e) {
-        throw ParseError("cannot parse line number '%s'", pos);
+        fail();
+        abort();
     }
-
-    return { std::move(filename), lineno };
 }
 
 

src/libexpr/attr-path.hh

@@ -20,7 +20,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(
 /**
  * Heuristic to find the filename and lineno or a nix value.
  */
-std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what);
+std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what);
 
 std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
 

src/libexpr/eval-cache.cc

@@ -47,7 +47,7 @@ struct AttrDb
     {
         auto state(_state->lock());
 
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v4";
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v5";
         createDirs(cacheDir);
 
         Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@@ -300,7 +300,7 @@ struct AttrDb
                 NixStringContext context;
                 if (!queryAttribute.isNull(3))
                     for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(NixStringContextElem::parse(cfg, s));
+                        context.insert(NixStringContextElem::parse(s));
                 return {{rowId, string_t{queryAttribute.getStr(2), context}}};
             }
             case AttrType::Bool:
@@ -442,8 +442,10 @@ Value & AttrCursor::forceValue()
         if (v.type() == nString)
             cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
                            string_t{v.string.s, {}}};
-        else if (v.type() == nPath)
-            cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
+        else if (v.type() == nPath) {
+            auto path = v.path().path;
+            cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}};
+        }
         else if (v.type() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
         else if (v.type() == nInt)
@@ -580,7 +582,7 @@ std::string AttrCursor::getString()
     if (v.type() != nString && v.type() != nPath)
         root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>();
 
-    return v.type() == nString ? v.string.s : v.path;
+    return v.type() == nString ? v.string.s : v.path().to_string();
 }
 
 string_t AttrCursor::getStringWithContext()
@@ -619,10 +621,13 @@ string_t AttrCursor::getStringWithContext()
 
     auto & v = forceValue();
 
-    if (v.type() == nString)
-        return {v.string.s, v.getContext(*root->state.store)};
+    if (v.type() == nString) {
+        NixStringContext context;
+        copyContext(v, context);
+        return {v.string.s, std::move(context)};
+    }
     else if (v.type() == nPath)
-        return {v.path, {}};
+        return {v.path().to_string(), {}};
     else
         root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>();
 }

src/libexpr/eval.cc

@@ -4,6 +4,7 @@
 #include "util.hh"
 #include "store-api.hh"
 #include "derivations.hh"
+#include "downstream-placeholder.hh"
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "filetransfer.hh"
@@ -94,7 +95,6 @@ RootValue allocRootValue(Value * v)
 #endif
 }
 
-
 void Value::print(const SymbolTable & symbols, std::ostream & str,
     std::set<const void *> * seen) const
 {
@@ -111,7 +111,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
         printLiteralString(str, string.s);
         break;
     case tPath:
-        str << path; // !!! escaping?
+        str << path().to_string(); // !!! escaping?
         break;
     case tNull:
         str << "null";
@@ -535,6 +535,7 @@ EvalState::EvalState(
     , sOutputSpecified(symbols.create("outputSpecified"))
     , repair(NoRepair)
     , emptyBindings(0)
+    , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix")))
     , store(store)
     , buildStore(buildStore ? buildStore : store)
     , debugRepl(nullptr)
@@ -609,15 +610,14 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
 {
     allowPath(storePath);
 
-    auto path = store->printStorePath(storePath);
-    v.mkString(path, PathSet({path}));
+    mkStorePathString(storePath, v);
 }
 
-Path EvalState::checkSourcePath(const Path & path_)
+SourcePath EvalState::checkSourcePath(const SourcePath & path_)
 {
     if (!allowedPaths) return path_;
 
-    auto i = resolvedPaths.find(path_);
+    auto i = resolvedPaths.find(path_.path.abs());
     if (i != resolvedPaths.end())
         return i->second;
 
@@ -627,9 +627,9 @@ Path EvalState::checkSourcePath(const Path & path_)
      * attacker can't append ../../... to a path that would be in allowedPaths
      * and thus leak symlink targets.
      */
-    Path abspath = canonPath(path_);
+    Path abspath = canonPath(path_.path.abs());
 
-    if (hasPrefix(abspath, corepkgsPrefix)) return abspath;
+    if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath);
 
     for (auto & i : *allowedPaths) {
         if (isDirOrInDir(abspath, i)) {
@@ -647,11 +647,11 @@ Path EvalState::checkSourcePath(const Path & path_)
 
     /* Resolve symlinks. */
     debug("checking access to '%s'", abspath);
-    Path path = canonPath(abspath, true);
+    SourcePath path = CanonPath(canonPath(abspath, true));
 
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path, i)) {
-            resolvedPaths[path_] = path;
+        if (isDirOrInDir(path.path.abs(), i)) {
+            resolvedPaths.insert_or_assign(path_.path.abs(), path);
             return path;
         }
     }
@@ -679,12 +679,12 @@ void EvalState::checkURI(const std::string & uri)
     /* If the URI is a path, then check it against allowedPaths as
        well. */
     if (hasPrefix(uri, "/")) {
-        checkSourcePath(uri);
+        checkSourcePath(CanonPath(uri));
         return;
     }
 
     if (hasPrefix(uri, "file://")) {
-        checkSourcePath(std::string(uri, 7));
+        checkSourcePath(CanonPath(std::string(uri, 7)));
         return;
     }
 
@@ -692,7 +692,7 @@ void EvalState::checkURI(const std::string & uri)
 }
 
 
-Path EvalState::toRealPath(const Path & path, const PathSet & context)
+Path EvalState::toRealPath(const Path & path, const NixStringContext & context)
 {
     // FIXME: check whether 'path' is in 'context'.
     return
@@ -944,34 +944,34 @@ void Value::mkString(std::string_view s)
 }
 
 
-static void copyContextToValue(Value & v, const PathSet & context)
+static void copyContextToValue(Value & v, const NixStringContext & context)
 {
     if (!context.empty()) {
         size_t n = 0;
         v.string.context = (const char * *)
             allocBytes((context.size() + 1) * sizeof(char *));
         for (auto & i : context)
-            v.string.context[n++] = dupString(i.c_str());
+            v.string.context[n++] = dupString(i.to_string().c_str());
         v.string.context[n] = 0;
     }
 }
 
-void Value::mkString(std::string_view s, const PathSet & context)
+void Value::mkString(std::string_view s, const NixStringContext & context)
 {
     mkString(s);
     copyContextToValue(*this, context);
 }
 
-void Value::mkStringMove(const char * s, const PathSet & context)
+void Value::mkStringMove(const char * s, const NixStringContext & context)
 {
     mkString(s);
     copyContextToValue(*this, context);
 }
 
 
-void Value::mkPath(std::string_view s)
+void Value::mkPath(const SourcePath & path)
 {
-    mkPath(makeImmutableString(s));
+    mkPath(makeImmutableString(path.path.abs()));
 }
 
 
@@ -1027,9 +1027,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 void EvalState::mkPos(Value & v, PosIdx p)
 {
     auto pos = positions[p];
-    if (auto path = std::get_if<Path>(&pos.origin)) {
+    if (auto path = std::get_if<SourcePath>(&pos.origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(sFile).mkString(*path);
+        attrs.alloc(sFile).mkString(path->path.abs());
         attrs.alloc(sLine).mkInt(pos.line);
         attrs.alloc(sColumn).mkInt(pos.column);
         v.mkAttrs(attrs);
@@ -1038,6 +1038,37 @@ void EvalState::mkPos(Value & v, PosIdx p)
 }
 
 
+void EvalState::mkStorePathString(const StorePath & p, Value & v)
+{
+    v.mkString(
+        store->printStorePath(p),
+        NixStringContext {
+            NixStringContextElem::Opaque { .path = p },
+        });
+}
+
+
+void EvalState::mkOutputString(
+    Value & value,
+    const StorePath & drvPath,
+    const std::string outputName,
+    std::optional<StorePath> optOutputPath)
+{
+    value.mkString(
+        optOutputPath
+            ? store->printStorePath(*std::move(optOutputPath))
+            /* Downstream we would substitute this for an actual path once
+               we build the floating CA derivation */
+            : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(),
+        NixStringContext {
+            NixStringContextElem::Built {
+                .drvPath = drvPath,
+                .output = outputName,
+            }
+        });
+}
+
+
 /* Create a thunk for the delayed computation of the given expression
    in the given environment.  But if the expression is a variable,
    then look it up right away.  This significantly reduces the number
@@ -1085,7 +1116,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 }
 
 
-void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
+void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial)
 {
     auto path = checkSourcePath(path_);
 
@@ -1095,7 +1126,7 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
         return;
     }
 
-    Path resolvedPath = resolveExprPath(path);
+    auto resolvedPath = resolveExprPath(path);
     if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) {
         v = i->second;
         return;
@@ -1123,8 +1154,8 @@ void EvalState::resetFileCache()
 
 
 void EvalState::cacheFile(
-    const Path & path,
-    const Path & resolvedPath,
+    const SourcePath & path,
+    const SourcePath & resolvedPath,
     Expr * e,
     Value & v,
     bool mustBeTrivial)
@@ -1138,7 +1169,7 @@ void EvalState::cacheFile(
                 *e,
                 this->baseEnv,
                 e->getPos() ? static_cast<std::shared_ptr<AbstractPos>>(positions[e->getPos()]) : nullptr,
-                "while evaluating the file '%1%':", resolvedPath)
+                "while evaluating the file '%1%':", resolvedPath.to_string())
             : nullptr;
 
         // Enforce that 'flake.nix' is a direct attrset, not a
@@ -1148,7 +1179,7 @@ void EvalState::cacheFile(
             error("file '%s' must be an attribute set", path).debugThrow<EvalError>();
         eval(e, v);
     } catch (Error & e) {
-        addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath);
+        addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string());
         throw;
     }
 
@@ -1409,8 +1440,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
     } catch (Error & e) {
         if (pos2) {
             auto pos2r = state.positions[pos2];
-            auto origin = std::get_if<Path>(&pos2r.origin);
-            if (!(origin && *origin == state.derivationNixPath))
+            auto origin = std::get_if<SourcePath>(&pos2r.origin);
+            if (!(origin && *origin == state.derivationInternal))
                 state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'",
                     showAttrPath(state, env, attrPath));
         }
@@ -1900,7 +1931,7 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po
 
 void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     std::vector<BackedStringView> s;
     size_t sSize = 0;
     NixInt n = 0;
@@ -1983,7 +2014,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == nPath) {
         if (!context.empty())
             state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>();
-        v.mkPath(canonPath(str()));
+        v.mkPath(CanonPath(canonPath(str())));
     } else
         v.mkStringMove(c_str(), context);
 }
@@ -2109,26 +2140,15 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
 }
 
 
-void copyContext(const Value & v, PathSet & context)
+void copyContext(const Value & v, NixStringContext & context)
 {
     if (v.string.context)
         for (const char * * p = v.string.context; *p; ++p)
-            context.insert(*p);
+            context.insert(NixStringContextElem::parse(*p));
 }
 
 
-NixStringContext Value::getContext(const Store & store)
-{
-    NixStringContext res;
-    assert(internalType == tString);
-    if (string.context)
-        for (const char * * p = string.context; *p; ++p)
-            res.push_back(NixStringContextElem::parse(store, *p));
-    return res;
-}
-
-
-std::string_view EvalState::forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx)
+std::string_view EvalState::forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx)
 {
     auto s = forceString(v, pos, errorCtx);
     copyContext(v, context);
@@ -2158,7 +2178,7 @@ bool EvalState::isDerivation(Value & v)
 
 
 std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value & v,
-    PathSet & context, bool coerceMore, bool copyToStore)
+    NixStringContext & context, bool coerceMore, bool copyToStore)
 {
     auto i = v.attrs->find(sToString);
     if (i != v.attrs->end()) {
@@ -2172,8 +2192,14 @@ std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value &
     return {};
 }
 
-BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet &context,
-    std::string_view errorCtx, bool coerceMore, bool copyToStore, bool canonicalizePath)
+BackedStringView EvalState::coerceToString(
+    const PosIdx pos,
+    Value & v,
+    NixStringContext & context,
+    std::string_view errorCtx,
+    bool coerceMore,
+    bool copyToStore,
+    bool canonicalizePath)
 {
     forceValue(v, pos);
 
@@ -2183,12 +2209,14 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet &
     }
 
     if (v.type() == nPath) {
-        BackedStringView path(PathView(v.path));
-        if (canonicalizePath)
-            path = canonPath(*path);
-        if (copyToStore)
-            path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned()));
-        return path;
+        return
+            !canonicalizePath && !copyToStore
+            ? // FIXME: hack to preserve path literals that end in a
+              // slash, as in /foo/${x}.
+              v._path
+            : copyToStore
+            ? store->printStorePath(copyPathToStore(context, v.path()))
+            : std::string(v.path().path.abs());
     }
 
     if (v.type() == nAttrs) {
@@ -2249,40 +2277,40 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet &
 }
 
 
-StorePath EvalState::copyPathToStore(PathSet & context, const Path & path)
+StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path)
 {
-    if (nix::isDerivation(path))
+    if (nix::isDerivation(path.path.abs()))
         error("file names are not allowed to end in '%1%'", drvExtension).debugThrow<EvalError>();
 
-    auto dstPath = [&]() -> StorePath
-    {
-        auto i = srcToStore.find(path);
-        if (i != srcToStore.end()) return i->second;
+    auto i = srcToStore.find(path);
 
-        auto dstPath = settings.readOnlyMode
-            ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
-            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
-        allowPath(dstPath);
-        srcToStore.insert_or_assign(path, dstPath);
-        printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
-        return dstPath;
-    }();
+    auto dstPath = i != srcToStore.end()
+        ? i->second
+        : [&]() {
+            auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair);
+            allowPath(dstPath);
+            srcToStore.insert_or_assign(path, dstPath);
+            printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
+            return dstPath;
+        }();
 
-    context.insert(store->printStorePath(dstPath));
+    context.insert(NixStringContextElem::Opaque {
+        .path = dstPath
+    });
     return dstPath;
 }
 
 
-Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx)
+SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
     auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (path == "" || path[0] != '/')
         error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
-    return path;
+    return CanonPath(path);
 }
 
 
-StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx)
+StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
     auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (auto storePath = store->maybeParseStorePath(path))
@@ -2291,6 +2319,80 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & co
 }
 
 
+std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
+{
+    NixStringContext context;
+    auto s = forceString(v, context, pos, errorCtx);
+    auto csize = context.size();
+    if (csize != 1)
+        error(
+            "string '%s' has %d entries in its context. It should only have exactly one entry",
+            s, csize)
+            .withTrace(pos, errorCtx).debugThrow<EvalError>();
+    auto derivedPath = std::visit(overloaded {
+        [&](NixStringContextElem::Opaque && o) -> DerivedPath {
+            return DerivedPath::Opaque {
+                .path = std::move(o.path),
+            };
+        },
+        [&](NixStringContextElem::DrvDeep &&) -> DerivedPath {
+            error(
+                "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
+                s).withTrace(pos, errorCtx).debugThrow<EvalError>();
+        },
+        [&](NixStringContextElem::Built && b) -> DerivedPath {
+            return DerivedPath::Built {
+                .drvPath = std::move(b.drvPath),
+                .outputs = OutputsSpec::Names { std::move(b.output) },
+            };
+        },
+    }, ((NixStringContextElem &&) *context.begin()).raw());
+    return {
+        std::move(derivedPath),
+        std::move(s),
+    };
+}
+
+
+DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
+{
+    auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx);
+    auto s = s_;
+    std::visit(overloaded {
+        [&](const DerivedPath::Opaque & o) {
+            auto sExpected = store->printStorePath(o.path);
+            if (s != sExpected)
+                error(
+                    "path string '%s' has context with the different path '%s'",
+                    s, sExpected)
+                    .withTrace(pos, errorCtx).debugThrow<EvalError>();
+        },
+        [&](const DerivedPath::Built & b) {
+            // TODO need derived path with single output to make this
+            // total. Will add as part of RFC 92 work and then this is
+            // cleaned up.
+            auto output = *std::get<OutputsSpec::Names>(b.outputs).begin();
+
+            auto drv = store->readDerivation(b.drvPath);
+            auto i = drv.outputs.find(output);
+            if (i == drv.outputs.end())
+                throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output);
+            auto optOutputPath = i->second.path(*store, drv.name, output);
+            // This is testing for the case of CA derivations
+            auto sExpected = optOutputPath
+                ? store->printStorePath(*optOutputPath)
+                : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render();
+            if (s != sExpected)
+                error(
+                    "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
+                    s, output, store->printStorePath(b.drvPath), sExpected)
+                    .withTrace(pos, errorCtx).debugThrow<EvalError>();
+        }
+    }, derivedPath.raw());
+    return derivedPath;
+}
+
+
 bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
 {
     forceValue(v1, noPos);
@@ -2321,7 +2423,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
             return strcmp(v1.string.s, v2.string.s) == 0;
 
         case nPath:
-            return strcmp(v1.path, v2.path) == 0;
+            return strcmp(v1._path, v2._path) == 0;
 
         case nNull:
             return true;
@@ -2448,8 +2550,8 @@ void EvalState::printStats()
                     else
                         obj["name"] = nullptr;
                     if (auto pos = positions[fun->pos]) {
-                        if (auto path = std::get_if<Path>(&pos.origin))
-                            obj["file"] = *path;
+                        if (auto path = std::get_if<SourcePath>(&pos.origin))
+                            obj["file"] = path->to_string();
                         obj["line"] = pos.line;
                         obj["column"] = pos.column;
                     }
@@ -2463,8 +2565,8 @@ void EvalState::printStats()
                 for (auto & i : attrSelects) {
                     json obj = json::object();
                     if (auto pos = positions[i.first]) {
-                        if (auto path = std::get_if<Path>(&pos.origin))
-                            obj["file"] = *path;
+                        if (auto path = std::get_if<SourcePath>(&pos.origin))
+                            obj["file"] = path->to_string();
                         obj["line"] = pos.line;
                         obj["column"] = pos.column;
                     }
@@ -2489,7 +2591,7 @@ void EvalState::printStats()
 }
 
 
-std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
+std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
     throw TypeError({
         .msg = hintfmt("cannot coerce %1% to a string", showType())
@@ -2518,7 +2620,7 @@ Strings EvalSettings::getDefaultNixPath()
 {
     Strings res;
     auto add = [&](const Path & p, const std::string & s = std::string()) {
-        if (pathExists(p)) {
+        if (pathAccessible(p)) {
             if (s.empty()) {
                 res.push_back(p);
             } else {

src/libexpr/eval.hh

@@ -8,6 +8,7 @@
 #include "symbol-table.hh"
 #include "config.hh"
 #include "experimental-features.hh"
+#include "input-accessor.hh"
 
 #include <map>
 #include <optional>
@@ -20,6 +21,7 @@ namespace nix {
 class Store;
 class EvalState;
 class StorePath;
+struct DerivedPath;
 enum RepairFlag : bool;
 
 
@@ -56,20 +58,14 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env &
 
 std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env);
 
-void copyContext(const Value & v, PathSet & context);
-
-
-/**
- * Cache for calls to addToStore(); maps source paths to the store
- * paths.
- */
-typedef std::map<Path, StorePath> SrcToStore;
+void copyContext(const Value & v, NixStringContext & context);
 
 
 std::string printValue(const EvalState & state, const Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
+// FIXME: maybe change this to an std::variant<SourcePath, URL>.
 typedef std::pair<std::string, std::string> SearchPathElem;
 typedef std::list<SearchPathElem> SearchPath;
 
@@ -137,8 +133,6 @@ public:
     SymbolTable symbols;
     PosTable positions;
 
-    static inline std::string derivationNixPath = "//builtin/derivation.nix";
-
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
@@ -149,7 +143,6 @@ public:
         sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
         sPrefix,
         sOutputSpecified;
-    Symbol sDerivationNix;
 
     /**
      * If set, force copying files to the Nix store even if they
@@ -165,6 +158,8 @@ public:
 
     Bindings emptyBindings;
 
+    const SourcePath derivationInternal;
+
     /**
      * Store used to materialise .drv files.
      */
@@ -234,15 +229,18 @@ public:
     }
 
 private:
-    SrcToStore srcToStore;
+
+    /* Cache for calls to addToStore(); maps source paths to the store
+       paths. */
+    std::map<SourcePath, StorePath> srcToStore;
 
     /**
      * A cache from path names to parse trees.
      */
 #if HAVE_BOEHMGC
-    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *>>> FileParseCache;
+    typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
 #else
-    typedef std::map<Path, Expr *> FileParseCache;
+    typedef std::map<SourcePath, Expr *> FileParseCache;
 #endif
     FileParseCache fileParseCache;
 
@@ -250,9 +248,9 @@ private:
      * A cache from path names to values.
      */
 #if HAVE_BOEHMGC
-    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value>>> FileEvalCache;
+    typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
 #else
-    typedef std::map<Path, Value> FileEvalCache;
+    typedef std::map<SourcePath, Value> FileEvalCache;
 #endif
     FileEvalCache fileEvalCache;
 
@@ -263,7 +261,7 @@ private:
     /**
      * Cache used by checkSourcePath().
      */
-    std::unordered_map<Path, Path> resolvedPaths;
+    std::unordered_map<Path, SourcePath> resolvedPaths;
 
     /**
      * Cache used by prim_match().
@@ -294,6 +292,12 @@ public:
 
     SearchPath getSearchPath() { return searchPath; }
 
+    /**
+     * Return a `SourcePath` that refers to `path` in the root
+     * filesystem.
+     */
+    SourcePath rootPath(CanonPath path);
+
     /**
      * Allow access to a path.
      */
@@ -314,7 +318,7 @@ public:
      * Check whether access to a path is allowed and throw an error if
      * not. Otherwise return the canonicalised path.
      */
-    Path checkSourcePath(const Path & path);
+    SourcePath checkSourcePath(const SourcePath & path);
 
     void checkURI(const std::string & uri);
 
@@ -327,19 +331,19 @@ public:
      * intended to distinguish between import-from-derivation and
      * sources stored in the actual /nix/store.
      */
-    Path toRealPath(const Path & path, const PathSet & context);
+    Path toRealPath(const Path & path, const NixStringContext & context);
 
     /**
      * Parse a Nix expression from the specified file.
      */
-    Expr * parseExprFromFile(const Path & path);
-    Expr * parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv> & staticEnv);
+    Expr * parseExprFromFile(const SourcePath & path);
+    Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
 
     /**
      * Parse a Nix expression from the specified string.
      */
-    Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv);
-    Expr * parseExprFromString(std::string s, const Path & basePath);
+    Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
+    Expr * parseExprFromString(std::string s, const SourcePath & basePath);
 
     Expr * parseStdin();
 
@@ -348,14 +352,14 @@ public:
      * form. Optionally enforce that the top-level expression is
      * trivial (i.e. doesn't require arbitrary computation).
      */
-    void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
+    void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
 
     /**
      * Like `evalFile`, but with an already parsed expression.
      */
     void cacheFile(
-        const Path & path,
-        const Path & resolvedPath,
+        const SourcePath & path,
+        const SourcePath & resolvedPath,
         Expr * e,
         Value & v,
         bool mustBeTrivial = false);
@@ -365,8 +369,8 @@ public:
     /**
      * Look up a file in the search path.
      */
-    Path findFile(const std::string_view path);
-    Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
+    SourcePath findFile(const std::string_view path);
+    SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
      * If the specified search path element is a URI, download it.
@@ -423,7 +427,7 @@ public:
      */
     void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx);
     std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx);
-    std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx);
+    std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx);
     std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);
 
     [[gnu::noinline]]
@@ -439,7 +443,7 @@ public:
     bool isDerivation(Value & v);
 
     std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
-        PathSet & context, bool coerceMore = false, bool copyToStore = true);
+        NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
 
     /**
      * String coercion.
@@ -449,12 +453,12 @@ public:
      * booleans and lists to a string.  If `copyToStore` is set,
      * referenced paths are copied to the Nix store as a side effect.
      */
-    BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context,
+    BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context,
         std::string_view errorCtx,
         bool coerceMore = false, bool copyToStore = true,
         bool canonicalizePath = true);
 
-    StorePath copyPathToStore(PathSet & context, const Path & path);
+    StorePath copyPathToStore(NixStringContext & context, const SourcePath & path);
 
     /**
      * Path coercion.
@@ -463,12 +467,34 @@ public:
      * path.  The result is guaranteed to be a canonicalised, absolute
      * path.  Nothing is copied to the store.
      */
-    Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx);
+    SourcePath coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
 
     /**
      * Like coerceToPath, but the result must be a store path.
      */
-    StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx);
+    StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
+
+    /**
+     * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only.
+     */
+    std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
+
+    /**
+     * Coerce to `DerivedPath`.
+     *
+     * Must be a string which is either a literal store path or a
+     * "placeholder (see `DownstreamPlaceholder`).
+     *
+     * Even more importantly, the string context must be exactly one
+     * element, which is either a `NixStringContextElem::Opaque` or
+     * `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep`
+     * is not permitted).
+     *
+     * The string is parsed based on the context --- the context is the
+     * source of truth, and ultimately tells us what we want, and then
+     * we ensure the string corresponds to it.
+     */
+    DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
 
 public:
 
@@ -525,7 +551,7 @@ private:
         char * text,
         size_t length,
         Pos::Origin origin,
-        Path basePath,
+        const SourcePath & basePath,
         std::shared_ptr<StaticEnv> & staticEnv);
 
 public:
@@ -573,6 +599,37 @@ public:
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, PosIdx pos);
 
+    /**
+     * Create a string representing a store path.
+     *
+     * The string is the printed store path with a context containing a single
+     * `NixStringContextElem::Opaque` element of that store path.
+     */
+    void mkStorePathString(const StorePath & storePath, Value & v);
+
+    /**
+     * Create a string representing a `DerivedPath::Built`.
+     *
+     * The string is the printed store path with a context containing a single
+     * `NixStringContextElem::Built` element of the drv path and output name.
+     *
+     * @param value Value we are settings
+     *
+     * @param drvPath Path the drv whose output we are making a string for
+     *
+     * @param outputName Name of the output
+     *
+     * @param optOutputPath Optional output path for that string. Must
+     * be passed if and only if output store object is input-addressed.
+     * Will be printed to form string if passed, otherwise a placeholder
+     * will be used (see `DownstreamPlaceholder`).
+     */
+    void mkOutputString(
+        Value & value,
+        const StorePath & drvPath,
+        const std::string outputName,
+        std::optional<StorePath> optOutputPath);
+
     void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
     /**
@@ -584,7 +641,7 @@ public:
      * Realise the given context, and return a mapping from the placeholders
      * used to construct the associated value to their final store path
      */
-    [[nodiscard]] StringMap realiseContext(const PathSet & context);
+    [[nodiscard]] StringMap realiseContext(const NixStringContext & context);
 
 private:
 
@@ -650,7 +707,7 @@ std::string showType(const Value & v);
 /**
  * If `path` refers to a directory, then append "/default.nix".
  */
-Path resolveExprPath(Path path);
+SourcePath resolveExprPath(const SourcePath & path);
 
 struct InvalidPathError : EvalError
 {
@@ -688,7 +745,13 @@ struct EvalSettings : Config
         )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",
-        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+        R"(
+          Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
+
+          - Restrict file system and network access to files specified by cryptographic hash
+          - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
+        )"
+        };
 
     Setting<bool> enableImportFromDerivation{
         this, true, "allow-import-from-derivation",

src/libexpr/flake/flake.cc

@@ -222,9 +222,9 @@ static Flake getFlake(
         throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
 
     Value vInfo;
-    state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
+    state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack
 
-    expectType(state, nAttrs, vInfo, state.positions.add({flakeFile}, 1, 1));
+    expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1));
 
     if (auto description = vInfo.attrs->get(state.sDescription)) {
         expectType(state, nString, *description->value, description->pos);
@@ -265,7 +265,7 @@ static Flake getFlake(
                     state.symbols[setting.name],
                     std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
             else if (setting.value->type() == nPath) {
-                PathSet emptyContext = {};
+                NixStringContext emptyContext = {};
                 flake.config.settings.emplace(
                     state.symbols[setting.name],
                     state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned());
@@ -745,7 +745,7 @@ void callFlake(EvalState & state,
         state.vCallFlake = allocRootValue(state.allocValue());
         state.eval(state.parseExprFromString(
             #include "call-flake.nix.gen.hh"
-            , "/"), **state.vCallFlake);
+            , CanonPath::root), **state.vCallFlake);
     }
 
     state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);

src/libexpr/get-drvs.cc

@@ -71,7 +71,7 @@ std::optional<StorePath> DrvInfo::queryDrvPath() const
 {
     if (!drvPath && attrs) {
         Bindings::iterator i = attrs->find(state->sDrvPath);
-        PathSet context;
+        NixStringContext context;
         if (i == attrs->end())
             drvPath = {std::nullopt};
         else
@@ -93,7 +93,7 @@ StorePath DrvInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
-        PathSet context;
+        NixStringContext context;
         if (i != attrs->end())
             outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation");
     }
@@ -124,7 +124,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
                     /* And evaluate its ‘outPath’ attribute. */
                     Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
                     if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
-                    PathSet context;
+                    NixStringContext context;
                     outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation"));
                 } else
                     outputs.emplace(output, std::nullopt);

src/libexpr/nixexpr.cc

@@ -32,9 +32,9 @@ struct PosAdapter : AbstractPos
                 // Get rid of the null terminators added by the parser.
                 return std::string(s.source->c_str());
             },
-            [](const Path & path) -> std::optional<std::string> {
+            [](const SourcePath & path) -> std::optional<std::string> {
                 try {
-                    return readFile(path);
+                    return path.readFile();
                 } catch (Error &) {
                     return std::nullopt;
                 }
@@ -48,7 +48,7 @@ struct PosAdapter : AbstractPos
             [&](const Pos::none_tag &) { out << "«none»"; },
             [&](const Pos::Stdin &) { out << "«stdin»"; },
             [&](const Pos::String & s) { out << "«string»"; },
-            [&](const Path & path) { out << path; }
+            [&](const SourcePath & path) { out << path; }
         }, origin);
     }
 };

src/libexpr/nixexpr.hh

@@ -34,7 +34,7 @@ struct Pos
     struct Stdin { ref<std::string> source; };
     struct String { ref<std::string> source; };
 
-    typedef std::variant<none_tag, Stdin, String, Path> Origin;
+    typedef std::variant<none_tag, Stdin, String, SourcePath> Origin;
 
     Origin origin;
 

src/libexpr/parser.y

@@ -31,7 +31,7 @@ namespace nix {
         EvalState & state;
         SymbolTable & symbols;
         Expr * result;
-        Path basePath;
+        SourcePath basePath;
         PosTable::Origin origin;
         std::optional<ErrorInfo> error;
     };
@@ -509,7 +509,7 @@ string_parts_interpolated
 
 path_start
   : PATH {
-    Path path(absPath({$1.p, $1.l}, data->basePath));
+    Path path(absPath({$1.p, $1.l}, data->basePath.path.abs()));
     /* add back in the trailing '/' to the first segment */
     if ($1.p[$1.l-1] == '/' && $1.l > 1)
       path += "/";
@@ -651,7 +651,7 @@ Expr * EvalState::parse(
     char * text,
     size_t length,
     Pos::Origin origin,
-    Path basePath,
+    const SourcePath & basePath,
     std::shared_ptr<StaticEnv> & staticEnv)
 {
     yyscan_t scanner;
@@ -675,48 +675,36 @@ Expr * EvalState::parse(
 }
 
 
-Path resolveExprPath(Path path)
+SourcePath resolveExprPath(const SourcePath & path)
 {
-    assert(path[0] == '/');
-
-    unsigned int followCount = 0, maxFollow = 1024;
-
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
-    struct stat st;
-    while (true) {
-        // Basic cycle/depth limit to avoid infinite loops.
-        if (++followCount >= maxFollow)
-            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
-        st = lstat(path);
-        if (!S_ISLNK(st.st_mode)) break;
-        path = absPath(readLink(path), dirOf(path));
-    }
+    auto path2 = path.resolveSymlinks();
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (S_ISDIR(st.st_mode))
-        path = canonPath(path + "/default.nix");
+    if (path2.lstat().type == InputAccessor::tDirectory)
+        return path2 + "default.nix";
 
-    return path;
+    return path2;
 }
 
 
-Expr * EvalState::parseExprFromFile(const Path & path)
+Expr * EvalState::parseExprFromFile(const SourcePath & path)
 {
     return parseExprFromFile(path, staticBaseEnv);
 }
 
 
-Expr * EvalState::parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv> & staticEnv)
+Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
 {
-    auto buffer = readFile(path);
-    // readFile should have left some extra space for terminators
+    auto buffer = path.readFile();
+    // readFile hopefully have left some extra space for terminators
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), path, dirOf(path), staticEnv);
+    return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
 {
     auto s = make_ref<std::string>(std::move(s_));
     s->append("\0\0", 2);
@@ -724,7 +712,7 @@ Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
 {
     return parseExprFromString(std::move(s), basePath, staticBaseEnv);
 }
@@ -737,7 +725,7 @@ Expr * EvalState::parseStdin()
     // drainFD should have left some extra space for terminators
     buffer.append("\0\0", 2);
     auto s = make_ref<std::string>(std::move(buffer));
-    return parse(s->data(), s->size(), Pos::Stdin{.source = s}, absPath("."), staticBaseEnv);
+    return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
 }
 
 
@@ -757,13 +745,13 @@ void EvalState::addToSearchPath(const std::string & s)
 }
 
 
-Path EvalState::findFile(const std::string_view path)
+SourcePath EvalState::findFile(const std::string_view path)
 {
     return findFile(searchPath, path);
 }
 
 
-Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos)
+SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos)
 {
     for (auto & i : searchPath) {
         std::string suffix;
@@ -779,11 +767,11 @@ Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, c
         auto r = resolveSearchPathElem(i);
         if (!r.first) continue;
         Path res = r.second + suffix;
-        if (pathExists(res)) return canonPath(res);
+        if (pathExists(res)) return CanonPath(canonPath(res));
     }
 
     if (hasPrefix(path, "nix/"))
-        return concatStrings(corepkgsPrefix, path.substr(4));
+        return CanonPath(concatStrings(corepkgsPrefix, path.substr(4)));
 
     debugThrow(ThrownError({
         .msg = hintfmt(evalSettings.pureEval

src/libexpr/paths.cc

@@ -0,0 +1,10 @@
+#include "eval.hh"
+
+namespace nix {
+
+SourcePath EvalState::rootPath(CanonPath path)
+{
+    return std::move(path);
+}
+
+}

src/libexpr/primops.cc

@@ -1,5 +1,6 @@
 #include "archive.hh"
 #include "derivations.hh"
+#include "downstream-placeholder.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "globals.hh"
@@ -38,17 +39,16 @@ namespace nix {
 InvalidPathError::InvalidPathError(const Path & path) :
     EvalError("path '%s' is not valid", path), path(path) {}
 
-StringMap EvalState::realiseContext(const PathSet & context)
+StringMap EvalState::realiseContext(const NixStringContext & context)
 {
     std::vector<DerivedPath::Built> drvs;
     StringMap res;
 
-    for (auto & c_ : context) {
+    for (auto & c : context) {
         auto ensureValid = [&](const StorePath & p) {
             if (!store->isValidPath(p))
                 debugThrowLastTrace(InvalidPathError(store->printStorePath(p)));
         };
-        auto c = NixStringContextElem::parse(*store, c_);
         std::visit(overloaded {
             [&](const NixStringContextElem::Built & b) {
                 drvs.push_back(DerivedPath::Built {
@@ -88,7 +88,7 @@ StringMap EvalState::realiseContext(const PathSet & context)
         auto outputs = resolveDerivedPath(*store, drv);
         for (auto & [outputName, outputPath] : outputs) {
             res.insert_or_assign(
-                downstreamPlaceholder(*store, drv.drvPath, outputName),
+                DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(),
                 store->printStorePath(outputPath)
             );
         }
@@ -110,16 +110,16 @@ struct RealisePathFlags {
     bool checkForPureEval = true;
 };
 
-static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {})
+static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {})
 {
-    PathSet context;
+    NixStringContext context;
 
     auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
 
     try {
         StringMap rewrites = state.realiseContext(context);
 
-        auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context);
+        auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context)));
 
         return flags.checkForPureEval
             ? state.checkSourcePath(realPath)
@@ -130,35 +130,31 @@ static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const Re
     }
 }
 
-/* Add and attribute to the given attribute map from the output name to
-   the output path, or a placeholder.
-
-   Where possible the path is used, but for floating CA derivations we
-   may not know it. For sake of determinism we always assume we don't
-   and instead put in a place holder. In either case, however, the
-   string context will contain the drv path and output name, so
-   downstream derivations will have the proper dependency, and in
-   addition, before building, the placeholder will be rewritten to be
-   the actual path.
-
-   The 'drv' and 'drvPath' outputs must correspond. */
+/**
+ * Add and attribute to the given attribute map from the output name to
+ * the output path, or a placeholder.
+ *
+ * Where possible the path is used, but for floating CA derivations we
+ * may not know it. For sake of determinism we always assume we don't
+ * and instead put in a place holder. In either case, however, the
+ * string context will contain the drv path and output name, so
+ * downstream derivations will have the proper dependency, and in
+ * addition, before building, the placeholder will be rewritten to be
+ * the actual path.
+ *
+ * The 'drv' and 'drvPath' outputs must correspond.
+ */
 static void mkOutputString(
     EvalState & state,
     BindingsBuilder & attrs,
     const StorePath & drvPath,
-    const BasicDerivation & drv,
     const std::pair<std::string, DerivationOutput> & o)
 {
-    auto optOutputPath = o.second.path(*state.store, drv.name, o.first);
-    attrs.alloc(o.first).mkString(
-        optOutputPath
-            ? state.store->printStorePath(*optOutputPath)
-            /* Downstream we would substitute this for an actual path once
-               we build the floating CA derivation */
-            /* FIXME: we need to depend on the basic derivation, not
-               derivation */
-            : downstreamPlaceholder(*state.store, drvPath, o.first),
-        {"!" + o.first + "!" + state.store->printStorePath(drvPath)});
+    state.mkOutputString(
+        attrs.alloc(o.first),
+        drvPath,
+        o.first,
+        o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first));
 }
 
 /* Load and evaluate an expression from path specified by the
@@ -166,28 +162,30 @@ static void mkOutputString(
 static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
 {
     auto path = realisePath(state, pos, vPath);
+    auto path2 = path.path.abs();
 
     // FIXME
     auto isValidDerivationInStore = [&]() -> std::optional<StorePath> {
-        if (!state.store->isStorePath(path))
+        if (!state.store->isStorePath(path2))
             return std::nullopt;
-        auto storePath = state.store->parseStorePath(path);
-        if (!(state.store->isValidPath(storePath) && isDerivation(path)))
+        auto storePath = state.store->parseStorePath(path2);
+        if (!(state.store->isValidPath(storePath) && isDerivation(path2)))
             return std::nullopt;
         return storePath;
     };
 
-    if (auto optStorePath = isValidDerivationInStore()) {
-        auto storePath = *optStorePath;
-        Derivation drv = state.store->readDerivation(storePath);
+    if (auto storePath = isValidDerivationInStore()) {
+        Derivation drv = state.store->readDerivation(*storePath);
         auto attrs = state.buildBindings(3 + drv.outputs.size());
-        attrs.alloc(state.sDrvPath).mkString(path, {"=" + path});
+        attrs.alloc(state.sDrvPath).mkString(path2, {
+            NixStringContextElem::DrvDeep { .drvPath = *storePath },
+        });
         attrs.alloc(state.sName).mkString(drv.env["name"]);
         auto & outputsVal = attrs.alloc(state.sOutputs);
         state.mkList(outputsVal, drv.outputs.size());
 
         for (const auto & [i, o] : enumerate(drv.outputs)) {
-            mkOutputString(state, attrs, storePath, drv, o);
+            mkOutputString(state, attrs, *storePath, o);
             (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first);
         }
 
@@ -198,7 +196,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
             state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
             state.eval(state.parseExprFromString(
                 #include "imported-drv-to-derivation.nix.gen.hh"
-                , "/"), **state.vImportedDrvToDerivation);
+                , CanonPath::root), **state.vImportedDrvToDerivation);
         }
 
         state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh");
@@ -206,10 +204,10 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
         state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
     }
 
-    else if (path == corepkgsPrefix + "fetchurl.nix") {
+    else if (path2 == corepkgsPrefix + "fetchurl.nix") {
         state.eval(state.parseExprFromString(
             #include "fetchurl.nix.gen.hh"
-            , "/"), v);
+            , CanonPath::root), v);
     }
 
     else {
@@ -330,7 +328,7 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu
 
     std::string sym(state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative"));
 
-    void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
+    void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL);
     if (!handle)
         state.debugThrowLastTrace(EvalError("could not open '%1%': %2%", path, dlerror()));
 
@@ -358,7 +356,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
     auto count = args[0]->listSize();
     if (count == 0)
         state.error("at least one argument to 'exec' required").atPos(pos).debugThrow<EvalError>();
-    PathSet context;
+    NixStringContext context;
     auto program = state.coerceToString(pos, *elems[0], context,
             "while evaluating the first element of the argument passed to builtins.exec",
             false, false).toOwned();
@@ -378,7 +376,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
     auto output = runProgram(program, true, commandArgs);
     Expr * parsed;
     try {
-        parsed = state.parseExprFromString(std::move(output), "/");
+        parsed = state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root));
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while parsing the output from '%1%'", program);
         throw;
@@ -588,7 +586,7 @@ struct CompareValues
                 case nString:
                     return strcmp(v1->string.s, v2->string.s) < 0;
                 case nPath:
-                    return strcmp(v1->path, v2->path) < 0;
+                    return strcmp(v1->_path, v2->_path) < 0;
                 case nList:
                     // Lexicographic comparison
                     for (size_t i = 0;; i++) {
@@ -700,12 +698,14 @@ static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info {
     .arity = 1,
     .doc = R"(
       Take an *attrset* with values named `startSet` and `operator` in order to
-      return a *list of attrsets* by starting with the `startSet`, recursively
-      applying the `operator` function to each element. The *attrsets* in the
-      `startSet` and produced by the `operator` must each contain value named
-      `key` which are comparable to each other. The result is produced by
-      repeatedly calling the operator for each element encountered with a
-      unique key, terminating when no new elements are produced. For example,
+      return a *list of attrsets* by starting with the `startSet` and recursively
+      applying the `operator` function to each `item`. The *attrsets* in the
+      `startSet` and the *attrsets* produced by `operator` must contain a value
+      named `key` which is comparable. The result is produced by calling `operator`
+      for each `item` with a value for `key` that has not been called yet including
+      newly produced `item`s. The function terminates when no new `item`s are
+      produced. The resulting *list of attrsets* contains only *attrsets* with a
+      unique key. For example,
 
       ```
       builtins.genericClosure {
@@ -768,7 +768,7 @@ static RegisterPrimOp primop_abort({
     )",
     .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v)
     {
-        PathSet context;
+        NixStringContext context;
         auto s = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.abort").toOwned();
         state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s));
@@ -787,7 +787,7 @@ static RegisterPrimOp primop_throw({
     )",
     .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v)
     {
-      PathSet context;
+      NixStringContext context;
       auto s = state.coerceToString(pos, *args[0], context,
               "while evaluating the error message passed to builtin.throw").toOwned();
       state.debugThrowLastTrace(ThrownError(s));
@@ -800,7 +800,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
         state.forceValue(*args[1], pos);
         v = *args[1];
     } catch (Error & e) {
-        PathSet context;
+        NixStringContext context;
         auto message = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.addErrorContext",
                 false, false).toOwned();
@@ -1086,13 +1086,13 @@ drvName, Bindings * attrs, Value & v)
     Derivation drv;
     drv.name = drvName;
 
-    PathSet context;
+    NixStringContext context;
 
     bool contentAddressed = false;
     bool isImpure = false;
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
-    std::optional<FileIngestionMethod> ingestionMethod;
+    std::optional<ContentAddressMethod> ingestionMethod;
 
     StringSet outputs;
     outputs.insert("out");
@@ -1105,7 +1105,10 @@ drvName, Bindings * attrs, Value & v)
         auto handleHashMode = [&](const std::string_view s) {
             if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
-            else
+            else if (s == "text") {
+                experimentalFeatureSettings.require(Xp::DynamicDerivations);
+                ingestionMethod = TextIngestionMethod {};
+            } else
                 state.debugThrowLastTrace(EvalError({
                     .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
                     .errPos = state.positions[noPos]
@@ -1149,16 +1152,14 @@ drvName, Bindings * attrs, Value & v)
                 if (i->value->type() == nNull) continue;
             }
 
-            if (i->name == state.sContentAddressed) {
-                contentAddressed = state.forceBool(*i->value, noPos, context_below);
-                if (contentAddressed)
-                    experimentalFeatureSettings.require(Xp::CaDerivations);
+            if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) {
+                contentAddressed = true;
+                experimentalFeatureSettings.require(Xp::CaDerivations);
             }
 
-            else if (i->name == state.sImpure) {
-                isImpure = state.forceBool(*i->value, noPos, context_below);
-                if (isImpure)
-                    experimentalFeatureSettings.require(Xp::ImpureDerivations);
+            else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) {
+                isImpure = true;
+                experimentalFeatureSettings.require(Xp::ImpureDerivations);
             }
 
             /* The `args' attribute is special: it supplies the
@@ -1232,8 +1233,7 @@ drvName, Bindings * attrs, Value & v)
     /* Everything in the context of the strings in the derivation
        attributes should be added as dependencies of the resulting
        derivation. */
-    for (auto & c_ : context) {
-        auto c = NixStringContextElem::parse(*state.store, c_);
+    for (auto & c : context) {
         std::visit(overloaded {
             /* Since this allows the builder to gain access to every
                path in the dependency graph of the derivation (including
@@ -1273,11 +1273,16 @@ drvName, Bindings * attrs, Value & v)
         }));
 
     /* Check whether the derivation name is valid. */
-    if (isDerivation(drvName))
+    if (isDerivation(drvName) &&
+        !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
+          outputs.size() == 1 &&
+          *(outputs.begin()) == "out"))
+    {
         state.debugThrowLastTrace(EvalError({
-            .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
+            .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension),
             .errPos = state.positions[noPos]
         }));
+    }
 
     if (outputHash) {
         /* Handle fixed-output derivations.
@@ -1293,21 +1298,15 @@ drvName, Bindings * attrs, Value & v)
         auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
 
         auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
-        auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
-            .hash = {
-                .method = method,
-                .hash = h,
-            },
-            .references = {},
-        });
-        drv.env["out"] = state.store->printStorePath(outPath);
-        drv.outputs.insert_or_assign("out",
-            DerivationOutput::CAFixed {
-                .hash = FixedOutputHash {
-                    .method = method,
-                    .hash = std::move(h),
-                },
-            });
+
+        DerivationOutput::CAFixed dof {
+            .ca = ContentAddress::fromParts(
+                std::move(method),
+                std::move(h)),
+        };
+
+        drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
+        drv.outputs.insert_or_assign("out", std::move(dof));
     }
 
     else if (contentAddressed || isImpure) {
@@ -1325,13 +1324,13 @@ drvName, Bindings * attrs, Value & v)
             if (isImpure)
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::Impure {
-                        .method = method,
+                        .method = method.raw,
                         .hashType = ht,
                     });
             else
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::CAFloating {
-                        .method = method,
+                        .method = method.raw,
                         .hashType = ht,
                     });
         }
@@ -1392,9 +1391,11 @@ drvName, Bindings * attrs, Value & v)
     }
 
     auto result = state.buildBindings(1 + drv.outputs.size());
-    result.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS});
+    result.alloc(state.sDrvPath).mkString(drvPathS, {
+        NixStringContextElem::DrvDeep { .drvPath = drvPath },
+    });
     for (auto & i : drv.outputs)
-        mkOutputString(state, result, drvPath, drv, i);
+        mkOutputString(state, result, drvPath, i);
 
     v.mkAttrs(result);
 }
@@ -1437,9 +1438,9 @@ static RegisterPrimOp primop_placeholder({
 /* Convert the argument to a path.  !!! obsolete? */
 static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
-    Path path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath");
-    v.mkString(canonPath(path), context);
+    NixStringContext context;
+    auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath");
+    v.mkString(path.path.abs(), context);
 }
 
 static RegisterPrimOp primop_toPath({
@@ -1468,22 +1469,23 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
             .errPos = state.positions[pos]
         }));
 
-    PathSet context;
-    Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath"));
+    NixStringContext context;
+    auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path;
     /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
        directly in the store.  The latter condition is necessary so
        e.g. nix-push does the right thing. */
-    if (!state.store->isStorePath(path)) path = canonPath(path, true);
-    if (!state.store->isInStore(path))
+    if (!state.store->isStorePath(path.abs()))
+        path = CanonPath(canonPath(path.abs(), true));
+    if (!state.store->isInStore(path.abs()))
         state.debugThrowLastTrace(EvalError({
             .msg = hintfmt("path '%1%' is not in the Nix store", path),
             .errPos = state.positions[pos]
         }));
-    auto path2 = state.store->toStorePath(path).first;
+    auto path2 = state.store->toStorePath(path.abs()).first;
     if (!settings.readOnlyMode)
         state.store->ensurePath(path2);
-    context.insert(state.store->printStorePath(path2));
-    v.mkString(path, context);
+    context.insert(NixStringContextElem::Opaque { .path = path2 });
+    v.mkString(path.abs(), context);
 }
 
 static RegisterPrimOp primop_storePath({
@@ -1499,7 +1501,7 @@ static RegisterPrimOp primop_storePath({
       causes the path to be *copied* again to the Nix store, resulting
       in a new path (e.g. `/nix/store/ld01dnzc…-source-source`).
 
-      This function is not available in pure evaluation mode.
+      Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
     )",
     .fun = prim_storePath,
 });
@@ -1514,7 +1516,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
     auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false });
 
     try {
-        v.mkBool(pathExists(state.checkSourcePath(path)));
+        v.mkBool(state.checkSourcePath(path).pathExists());
     } catch (SysError & e) {
         /* Don't give away info from errors while canonicalising
            ‘path’ in restricted mode. */
@@ -1538,7 +1540,7 @@ static RegisterPrimOp primop_pathExists({
    following the last slash. */
 static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context,
             "while evaluating the first argument passed to builtins.baseNameOf",
             false, false)), context);
@@ -1560,12 +1562,18 @@ static RegisterPrimOp primop_baseNameOf({
    of the argument. */
 static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
-    auto path = state.coerceToString(pos, *args[0], context,
-            "while evaluating the first argument passed to builtins.dirOf",
+    state.forceValue(*args[0], pos);
+    if (args[0]->type() == nPath) {
+        auto path = args[0]->path();
+        v.mkPath(path.path.isRoot() ? path : path.parent());
+    } else {
+        NixStringContext context;
+        auto path = state.coerceToString(pos, *args[0], context,
+            "while evaluating the first argument passed to 'builtins.dirOf'",
             false, false);
-    auto dir = dirOf(*path);
-    if (args[0]->type() == nPath) v.mkPath(dir); else v.mkString(dir, context);
+        auto dir = dirOf(*path);
+        v.mkString(dir, context);
+    }
 }
 
 static RegisterPrimOp primop_dirOf({
@@ -1583,13 +1591,13 @@ static RegisterPrimOp primop_dirOf({
 static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     auto path = realisePath(state, pos, *args[0]);
-    auto s = readFile(path);
+    auto s = path.readFile();
     if (s.find((char) 0) != std::string::npos)
         state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path));
     StorePathSet refs;
-    if (state.store->isInStore(path)) {
+    if (state.store->isInStore(path.path.abs())) {
         try {
-            refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references;
+            refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references;
         } catch (Error &) { // FIXME: should be InvalidPathError
         }
         // Re-scan references to filter down to just the ones that actually occur in the file.
@@ -1597,7 +1605,12 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V
         refsSink << s;
         refs = refsSink.getResultPaths();
     }
-    auto context = state.store->printStorePathSet(refs);
+    NixStringContext context;
+    for (auto && p : std::move(refs)) {
+        context.insert(NixStringContextElem::Opaque {
+            .path = std::move((StorePath &&)p),
+        });
+    }
     v.mkString(s, context);
 }
 
@@ -1628,7 +1641,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
 
         i = getAttr(state, state.sPath, v2->attrs, "in an element of the __nixPath");
 
-        PathSet context;
+        NixStringContext context;
         auto path = state.coerceToString(pos, *i->value, context,
                 "while evaluating the `path` attribute of an element of the list passed to builtins.findFile",
                 false, false).toOwned();
@@ -1670,7 +1683,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V
 
     auto path = realisePath(state, pos, *args[1]);
 
-    v.mkString(hashFile(*ht, path).to_string(Base16, false));
+    v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -1684,26 +1697,20 @@ static RegisterPrimOp primop_hashFile({
     .fun = prim_hashFile,
 });
 
-
-/* Stringize a directory entry enum. Used by `readFileType' and `readDir'. */
-static const char * dirEntTypeToString(unsigned char dtType)
+static std::string_view fileTypeToString(InputAccessor::Type type)
 {
-    /* Enum DT_(DIR|LNK|REG|UNKNOWN) */
-    switch(dtType) {
-        case DT_REG: return "regular";   break;
-        case DT_DIR: return "directory"; break;
-        case DT_LNK: return "symlink";   break;
-        default:     return "unknown";   break;
-    }
-    return "unknown";  /* Unreachable */
+    return
+        type == InputAccessor::Type::tRegular ? "regular" :
+        type == InputAccessor::Type::tDirectory ? "directory" :
+        type == InputAccessor::Type::tSymlink ? "symlink" :
+        "unknown";
 }
 
-
 static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     auto path = realisePath(state, pos, *args[0]);
     /* Retrieve the directory entry type and stringize it. */
-    v.mkString(dirEntTypeToString(getFileType(path)));
+    v.mkString(fileTypeToString(path.lstat().type));
 }
 
 static RegisterPrimOp primop_readFileType({
@@ -1724,8 +1731,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va
     // Retrieve directory entries for all nodes in a directory.
     // This is similar to `getFileType` but is optimized to reduce system calls
     // on many systems.
-    DirEntries entries = readDirectory(path);
-
+    auto entries = path.readDirectory();
     auto attrs = state.buildBindings(entries.size());
 
     // If we hit unknown directory entry types we may need to fallback to
@@ -1734,22 +1740,21 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va
     // `builtins.readFileType` application.
     Value * readFileType = nullptr;
 
-    for (auto & ent : entries) {
-        auto & attr = attrs.alloc(ent.name);
-        if (ent.type == DT_UNKNOWN) {
+    for (auto & [name, type] : entries) {
+        auto & attr = attrs.alloc(name);
+        if (!type) {
             // Some filesystems or operating systems may not be able to return
             // detailed node info quickly in this case we produce a thunk to
             // query the file type lazily.
             auto epath = state.allocValue();
-            Path path2 = path + "/" + ent.name;
-            epath->mkString(path2);
+            epath->mkPath(path + name);
             if (!readFileType)
                 readFileType = &state.getBuiltin("readFileType");
             attr.mkApp(readFileType, epath);
         } else {
             // This branch of the conditional is much more likely.
             // Here we just stringize the directory entry type.
-            attr.mkString(dirEntTypeToString(ent.type));
+            attr.mkString(fileTypeToString(*type));
         }
     }
 
@@ -1787,7 +1792,7 @@ static RegisterPrimOp primop_readDir({
 static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     std::ostringstream out;
-    PathSet context;
+    NixStringContext context;
     printValueAsXML(state, true, false, *args[0], out, context, pos);
     v.mkString(out.str(), context);
 }
@@ -1895,7 +1900,7 @@ static RegisterPrimOp primop_toXML({
 static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     std::ostringstream out;
-    PathSet context;
+    NixStringContext context;
     printValueAsJSON(state, true, *args[0], pos, out, context);
     v.mkString(out.str(), context);
 }
@@ -1945,22 +1950,23 @@ static RegisterPrimOp primop_fromJSON({
    as an input by derivations. */
 static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile"));
     std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile"));
 
     StorePathSet refs;
 
-    for (auto path : context) {
-        if (path.at(0) != '/')
+    for (auto c : context) {
+        if (auto p = std::get_if<NixStringContextElem::Opaque>(&c))
+            refs.insert(p->path);
+        else
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt(
                     "in 'toFile': the file named '%1%' must not contain a reference "
                     "to a derivation but contains (%2%)",
-                    name, path),
+                    name, c.to_string()),
                 .errPos = state.positions[pos]
             }));
-        refs.insert(state.store->parseStorePath(path));
     }
 
     auto storePath = settings.readOnlyMode
@@ -2055,13 +2061,13 @@ static RegisterPrimOp primop_toFile({
 static void addPath(
     EvalState & state,
     const PosIdx pos,
-    const std::string & name,
+    std::string_view name,
     Path path,
     Value * filterFun,
     FileIngestionMethod method,
     const std::optional<Hash> expectedHash,
     Value & v,
-    const PathSet & context)
+    const NixStringContext & context)
 {
     try {
         // FIXME: handle CA derivation outputs (where path needs to
@@ -2083,7 +2089,7 @@ static void addPath(
 
         path = evalSettings.pureEval && expectedHash
             ? path
-            : state.checkSourcePath(path);
+            : state.checkSourcePath(CanonPath(path)).path.abs();
 
         PathFilter filter = filterFun ? ([&](const Path & path) {
             auto st = lstat(path);
@@ -2135,10 +2141,11 @@ static void addPath(
 
 static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
-    Path path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to builtins.filterSource");
+    NixStringContext context;
+    auto path = state.coerceToPath(pos, *args[1], context,
+        "while evaluating the second argument (the path to filter) passed to builtins.filterSource");
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
-    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
+    addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({
@@ -2198,18 +2205,19 @@ static RegisterPrimOp primop_filterSource({
 
 static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.path");
-    Path path;
+    std::optional<SourcePath> path;
     std::string name;
     Value * filterFun = nullptr;
     auto method = FileIngestionMethod::Recursive;
     std::optional<Hash> expectedHash;
-    PathSet context;
+    NixStringContext context;
+
+    state.forceAttrs(*args[0], pos, "while evaluating the argument passed to 'builtins.path'");
 
     for (auto & attr : *args[0]->attrs) {
         auto n = state.symbols[attr.name];
         if (n == "path")
-            path = state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the `path` attribute passed to builtins.path");
+            path.emplace(state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'"));
         else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.path");
         else if (n == "filter")
@@ -2224,15 +2232,15 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
                 .errPos = state.positions[attr.pos]
             }));
     }
-    if (path.empty())
+    if (!path)
         state.debugThrowLastTrace(EvalError({
             .msg = hintfmt("missing required 'path' attribute in the first argument to builtins.path"),
             .errPos = state.positions[pos]
         }));
     if (name.empty())
-        name = baseNameOf(path);
+        name = path->baseName();
 
-    addPath(state, pos, name, path, filterFun, method, expectedHash, v, context);
+    addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context);
 }
 
 static RegisterPrimOp primop_path({
@@ -3538,7 +3546,7 @@ static RegisterPrimOp primop_lessThan({
    `"/nix/store/whatever..."'. */
 static void prim_toString(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     auto s = state.coerceToString(pos, *args[0], context,
             "while evaluating the first argument passed to builtins.toString",
             true, false);
@@ -3577,7 +3585,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args,
 {
     int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring");
     int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
-    PathSet context;
+    NixStringContext context;
     auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
 
     if (start < 0)
@@ -3611,7 +3619,7 @@ static RegisterPrimOp primop_substring({
 
 static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength");
     v.mkInt(s->size());
 }
@@ -3637,7 +3645,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args,
             .errPos = state.positions[pos]
         }));
 
-    PathSet context; // discarded
+    NixStringContext context; // discarded
     auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
 
     v.mkString(hashString(*ht, s).to_string(Base16, false));
@@ -3683,7 +3691,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 
         auto regex = state.regexCache->get(re);
 
-        PathSet context;
+        NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match");
 
         std::cmatch match;
@@ -3763,7 +3771,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 
         auto regex = state.regexCache->get(re);
 
-        PathSet context;
+        NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split");
 
         auto begin = std::cregex_iterator(str.begin(), str.end(), regex);
@@ -3860,7 +3868,7 @@ static RegisterPrimOp primop_split({
 
 static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
 
     auto sep = state.forceString(*args[0], context, pos, "while evaluating the first argument (the separator string) passed to builtins.concatStringsSep");
     state.forceList(*args[1], pos, "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep");
@@ -3900,15 +3908,10 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
     for (auto elem : args[0]->listItems())
         from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings"));
 
-    std::vector<std::pair<std::string, PathSet>> to;
-    to.reserve(args[1]->listSize());
-    for (auto elem : args[1]->listItems()) {
-        PathSet ctx;
-        auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings");
-        to.emplace_back(s, std::move(ctx));
-    }
+    std::unordered_map<size_t, std::string> cache;
+    auto to = args[1]->listItems();
 
-    PathSet context;
+    NixStringContext context;
     auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings");
 
     std::string res;
@@ -3917,10 +3920,19 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
         bool found = false;
         auto i = from.begin();
         auto j = to.begin();
-        for (; i != from.end(); ++i, ++j)
+        size_t j_index = 0;
+        for (; i != from.end(); ++i, ++j, ++j_index)
             if (s.compare(p, i->size(), *i) == 0) {
                 found = true;
-                res += j->first;
+                auto v = cache.find(j_index);
+                if (v == cache.end()) {
+                    NixStringContext ctx;
+                    auto ts = state.forceString(**j, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings");
+                    v = (cache.emplace(j_index, ts)).first;
+                    for (auto& path : ctx)
+                        context.insert(path);
+                }
+                res += v->second;
                 if (i->empty()) {
                     if (p < s.size())
                         res += s[p];
@@ -3928,9 +3940,6 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
                 } else {
                     p += i->size();
                 }
-                for (auto& path : j->second)
-                    context.insert(path);
-                j->second.clear();
                 break;
             }
         if (!found) {
@@ -3948,7 +3957,11 @@ static RegisterPrimOp primop_replaceStrings({
     .args = {"from", "to", "s"},
     .doc = R"(
       Given string *s*, replace every occurrence of the strings in *from*
-      with the corresponding string in *to*. For example,
+      with the corresponding string in *to*.
+
+      The argument *to* is lazy, that is, it is only evaluated when its corresponding pattern in *from* is matched in the string *s*
+
+      Example:
 
       ```nix
       builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
@@ -4150,7 +4163,6 @@ void EvalState::createBaseEnv()
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    sDerivationNix = symbols.create(derivationNixPath);
     auto vDerivation = allocValue();
     addConstant("derivation", vDerivation);
 
@@ -4167,7 +4179,7 @@ void EvalState::createBaseEnv()
         // the parser needs two NUL bytes as terminators; one of them
         // is implied by being a C string.
         "\0";
-    eval(parse(code, sizeof(code), derivationNixPath, "/", staticBaseEnv), *vDerivation);
+    eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
 }
 
 

src/libexpr/primops/context.cc

@@ -7,7 +7,7 @@ namespace nix {
 
 static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
     v.mkString(*s);
 }
@@ -17,7 +17,7 @@ static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringCo
 
 static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext");
     v.mkBool(!context.empty());
 }
@@ -33,17 +33,18 @@ static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
    drv.inputDrvs. */
 static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency");
 
-    PathSet context2;
-    for (auto && p : context) {
-        auto c = NixStringContextElem::parse(*state.store, p);
+    NixStringContext context2;
+    for (auto && c : context) {
         if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) {
-            context2.emplace(state.store->printStorePath(ptr->drvPath));
+            context2.emplace(NixStringContextElem::Opaque {
+                .path = ptr->drvPath
+            });
         } else {
             /* Can reuse original item */
-            context2.emplace(std::move(p));
+            context2.emplace(std::move(c));
         }
     }
 
@@ -79,22 +80,21 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
         bool allOutputs = false;
         Strings outputs;
     };
-    PathSet context;
+    NixStringContext context;
     state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext");
     auto contextInfos = std::map<StorePath, ContextInfo>();
-    for (const auto & p : context) {
-        NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p);
+    for (auto && i : context) {
         std::visit(overloaded {
-            [&](NixStringContextElem::DrvDeep & d) {
-                contextInfos[d.drvPath].allOutputs = true;
+            [&](NixStringContextElem::DrvDeep && d) {
+                contextInfos[std::move(d.drvPath)].allOutputs = true;
             },
-            [&](NixStringContextElem::Built & b) {
-                contextInfos[b.drvPath].outputs.emplace_back(std::move(b.output));
+            [&](NixStringContextElem::Built && b) {
+                contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output));
             },
-            [&](NixStringContextElem::Opaque & o) {
-                contextInfos[o.path].path = true;
+            [&](NixStringContextElem::Opaque && o) {
+                contextInfos[std::move(o.path)].path = true;
             },
-        }, ctx.raw());
+        }, ((NixStringContextElem &&) i).raw());
     }
 
     auto attrs = state.buildBindings(contextInfos.size());
@@ -129,7 +129,7 @@ static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext);
 */
 static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    PathSet context;
+    NixStringContext context;
     auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext");
 
     state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.appendContext");
@@ -143,13 +143,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
                 .msg = hintfmt("context key '%s' is not a store path", name),
                 .errPos = state.positions[i.pos]
             });
+        auto namePath = state.store->parseStorePath(name);
         if (!settings.readOnlyMode)
-            state.store->ensurePath(state.store->parseStorePath(name));
+            state.store->ensurePath(namePath);
         state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context");
         auto iter = i.value->attrs->find(sPath);
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context"))
-                context.emplace(name);
+                context.emplace(NixStringContextElem::Opaque {
+                    .path = namePath,
+                });
         }
 
         iter = i.value->attrs->find(sAllOutputs);
@@ -161,7 +164,9 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
                         .errPos = state.positions[i.pos]
                     });
                 }
-                context.insert(concatStrings("=", name));
+                context.emplace(NixStringContextElem::DrvDeep {
+                    .drvPath = namePath,
+                });
             }
         }
 
@@ -176,7 +181,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
             }
             for (auto elem : iter->value->listItems()) {
                 auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context");
-                context.insert(concatStrings("!", outputName, "!", name));
+                context.emplace(NixStringContextElem::Built {
+                    .drvPath = namePath,
+                    .output = std::string { outputName },
+                });
             }
         }
     }

src/libexpr/primops/fetchClosure.cc

@@ -18,7 +18,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
         const auto & attrName = state.symbols[attr.name];
 
         if (attrName == "fromPath") {
-            PathSet context;
+            NixStringContext context;
             fromPath = state.coerceToStorePath(attr.pos, *attr.value, context,
                     "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure");
         }
@@ -27,7 +27,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
             state.forceValue(*attr.value, attr.pos);
             toCA = true;
             if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
-                PathSet context;
+                NixStringContext context;
                 toPath = state.coerceToStorePath(attr.pos, *attr.value, context,
                         "while evaluating the 'toPath' attribute passed to builtins.fetchClosure");
             }
@@ -114,8 +114,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
             });
     }
 
-    auto toPathS = state.store->printStorePath(*toPath);
-    v.mkString(toPathS, {toPathS});
+    state.mkStorePathString(*toPath, v);
 }
 
 static RegisterPrimOp primop_fetchClosure({

src/libexpr/primops/fetchMercurial.cc

@@ -13,7 +13,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     std::optional<Hash> rev;
     std::optional<std::string> ref;
     std::string_view name = "source";
-    PathSet context;
+    NixStringContext context;
 
     state.forceValue(*args[0], pos);
 
@@ -73,8 +73,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
     auto [tree, input2] = input.fetch(state.store);
 
     auto attrs2 = state.buildBindings(8);
-    auto storePath = state.store->printStorePath(tree.storePath);
-    attrs2.alloc(state.sOutPath).mkString(storePath, {storePath});
+    state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath));
     if (input2.getRef())
         attrs2.alloc("branch").mkString(*input2.getRef());
     // Backward compatibility: set 'rev' to

src/libexpr/primops/fetchTree.cc

@@ -24,9 +24,8 @@ void emitTreeAttrs(
 
     auto attrs = state.buildBindings(8);
 
-    auto storePath = state.store->printStorePath(tree.storePath);
 
-    attrs.alloc(state.sOutPath).mkString(storePath, {storePath});
+    state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath));
 
     // FIXME: support arbitrary input attributes.
 
@@ -107,7 +106,7 @@ static void fetchTree(
     const FetchTreeParams & params = FetchTreeParams{}
 ) {
     fetchers::Input input;
-    PathSet context;
+    NixStringContext context;
 
     state.forceValue(*args[0], pos);
 
@@ -287,9 +286,9 @@ static RegisterPrimOp primop_fetchurl({
     .name = "__fetchurl",
     .args = {"url"},
     .doc = R"(
-      Download the specified URL and return the path of the downloaded
-      file. This function is not available if [restricted evaluation
-      mode](../command-ref/conf-file.md) is enabled.
+      Download the specified URL and return the path of the downloaded file.
+
+      Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
     )",
     .fun = prim_fetchurl,
 });
@@ -339,8 +338,7 @@ static RegisterPrimOp primop_fetchTarball({
       stdenv.mkDerivation { … }
       ```
 
-      This function is not available if [restricted evaluation
-      mode](../command-ref/conf-file.md) is enabled.
+      Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
     )",
     .fun = prim_fetchTarball,
 });
@@ -471,14 +469,9 @@ static RegisterPrimOp primop_fetchGit({
           }
           ```
 
-          > **Note**
-          >
-          > Nix will refetch the branch in accordance with
-          > the option `tarball-ttl`.
+          Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting.
 
-          > **Note**
-          >
-          > This behavior is disabled in *Pure evaluation mode*.
+          This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
 
         - To fetch the content of a checked-out work directory:
 

src/libexpr/primops/fromTOML.cc

@@ -3,6 +3,8 @@
 
 #include "../../toml11/toml.hpp"
 
+#include <sstream>
+
 namespace nix {
 
 static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val)
@@ -58,8 +60,18 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
             case toml::value_t::offset_datetime:
             case toml::value_t::local_date:
             case toml::value_t::local_time:
-                // We fail since Nix doesn't have date and time types
-                throw std::runtime_error("Dates and times are not supported");
+                {
+                    if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) {
+                        auto attrs = state.buildBindings(2);
+                        attrs.alloc("_type").mkString("timestamp");
+                        std::ostringstream s;
+                        s << t;
+                        attrs.alloc("value").mkString(s.str());
+                        v.mkAttrs(attrs);
+                    } else {
+                        throw std::runtime_error("Dates and times are not supported");
+                    }
+                }
                 break;;
             case toml::value_t::empty:
                 v.mkNull();

src/libexpr/print.cc

@@ -1,4 +1,5 @@
 #include "print.hh"
+#include <unordered_set>
 
 namespace nix {
 
@@ -25,11 +26,26 @@ printLiteralBool(std::ostream & str, bool boolean)
     return str;
 }
 
+// Returns `true' is a string is a reserved keyword which requires quotation
+// when printing attribute set field names.
+//
+// This list should generally be kept in sync with `./lexer.l'.
+// You can test if a keyword needs to be added by running:
+//   $ nix eval --expr '{ <KEYWORD> = 1; }'
+// For example `or' doesn't need to be quoted.
+bool isReservedKeyword(const std::string_view str)
+{
+    static const std::unordered_set<std::string_view> reservedKeywords = {
+        "if", "then", "else", "assert", "with", "let", "in", "rec", "inherit"
+    };
+    return reservedKeywords.contains(str);
+}
+
 std::ostream &
 printIdentifier(std::ostream & str, std::string_view s) {
     if (s.empty())
         str << "\"\"";
-    else if (s == "if") // FIXME: handle other keywords
+    else if (isReservedKeyword(s))
         str << '"' << s << '"';
     else {
         char c = s[0];
@@ -50,10 +66,10 @@ printIdentifier(std::ostream & str, std::string_view s) {
     return str;
 }
 
-// FIXME: keywords
 static bool isVarName(std::string_view s)
 {
     if (s.size() == 0) return false;
+    if (isReservedKeyword(s)) return false;
     char c = s[0];
     if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false;
     for (auto & i : s)

src/libexpr/print.hh

@@ -35,6 +35,12 @@ namespace nix {
      */
     std::ostream & printAttributeName(std::ostream & o, std::string_view s);
 
+    /**
+     * Returns `true' is a string is a reserved keyword which requires quotation
+     * when printing attribute set field names.
+     */
+    bool isReservedKeyword(const std::string_view str);
+
     /**
      * Print a string as an identifier in the Nix expression language syntax.
      *

src/libexpr/tests/derived-path.cc

@@ -0,0 +1,65 @@
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+#include <rapidcheck/gtest.h>
+
+#include "tests/derived-path.hh"
+#include "tests/libexpr.hh"
+
+namespace nix {
+
+// Testing of trivial expressions
+class DerivedPathExpressionTest : public LibExprTest {};
+
+// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is
+// no a real fixture.
+//
+// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args
+TEST_F(DerivedPathExpressionTest, force_init)
+{
+}
+
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathExpressionTest,
+    prop_opaque_path_round_trip,
+    (const DerivedPath::Opaque & o))
+{
+    auto * v = state.allocValue();
+    state.mkStorePathString(o.path, *v);
+    auto d = state.coerceToDerivedPath(noPos, *v, "");
+    RC_ASSERT(DerivedPath { o } == d);
+}
+
+// TODO use DerivedPath::Built for parameter once it supports a single output
+// path only.
+
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathExpressionTest,
+    prop_built_path_placeholder_round_trip,
+    (const StorePath & drvPath, const StorePathName & outputName))
+{
+    auto * v = state.allocValue();
+    state.mkOutputString(*v, drvPath, outputName.name, std::nullopt);
+    auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
+    DerivedPath::Built b {
+        .drvPath = drvPath,
+        .outputs = OutputsSpec::Names { outputName.name },
+    };
+    RC_ASSERT(DerivedPath { b } == d);
+}
+
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathExpressionTest,
+    prop_built_path_out_path_round_trip,
+    (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath))
+{
+    auto * v = state.allocValue();
+    state.mkOutputString(*v, drvPath, outputName.name, outPath);
+    auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
+    DerivedPath::Built b {
+        .drvPath = drvPath,
+        .outputs = OutputsSpec::Names { outputName.name },
+    };
+    RC_ASSERT(DerivedPath { b } == d);
+}
+
+} /* namespace nix */

src/libexpr/tests/error_traces.cc

@@ -171,7 +171,7 @@ namespace nix {
                       hintfmt("value is %s while a string was expected", "an integer"),
                       hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings"));
 
-        ASSERT_TRACE2("replaceStrings [ \"old\" ] [ true ] {}",
+        ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"",
                       TypeError,
                       hintfmt("value is %s while a string was expected", "a Boolean"),
                       hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings"));

src/libexpr/tests/json.cc

@@ -8,7 +8,7 @@ namespace nix {
         protected:
             std::string getJSONValue(Value& value) {
                 std::stringstream ss;
-                PathSet ps;
+                NixStringContext ps;
                 printValueAsJSON(state, true, value, noPos, ss, ps);
                 return ss.str();
             }

src/libexpr/tests/libexpr.hh

@@ -28,7 +28,7 @@ namespace nix {
             }
             Value eval(std::string input, bool forceValue = true) {
                 Value v;
-                Expr * e = state.parseExprFromString(input, "");
+                Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root));
                 assert(e);
                 state.eval(e, v);
                 if (forceValue)

src/libexpr/tests/local.mk

@@ -12,7 +12,7 @@ libexpr-tests_SOURCES := \
     $(wildcard $(d)/*.cc) \
     $(wildcard $(d)/value/*.cc)
 
-libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests
+libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
 
 libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
 

src/libexpr/tests/value/context.cc

@@ -8,69 +8,62 @@
 
 namespace nix {
 
-// Testing of trivial expressions
-struct NixStringContextElemTest : public LibExprTest {
-   const Store & store() const {
-       return *LibExprTest::store;
-   }
-};
-
-TEST_F(NixStringContextElemTest, empty_invalid) {
+TEST(NixStringContextElemTest, empty_invalid) {
     EXPECT_THROW(
-        NixStringContextElem::parse(store(), ""),
+        NixStringContextElem::parse(""),
         BadNixStringContextElem);
 }
 
-TEST_F(NixStringContextElemTest, single_bang_invalid) {
+TEST(NixStringContextElemTest, single_bang_invalid) {
     EXPECT_THROW(
-        NixStringContextElem::parse(store(), "!"),
+        NixStringContextElem::parse("!"),
         BadNixStringContextElem);
 }
 
-TEST_F(NixStringContextElemTest, double_bang_invalid) {
+TEST(NixStringContextElemTest, double_bang_invalid) {
     EXPECT_THROW(
-        NixStringContextElem::parse(store(), "!!/"),
+        NixStringContextElem::parse("!!/"),
         BadStorePath);
 }
 
-TEST_F(NixStringContextElemTest, eq_slash_invalid) {
+TEST(NixStringContextElemTest, eq_slash_invalid) {
     EXPECT_THROW(
-        NixStringContextElem::parse(store(), "=/"),
+        NixStringContextElem::parse("=/"),
         BadStorePath);
 }
 
-TEST_F(NixStringContextElemTest, slash_invalid) {
+TEST(NixStringContextElemTest, slash_invalid) {
     EXPECT_THROW(
-        NixStringContextElem::parse(store(), "/"),
+        NixStringContextElem::parse("/"),
         BadStorePath);
 }
 
-TEST_F(NixStringContextElemTest, opaque) {
-    std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
-    auto elem = NixStringContextElem::parse(store(), opaque);
+TEST(NixStringContextElemTest, opaque) {
+    std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
+    auto elem = NixStringContextElem::parse(opaque);
     auto * p = std::get_if<NixStringContextElem::Opaque>(&elem);
     ASSERT_TRUE(p);
-    ASSERT_EQ(p->path, store().parseStorePath(opaque));
-    ASSERT_EQ(elem.to_string(store()), opaque);
+    ASSERT_EQ(p->path, StorePath { opaque });
+    ASSERT_EQ(elem.to_string(), opaque);
 }
 
-TEST_F(NixStringContextElemTest, drvDeep) {
-    std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
-    auto elem = NixStringContextElem::parse(store(), drvDeep);
+TEST(NixStringContextElemTest, drvDeep) {
+    std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
+    auto elem = NixStringContextElem::parse(drvDeep);
     auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem);
     ASSERT_TRUE(p);
-    ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1)));
-    ASSERT_EQ(elem.to_string(store()), drvDeep);
+    ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) });
+    ASSERT_EQ(elem.to_string(), drvDeep);
 }
 
-TEST_F(NixStringContextElemTest, built) {
-    std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
-    auto elem = NixStringContextElem::parse(store(), built);
+TEST(NixStringContextElemTest, built) {
+    std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
+    auto elem = NixStringContextElem::parse(built);
     auto * p = std::get_if<NixStringContextElem::Built>(&elem);
     ASSERT_TRUE(p);
     ASSERT_EQ(p->output, "foo");
-    ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5)));
-    ASSERT_EQ(elem.to_string(store()), built);
+    ASSERT_EQ(p->drvPath, StorePath { built.substr(5) });
+    ASSERT_EQ(elem.to_string(), built);
 }
 
 }
@@ -102,13 +95,15 @@ Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitra
 
 Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
 {
-    switch (*gen::inRange<uint8_t>(0, 2)) {
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {
     case 0:
         return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>());
     case 1:
         return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>());
-    default:
+    case 2:
         return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>());
+    default:
+        assert(false);
     }
 }
 
@@ -116,12 +111,12 @@ Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
 
 namespace nix {
 
-RC_GTEST_FIXTURE_PROP(
+RC_GTEST_PROP(
     NixStringContextElemTest,
     prop_round_rip,
     (const NixStringContextElem & o))
 {
-    RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store())));
+    RC_ASSERT(o == NixStringContextElem::parse(o.to_string()));
 }
 
 }

src/libexpr/value-to-json.cc

@@ -11,7 +11,7 @@
 namespace nix {
 using json = nlohmann::json;
 json printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, PathSet & context, bool copyToStore)
+    Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore)
 {
     checkInterrupt();
 
@@ -36,9 +36,10 @@ json printValueAsJSON(EvalState & state, bool strict,
 
         case nPath:
             if (copyToStore)
-                out = state.store->printStorePath(state.copyPathToStore(context, v.path));
+                out = state.store->printStorePath(
+                    state.copyPathToStore(context, v.path()));
             else
-                out = v.path;
+                out = v.path().path.abs();
             break;
 
         case nNull:
@@ -94,13 +95,13 @@ json printValueAsJSON(EvalState & state, bool strict,
 }
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore)
+    Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore)
 {
     str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
 }
 
 json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
-    PathSet & context, bool copyToStore) const
+    NixStringContext & context, bool copyToStore) const
 {
     state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType()));
 }

src/libexpr/value-to-json.hh

@@ -11,9 +11,9 @@
 namespace nix {
 
 nlohmann::json printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true);
+    Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore = true);
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true);
+    Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true);
 
 }

src/libexpr/value-to-xml.cc

@@ -18,21 +18,21 @@ static XMLAttrs singletonAttrs(const std::string & name, const std::string & val
 
 
 static void printValueAsXML(EvalState & state, bool strict, bool location,
-    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
+    Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen,
     const PosIdx pos);
 
 
 static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos)
 {
-    if (auto path = std::get_if<Path>(&pos.origin))
-        xmlAttrs["path"] = *path;
+    if (auto path = std::get_if<SourcePath>(&pos.origin))
+        xmlAttrs["path"] = path->path.abs();
     xmlAttrs["line"] = fmt("%1%", pos.line);
     xmlAttrs["column"] = fmt("%1%", pos.column);
 }
 
 
 static void showAttrs(EvalState & state, bool strict, bool location,
-    Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
+    Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen)
 {
     StringSet names;
 
@@ -54,7 +54,7 @@ static void showAttrs(EvalState & state, bool strict, bool location,
 
 
 static void printValueAsXML(EvalState & state, bool strict, bool location,
-    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
+    Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen,
     const PosIdx pos)
 {
     checkInterrupt();
@@ -78,7 +78,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             break;
 
         case nPath:
-            doc.writeEmptyElement("path", singletonAttrs("value", v.path));
+            doc.writeEmptyElement("path", singletonAttrs("value", v.path().to_string()));
             break;
 
         case nNull:
@@ -166,7 +166,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
 
 void ExternalValueBase::printValueAsXML(EvalState & state, bool strict,
-    bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
+    bool location, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen,
     const PosIdx pos) const
 {
     doc.writeEmptyElement("unevaluated");
@@ -174,7 +174,7 @@ void ExternalValueBase::printValueAsXML(EvalState & state, bool strict,
 
 
 void printValueAsXML(EvalState & state, bool strict, bool location,
-    Value & v, std::ostream & out, PathSet & context, const PosIdx pos)
+    Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos)
 {
     XMLWriter doc(true, out);
     XMLOpenElement root(doc, "expr");

src/libexpr/value-to-xml.hh

@@ -10,6 +10,6 @@
 namespace nix {
 
 void printValueAsXML(EvalState & state, bool strict, bool location,
-    Value & v, std::ostream & out, PathSet & context, const PosIdx pos);
+    Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos);
 
 }

src/libexpr/value.hh

@@ -5,6 +5,7 @@
 
 #include "symbol-table.hh"
 #include "value/context.hh"
+#include "input-accessor.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
@@ -100,7 +101,7 @@ class ExternalValueBase
      * Coerce the value to a string. Defaults to uncoercable, i.e. throws an
      * error.
      */
-    virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
+    virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const;
 
     /**
      * Compare to another value of the same type. Defaults to uncomparable,
@@ -112,13 +113,13 @@ class ExternalValueBase
      * Print the value as JSON. Defaults to unconvertable, i.e. throws an error
      */
     virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict,
-        PathSet & context, bool copyToStore = true) const;
+        NixStringContext & context, bool copyToStore = true) const;
 
     /**
      * Print the value as XML. Defaults to unevaluated
      */
     virtual void printValueAsXML(EvalState & state, bool strict, bool location,
-        XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
+        XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen,
         const PosIdx pos) const;
 
     virtual ~ExternalValueBase()
@@ -188,7 +189,7 @@ public:
             const char * * context; // must be in sorted order
         } string;
 
-        const char * path;
+        const char * _path;
         Bindings * attrs;
         struct {
             size_t size;
@@ -268,19 +269,24 @@ public:
 
     void mkString(std::string_view s);
 
-    void mkString(std::string_view s, const PathSet & context);
+    void mkString(std::string_view s, const NixStringContext & context);
 
-    void mkStringMove(const char * s, const PathSet & context);
+    void mkStringMove(const char * s, const NixStringContext & context);
 
-    inline void mkPath(const char * s)
+    inline void mkString(const Symbol & s)
+    {
+        mkString(((const std::string &) s).c_str());
+    }
+
+    void mkPath(const SourcePath & path);
+
+    inline void mkPath(const char * path)
     {
         clearValue();
         internalType = tPath;
-        path = s;
+        _path = path;
     }
 
-    void mkPath(std::string_view s);
-
     inline void mkNull()
     {
         clearValue();
@@ -394,8 +400,6 @@ public:
      */
     bool isTrivial() const;
 
-    NixStringContext getContext(const Store &);
-
     auto listItems()
     {
         struct ListIterable
@@ -423,6 +427,18 @@ public:
         auto begin = listElems();
         return ConstListIterable { begin, begin + listSize() };
     }
+
+    SourcePath path() const
+    {
+        assert(internalType == tPath);
+        return SourcePath{CanonPath(_path)};
+    }
+
+    std::string_view str() const
+    {
+        assert(internalType == tString);
+        return std::string_view(string.s);
+    }
 };
 
 

src/libexpr/value/context.cc

@@ -1,11 +1,10 @@
 #include "value/context.hh"
-#include "store-api.hh"
 
 #include <optional>
 
 namespace nix {
 
-NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0)
+NixStringContextElem NixStringContextElem::parse(std::string_view s0)
 {
     std::string_view s = s0;
 
@@ -25,41 +24,41 @@ NixStringContextElem NixStringContextElem::parse(const Store & store, std::strin
                 "String content element beginning with '!' should have a second '!'");
         }
         return NixStringContextElem::Built {
-            .drvPath = store.parseStorePath(s.substr(index + 1)),
+            .drvPath = StorePath { s.substr(index + 1) },
             .output = std::string(s.substr(0, index)),
         };
     }
     case '=': {
         return NixStringContextElem::DrvDeep {
-            .drvPath = store.parseStorePath(s.substr(1)),
+            .drvPath = StorePath { s.substr(1) },
         };
     }
     default: {
         return NixStringContextElem::Opaque {
-            .path = store.parseStorePath(s),
+            .path = StorePath { s },
         };
     }
     }
 }
 
-std::string NixStringContextElem::to_string(const Store & store) const {
+std::string NixStringContextElem::to_string() const {
     return std::visit(overloaded {
         [&](const NixStringContextElem::Built & b) {
             std::string res;
             res += '!';
             res += b.output;
             res += '!';
-            res += store.printStorePath(b.drvPath);
+            res += b.drvPath.to_string();
             return res;
         },
         [&](const NixStringContextElem::DrvDeep & d) {
             std::string res;
             res += '=';
-            res += store.printStorePath(d.drvPath);
+            res += d.drvPath.to_string();
             return res;
         },
         [&](const NixStringContextElem::Opaque & o) {
-            return store.printStorePath(o.path);
+            return std::string { o.path.to_string() };
         },
     }, raw());
 }

src/libexpr/value/context.hh

@@ -26,8 +26,6 @@ public:
     }
 };
 
-class Store;
-
 /**
  * Plain opaque path to some store object.
  *
@@ -80,12 +78,15 @@ struct NixStringContextElem : _NixStringContextElem_Raw {
     using DrvDeep = NixStringContextElem_DrvDeep;
     using Built = NixStringContextElem_Built;
 
-    inline const Raw & raw() const {
+    inline const Raw & raw() const & {
         return static_cast<const Raw &>(*this);
     }
-    inline Raw & raw() {
+    inline Raw & raw() & {
         return static_cast<Raw &>(*this);
     }
+    inline Raw && raw() && {
+        return static_cast<Raw &&>(*this);
+    }
 
     /**
      * Decode a context string, one of:
@@ -93,10 +94,10 @@ struct NixStringContextElem : _NixStringContextElem_Raw {
      * - ‘=<path>’
      * - ‘!<name>!<path>’
      */
-    static NixStringContextElem parse(const Store & store, std::string_view s);
-    std::string to_string(const Store & store) const;
+    static NixStringContextElem parse(std::string_view s);
+    std::string to_string() const;
 };
 
-typedef std::vector<NixStringContextElem> NixStringContext;
+typedef std::set<NixStringContextElem> NixStringContext;
 
 }

src/libfetchers/git.cc

@@ -62,6 +62,7 @@ std::optional<std::string> readHead(const Path & path)
         .program = "git",
         // FIXME: use 'HEAD' to avoid returning all refs
         .args = {"ls-remote", "--symref", path},
+        .isInteractive = true,
     });
     if (status != 0) return std::nullopt;
 
@@ -350,7 +351,7 @@ struct GitInputScheme : InputScheme
 
         args.push_back(destDir);
 
-        runProgram("git", true, args);
+        runProgram("git", true, args, {}, true);
     }
 
     std::optional<Path> getSourcePath(const Input & input) override
@@ -555,7 +556,7 @@ struct GitInputScheme : InputScheme
                             : ref == "HEAD"
                                 ? *ref
                                 : "refs/heads/" + *ref;
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
+                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@@ -622,7 +623,7 @@ struct GitInputScheme : InputScheme
                 // everything to ensure we get the rev.
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
                 runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
+                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
             }
 
             runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
@@ -649,7 +650,7 @@ struct GitInputScheme : InputScheme
 
             {
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
-                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
+                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
             }
 
             filter = isNotDotGitDirectory;

src/libfetchers/github.cc

@@ -21,7 +21,7 @@ struct DownloadUrl
 };
 
 // A github, gitlab, or sourcehut host
-const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
+const static std::string hostRegexS = "[a-zA-Z0-9.-]*"; // FIXME: check
 std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
 
 struct GitArchiveInputScheme : InputScheme

src/libfetchers/input-accessor.cc

@@ -0,0 +1,106 @@
+#include "input-accessor.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+std::ostream & operator << (std::ostream & str, const SourcePath & path)
+{
+    str << path.to_string();
+    return str;
+}
+
+std::string_view SourcePath::baseName() const
+{
+    return path.baseName().value_or("source");
+}
+
+SourcePath SourcePath::parent() const
+{
+    auto p = path.parent();
+    assert(p);
+    return std::move(*p);
+}
+
+InputAccessor::Stat SourcePath::lstat() const
+{
+    auto st = nix::lstat(path.abs());
+    return InputAccessor::Stat {
+        .type =
+            S_ISREG(st.st_mode) ? InputAccessor::tRegular :
+            S_ISDIR(st.st_mode) ? InputAccessor::tDirectory :
+            S_ISLNK(st.st_mode) ? InputAccessor::tSymlink :
+            InputAccessor::tMisc,
+        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
+    };
+}
+
+std::optional<InputAccessor::Stat> SourcePath::maybeLstat() const
+{
+    // FIXME: merge these into one operation.
+    if (!pathExists())
+        return {};
+    return lstat();
+}
+
+InputAccessor::DirEntries SourcePath::readDirectory() const
+{
+    InputAccessor::DirEntries res;
+    for (auto & entry : nix::readDirectory(path.abs())) {
+        std::optional<InputAccessor::Type> type;
+        switch (entry.type) {
+        case DT_REG: type = InputAccessor::Type::tRegular; break;
+        case DT_LNK: type = InputAccessor::Type::tSymlink; break;
+        case DT_DIR: type = InputAccessor::Type::tDirectory; break;
+        }
+        res.emplace(entry.name, type);
+    }
+    return res;
+}
+
+StorePath SourcePath::fetchToStore(
+    ref<Store> store,
+    std::string_view name,
+    PathFilter * filter,
+    RepairFlag repair) const
+{
+    return
+        settings.readOnlyMode
+        ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first
+        : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair);
+}
+
+SourcePath SourcePath::resolveSymlinks() const
+{
+    SourcePath res(CanonPath::root);
+
+    int linksAllowed = 1024;
+
+    std::list<std::string> todo;
+    for (auto & c : path)
+        todo.push_back(std::string(c));
+
+    while (!todo.empty()) {
+        auto c = *todo.begin();
+        todo.pop_front();
+        if (c == "" || c == ".")
+            ;
+        else if (c == "..")
+            res.path.pop();
+        else {
+            res.path.push(c);
+            if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) {
+                if (!linksAllowed--)
+                    throw Error("infinite symlink recursion in path '%s'", path);
+                auto target = res.readLink();
+                res.path.pop();
+                if (hasPrefix(target, "/"))
+                    res.path = CanonPath::root;
+                todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
+            }
+        }
+    }
+
+    return res;
+}
+
+}

src/libfetchers/input-accessor.hh

@@ -0,0 +1,167 @@
+#pragma once
+
+#include "ref.hh"
+#include "types.hh"
+#include "archive.hh"
+#include "canon-path.hh"
+#include "repair-flag.hh"
+
+namespace nix {
+
+class StorePath;
+class Store;
+
+struct InputAccessor
+{
+    enum Type {
+      tRegular, tSymlink, tDirectory,
+      /**
+        Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things.
+
+        Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`.
+
+        Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types.
+      */
+      tMisc
+    };
+
+    struct Stat
+    {
+        Type type = tMisc;
+        //uint64_t fileSize = 0; // regular files only
+        bool isExecutable = false; // regular files only
+    };
+
+    typedef std::optional<Type> DirEntry;
+
+    typedef std::map<std::string, DirEntry> DirEntries;
+};
+
+/**
+ * An abstraction for accessing source files during
+ * evaluation. Currently, it's just a wrapper around `CanonPath` that
+ * accesses files in the regular filesystem, but in the future it will
+ * support fetching files in other ways.
+ */
+struct SourcePath
+{
+    CanonPath path;
+
+    SourcePath(CanonPath path)
+        : path(std::move(path))
+    { }
+
+    std::string_view baseName() const;
+
+    /**
+     * Construct the parent of this `SourcePath`. Aborts if `this`
+     * denotes the root.
+     */
+    SourcePath parent() const;
+
+    /**
+     * If this `SourcePath` denotes a regular file (not a symlink),
+     * return its contents; otherwise throw an error.
+     */
+    std::string readFile() const
+    { return nix::readFile(path.abs()); }
+
+    /**
+     * Return whether this `SourcePath` denotes a file (of any type)
+     * that exists
+    */
+    bool pathExists() const
+    { return nix::pathExists(path.abs()); }
+
+    /**
+     * Return stats about this `SourcePath`, or throw an exception if
+     * it doesn't exist.
+     */
+    InputAccessor::Stat lstat() const;
+
+    /**
+     * Return stats about this `SourcePath`, or std::nullopt if it
+     * doesn't exist.
+     */
+    std::optional<InputAccessor::Stat> maybeLstat() const;
+
+    /**
+     * If this `SourcePath` denotes a directory (not a symlink),
+     * return its directory entries; otherwise throw an error.
+     */
+    InputAccessor::DirEntries readDirectory() const;
+
+    /**
+     * If this `SourcePath` denotes a symlink, return its target;
+     * otherwise throw an error.
+     */
+    std::string readLink() const
+    { return nix::readLink(path.abs()); }
+
+    /**
+     * Dump this `SourcePath` to `sink` as a NAR archive.
+     */
+    void dumpPath(
+        Sink & sink,
+        PathFilter & filter = defaultPathFilter) const
+    { return nix::dumpPath(path.abs(), sink, filter); }
+
+    /**
+     * Copy this `SourcePath` to the Nix store.
+     */
+    StorePath fetchToStore(
+        ref<Store> store,
+        std::string_view name = "source",
+        PathFilter * filter = nullptr,
+        RepairFlag repair = NoRepair) const;
+
+    /**
+     * Return the location of this path in the "real" filesystem, if
+     * it has a physical location.
+     */
+    std::optional<CanonPath> getPhysicalPath() const
+    { return path; }
+
+    std::string to_string() const
+    { return path.abs(); }
+
+    /**
+     * Append a `CanonPath` to this path.
+     */
+    SourcePath operator + (const CanonPath & x) const
+    { return {path + x}; }
+
+    /**
+     * Append a single component `c` to this path. `c` must not
+     * contain a slash. A slash is implicitly added between this path
+     * and `c`.
+     */
+    SourcePath operator + (std::string_view c) const
+    {  return {path + c}; }
+
+    bool operator == (const SourcePath & x) const
+    {
+        return path == x.path;
+    }
+
+    bool operator != (const SourcePath & x) const
+    {
+        return path != x.path;
+    }
+
+    bool operator < (const SourcePath & x) const
+    {
+        return path < x.path;
+    }
+
+    /**
+     * Resolve any symlinks in this `SourcePath` (including its
+     * parents). The result is a `SourcePath` in which no element is a
+     * symlink.
+     */
+    SourcePath resolveSymlinks() const;
+};
+
+std::ostream & operator << (std::ostream & str, const SourcePath & path);
+
+}

src/libmain/common-args.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "args.hh"
+#include "repair-flag.hh"
 
 namespace nix {
 
@@ -49,4 +50,21 @@ struct MixJSON : virtual Args
     }
 };
 
+struct MixRepair : virtual Args
+{
+    RepairFlag repair = NoRepair;
+
+    MixRepair()
+    {
+        addFlag({
+            .longName = "repair",
+            .description =
+                "During evaluation, rewrite missing or corrupted files in the Nix store. "
+                "During building, rebuild missing or corrupted store paths.",
+            .category = miscCategory,
+            .handler = {&repair, Repair},
+        });
+    }
+};
+
 }

src/libstore/build/derivation-goal.cc

@@ -274,11 +274,13 @@ void DerivationGoal::haveDerivation()
                         )
                     )
                 );
-            else
+            else {
+                auto * cap = getDerivationCA(*drv);
                 addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
                     status.known->path,
                     buildMode == bmRepair ? Repair : NoRepair,
-                    getDerivationCA(*drv))));
+                    cap ? std::optional { *cap } : std::nullopt)));
+            }
         }
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -1020,43 +1022,33 @@ void DerivationGoal::resolvedFinished()
 
         StorePathSet outputPaths;
 
-        // `wantedOutputs` might merely indicate “all the outputs”
-        auto realWantedOutputs = std::visit(overloaded {
-            [&](const OutputsSpec::All &) {
-                return resolvedDrv.outputNames();
-            },
-            [&](const OutputsSpec::Names & names) {
-                return static_cast<std::set<std::string>>(names);
-            },
-        }, wantedOutputs.raw());
-
-        for (auto & wantedOutput : realWantedOutputs) {
-            auto initialOutput = get(initialOutputs, wantedOutput);
-            auto resolvedHash = get(resolvedHashes, wantedOutput);
+        for (auto & outputName : resolvedDrv.outputNames()) {
+            auto initialOutput = get(initialOutputs, outputName);
+            auto resolvedHash = get(resolvedHashes, outputName);
             if ((!initialOutput) || (!resolvedHash))
                 throw Error(
                     "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
-                    worker.store.printStorePath(drvPath), wantedOutput);
+                    worker.store.printStorePath(drvPath), outputName);
 
             auto realisation = [&]{
-              auto take1 = get(resolvedResult.builtOutputs, wantedOutput);
+              auto take1 = get(resolvedResult.builtOutputs, outputName);
               if (take1) return *take1;
 
               /* The above `get` should work. But sateful tracking of
                  outputs in resolvedResult, this can get out of sync with the
                  store, which is our actual source of truth. For now we just
                  check the store directly if it fails. */
-              auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput });
+              auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName });
               if (take2) return *take2;
 
               throw Error(
                   "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
-                  worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
+                  worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
             }();
 
             if (drv->type().isPure()) {
                 auto newRealisation = realisation;
-                newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
+                newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
                 newRealisation.signatures.clear();
                 if (!drv->type().isFixed())
                     newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
@@ -1064,7 +1056,7 @@ void DerivationGoal::resolvedFinished()
                 worker.store.registerDrvOutput(newRealisation);
             }
             outputPaths.insert(realisation.outPath);
-            builtOutputs.emplace(wantedOutput, realisation);
+            builtOutputs.emplace(outputName, realisation);
         }
 
         runPostBuildHook(
@@ -1160,7 +1152,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    worker_proto::write(worker.store, hook->sink, inputPaths);
+    workerProtoWrite(worker.store, hook->sink, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1171,7 +1163,7 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
         }
-        worker_proto::write(worker.store, hook->sink, missingOutputs);
+        workerProtoWrite(worker.store, hook->sink, missingOutputs);
     }
 
     hook->sink = FdSink();
@@ -1406,7 +1398,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
                 );
             }
         }
-        if (info.wanted && info.known && info.known->isValid())
+        if (info.known && info.known->isValid())
             validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
     }
 
@@ -1457,8 +1449,9 @@ void DerivationGoal::done(
     mcRunningBuilds.reset();
 
     if (buildResult.success()) {
-        assert(!builtOutputs.empty());
-        buildResult.builtOutputs = std::move(builtOutputs);
+        auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs));
+        assert(!wantedBuiltOutputs.empty());
+        buildResult.builtOutputs = std::move(wantedBuiltOutputs);
         if (status == BuildResult::Built)
             worker.doneBuilds++;
     } else {

src/libstore/build/derivation-goal.hh

@@ -306,15 +306,13 @@ struct DerivationGoal : public Goal
      * Update 'initialOutputs' to determine the current status of the
      * outputs of the derivation. Also returns a Boolean denoting
      * whether all outputs are valid and non-corrupt, and a
-     * 'SingleDrvOutputs' structure containing the valid and wanted
-     * outputs.
+     * 'SingleDrvOutputs' structure containing the valid outputs.
      */
     std::pair<bool, SingleDrvOutputs> checkPathValidity();
 
     /**
      * Aborts if any output is not valid or corrupt, and otherwise
-     * returns a 'SingleDrvOutputs' structure containing the wanted
-     * outputs.
+     * returns a 'SingleDrvOutputs' structure containing all outputs.
      */
     SingleDrvOutputs assertPathValidity();
 
@@ -335,6 +333,8 @@ struct DerivationGoal : public Goal
     void waiteeDone(GoalPtr waitee, ExitCode result) override;
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
+
+    JobCategory jobCategory() override { return JobCategory::Build; };
 };
 
 MakeError(NotDeterministic, BuildError);

src/libstore/build/drv-output-substitution-goal.hh

@@ -21,7 +21,7 @@ class Worker;
 class DrvOutputSubstitutionGoal : public Goal {
 
     /**
-     * The drv output we're trying to substitue
+     * The drv output we're trying to substitute
      */
     DrvOutput id;
 
@@ -72,6 +72,8 @@ public:
 
     void work() override;
     void handleEOF(int fd) override;
+
+    JobCategory jobCategory() override { return JobCategory::Substitution; };
 };
 
 }

src/libstore/build/entry-points.cc

@@ -110,7 +110,7 @@ void Store::ensurePath(const StorePath & path)
 }
 
 
-void LocalStore::repairPath(const StorePath & path)
+void Store::repairPath(const StorePath & path)
 {
     Worker worker(*this, *this);
     GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);

src/libstore/build/goal.hh

@@ -34,6 +34,17 @@ typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
  */
 typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
 
+/**
+ * Used as a hint to the worker on how to schedule a particular goal. For example,
+ * builds are typically CPU- and memory-bound, while substitutions are I/O bound.
+ * Using this information, the worker might decide to schedule more or fewer goals
+ * of each category in parallel.
+ */
+enum struct JobCategory {
+    Build,
+    Substitution,
+};
+
 struct Goal : public std::enable_shared_from_this<Goal>
 {
     typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
@@ -150,6 +161,8 @@ public:
     void amDone(ExitCode result, std::optional<Error> ex = {});
 
     virtual void cleanup() { }
+
+    virtual JobCategory jobCategory() = 0;
 };
 
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);

src/libstore/build/local-derivation-goal.cc

@@ -357,7 +357,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
         for (auto & [_, status] : initialOutputs) {
             if (!status.known) continue;
             if (buildMode != bmCheck && status.known->isValid()) continue;
-            auto p = worker.store.printStorePath(status.known->path);
+            auto p = worker.store.toRealPath(status.known->path);
             if (pathExists(chrootRootDir + p))
                 renameFile((chrootRootDir + p), p);
         }
@@ -1791,6 +1791,9 @@ void LocalDerivationGoal::runChild()
                 for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
                     if (pathExists(path))
                         ss.push_back(path);
+
+                if (settings.caFile != "")
+                    dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
             }
 
             for (auto & i : ss) dirsInChroot.emplace(i, i);
@@ -2441,37 +2444,51 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 throw BuildError(
                     "output path %1% without valid stats info",
                     actualPath);
-            if (outputHash.method == FileIngestionMethod::Flat) {
+            if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
+                outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
+            {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
                     throw BuildError(
                         "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
+                        "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
                         actualPath);
             }
             rewriteOutput();
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
             HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
+            std::visit(overloaded {
+                [&](const TextIngestionMethod &) {
+                    readFile(actualPath, caSink);
+                },
+                [&](const FileIngestionMethod & m2) {
+                    switch (m2) {
+                    case FileIngestionMethod::Recursive:
+                        dumpPath(actualPath, caSink);
+                        break;
+                    case FileIngestionMethod::Flat:
+                        readFile(actualPath, caSink);
+                        break;
+                    }
+                },
+            }, outputHash.method.raw);
             auto got = caSink.finish().first;
+
+            auto optCA = ContentAddressWithReferences::fromPartsOpt(
+                outputHash.method,
+                std::move(got),
+                rewriteRefs());
+            if (!optCA) {
+                // TODO track distinct failure modes separately (at the time of
+                // writing there is just one but `nullopt` is unclear) so this
+                // message can't get out of sync.
+                throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
+            }
             ValidPathInfo newInfo0 {
                 worker.store,
                 outputPathName(drv->name, outputName),
-                FixedOutputInfo {
-                    .hash = {
-                        .method = outputHash.method,
-                        .hash = got,
-                    },
-                    .references = rewriteRefs(),
-                },
+                *std::move(optCA),
                 Hash::dummy,
             };
             if (*scratchPath != newInfo0.path) {
@@ -2518,13 +2535,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             },
 
             [&](const DerivationOutput::CAFixed & dof) {
+                auto wanted = dof.ca.getHash();
+
                 auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
+                    .method = dof.ca.getMethod(),
+                    .hashType = wanted.type,
                 });
 
                 /* Check wanted hash */
-                const Hash & wanted = dof.hash.hash;
                 assert(newInfo0.ca);
                 auto got = newInfo0.ca->getHash();
                 if (wanted != got) {
@@ -2537,6 +2555,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                             wanted.to_string(SRI, true),
                             got.to_string(SRI, true)));
                 }
+                if (!newInfo0.references.empty())
+                    delayedException = std::make_exception_ptr(
+                        BuildError("illegal path references in fixed-output derivation '%s'",
+                            worker.store.printStorePath(drvPath)));
+
                 return newInfo0;
             },
 
@@ -2716,8 +2739,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             signRealisation(thisRealisation);
             worker.store.registerDrvOutput(thisRealisation);
         }
-        if (wantedOutputs.contains(outputName))
-            builtOutputs.emplace(outputName, thisRealisation);
+        builtOutputs.emplace(outputName, thisRealisation);
     }
 
     return builtOutputs;

src/libstore/build/personality.cc

@@ -21,7 +21,8 @@ void setPersonality(std::string_view system)
                 && (std::string_view(SYSTEM) == "x86_64-linux"
                     || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
             || system == "armv7l-linux"
-            || system == "armv6l-linux")
+            || system == "armv6l-linux"
+            || system == "armv5tel-linux")
         {
             if (personality(PER_LINUX32) == -1)
                 throw SysError("cannot set 32-bit personality");

src/libstore/build/substitution-goal.cc

@@ -200,11 +200,10 @@ void PathSubstitutionGoal::tryToRun()
 {
     trace("trying to run");
 
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
+    /* Make sure that we are allowed to start a substitution.  Note that even
+       if maxSubstitutionJobs == 0, we still allow a substituter to run. This
+       prevents infinite waiting. */
+    if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
         worker.waitForBuildSlot(shared_from_this());
         return;
     }

src/libstore/build/substitution-goal.hh

@@ -115,6 +115,8 @@ public:
     void handleEOF(int fd) override;
 
     void cleanup() override;
+
+    JobCategory jobCategory() override { return JobCategory::Substitution; };
 };
 
 }

src/libstore/build/worker.cc

@@ -18,6 +18,7 @@ Worker::Worker(Store & store, Store & evalStore)
 {
     /* Debugging: prevent recursive workers. */
     nrLocalBuilds = 0;
+    nrSubstitutions = 0;
     lastWokenUp = steady_time_point::min();
     permanentFailure = false;
     timedOut = false;
@@ -176,6 +177,12 @@ unsigned Worker::getNrLocalBuilds()
 }
 
 
+unsigned Worker::getNrSubstitutions()
+{
+    return nrSubstitutions;
+}
+
+
 void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     bool inBuildSlot, bool respectTimeouts)
 {
@@ -187,7 +194,10 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     child.inBuildSlot = inBuildSlot;
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
+    if (inBuildSlot) {
+        if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
+        else nrLocalBuilds++;
+    }
 }
 
 
@@ -198,8 +208,13 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
     if (i == children.end()) return;
 
     if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
+        if (goal->jobCategory() == JobCategory::Substitution) {
+            assert(nrSubstitutions > 0);
+            nrSubstitutions--;
+        } else {
+            assert(nrLocalBuilds > 0);
+            nrLocalBuilds--;
+        }
     }
 
     children.erase(i);
@@ -220,7 +235,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
 void Worker::waitForBuildSlot(GoalPtr goal)
 {
     debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
+    bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
+    if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
+        (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))
         wakeUp(goal); /* we can do it right away */
     else
         addToWeakGoals(wantingToBuild, goal);

src/libstore/build/worker.hh

@@ -88,11 +88,16 @@ private:
     std::list<Child> children;
 
     /**
-     * Number of build slots occupied.  This includes local builds and
-     * substitutions but not remote builds via the build hook.
+     * Number of build slots occupied.  This includes local builds but does not
+     * include substitutions or remote builds via the build hook.
      */
     unsigned int nrLocalBuilds;
 
+    /**
+     * Number of substitution slots occupied.
+     */
+    unsigned int nrSubstitutions;
+
     /**
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
@@ -220,12 +225,16 @@ public:
     void wakeUp(GoalPtr goal);
 
     /**
-     * Return the number of local build and substitution processes
-     * currently running (but not remote builds via the build
-     * hook).
+     * Return the number of local build processes currently running (but not
+     * remote builds via the build hook).
      */
     unsigned int getNrLocalBuilds();
 
+    /**
+     * Return the number of substitution processes currently running.
+     */
+    unsigned int getNrSubstitutions();
+
     /**
      * Registers a running child process.  `inBuildSlot` means that
      * the process counts towards the jobs limit.

src/libstore/content-address.cc

@@ -21,6 +21,27 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
     }
 }
 
+std::string ContentAddressMethod::renderPrefix() const
+{
+    return std::visit(overloaded {
+        [](TextIngestionMethod) -> std::string { return "text:"; },
+        [](FileIngestionMethod m2) {
+             /* Not prefixed for back compat with things that couldn't produce text before. */
+            return makeFileIngestionPrefix(m2);
+        },
+    }, raw);
+}
+
+ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
+{
+    ContentAddressMethod method = FileIngestionMethod::Flat;
+    if (splitPrefix(m, "r:"))
+        method = FileIngestionMethod::Recursive;
+    else if (splitPrefix(m, "text:"))
+        method = TextIngestionMethod {};
+    return method;
+}
+
 std::string ContentAddress::render() const
 {
     return std::visit(overloaded {
@@ -36,14 +57,14 @@ std::string ContentAddress::render() const
     }, raw);
 }
 
-std::string ContentAddressMethod::render() const
+std::string ContentAddressMethod::render(HashType ht) const
 {
     return std::visit(overloaded {
-        [](const TextHashMethod & th) {
-            return std::string{"text:"} + printHashType(htSHA256);
+        [&](const TextIngestionMethod & th) {
+            return std::string{"text:"} + printHashType(ht);
         },
-        [](const FixedOutputHashMethod & fshm) {
-            return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
+        [&](const FileIngestionMethod & fim) {
+            return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
         }
     }, raw);
 }
@@ -51,7 +72,7 @@ std::string ContentAddressMethod::render() const
 /**
  * Parses content address strings up to the hash.
  */
-static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
+static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
 {
     std::string_view wholeInput { rest };
 
@@ -75,46 +96,47 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
     if (prefix == "text") {
         // No parsing of the ingestion method, "text" only support flat.
         HashType hashType = parseHashType_();
-        if (hashType != htSHA256)
-            throw Error("text content address hash should use %s, but instead uses %s",
-                printHashType(htSHA256), printHashType(hashType));
-        return TextHashMethod {};
+        return {
+            TextIngestionMethod {},
+            std::move(hashType),
+        };
     } else if (prefix == "fixed") {
         // Parse method
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
         HashType hashType = parseHashType_();
-        return FixedOutputHashMethod {
-            .fileIngestionMethod = method,
-            .hashType = std::move(hashType),
+        return {
+            std::move(method),
+            std::move(hashType),
         };
     } else
         throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
 }
 
-ContentAddress ContentAddress::parse(std::string_view rawCa) {
+ContentAddress ContentAddress::parse(std::string_view rawCa)
+{
     auto rest = rawCa;
 
-    ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
+    auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest);
+    auto hashType = hashType_; // work around clang bug
 
-    return std::visit(
-        overloaded {
-            [&](TextHashMethod & thm) {
-                return ContentAddress(TextHash {
-                    .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
-                });
-            },
-            [&](FixedOutputHashMethod & fohMethod) {
-                return ContentAddress(FixedOutputHash {
-                    .method = fohMethod.fileIngestionMethod,
-                    .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
-                });
-            },
-        }, caMethod.raw);
+    return std::visit(overloaded {
+        [&](TextIngestionMethod &) {
+            return ContentAddress(TextHash {
+                .hash = Hash::parseNonSRIUnprefixed(rest, hashType)
+            });
+        },
+        [&](FileIngestionMethod & fim) {
+            return ContentAddress(FixedOutputHash {
+                .method = fim,
+                .hash = Hash::parseNonSRIUnprefixed(rest, hashType),
+            });
+        },
+    }, caMethod.raw);
 }
 
-ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod)
+std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
 {
     std::string asPrefix = std::string{caMethod} + ":";
     // parseContentAddressMethodPrefix takes its argument by reference
@@ -134,6 +156,36 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
     return ca ? ca->render() : "";
 }
 
+ContentAddress ContentAddress::fromParts(
+    ContentAddressMethod method, Hash hash) noexcept
+{
+    return std::visit(overloaded {
+        [&](TextIngestionMethod _) -> ContentAddress {
+            return TextHash {
+                .hash = std::move(hash),
+            };
+        },
+        [&](FileIngestionMethod m2) -> ContentAddress {
+            return FixedOutputHash {
+                .method = std::move(m2),
+                .hash = std::move(hash),
+            };
+        },
+    }, method.raw);
+}
+
+ContentAddressMethod ContentAddress::getMethod() const
+{
+    return std::visit(overloaded {
+        [](const TextHash & th) -> ContentAddressMethod {
+            return TextIngestionMethod {};
+        },
+        [](const FixedOutputHash & fsh) -> ContentAddressMethod {
+            return fsh.method;
+        },
+    }, raw);
+}
+
 const Hash & ContentAddress::getHash() const
 {
     return std::visit(overloaded {
@@ -146,6 +198,12 @@ const Hash & ContentAddress::getHash() const
     }, raw);
 }
 
+std::string ContentAddress::printMethodAlgo() const
+{
+    return getMethod().renderPrefix()
+        + printHashType(getHash().type);
+}
+
 bool StoreReferences::empty() const
 {
     return !self && others.empty();
@@ -156,7 +214,8 @@ size_t StoreReferences::size() const
     return (self ? 1 : 0) + others.size();
 }
 
-ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) {
+ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept
+{
     return std::visit(overloaded {
         [&](const TextHash & h) -> ContentAddressWithReferences {
             return TextInfo {
@@ -173,4 +232,56 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
     }, ca.raw);
 }
 
+std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
+    ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept
+{
+    return std::visit(overloaded {
+        [&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> {
+            if (refs.self)
+                return std::nullopt;
+            return ContentAddressWithReferences {
+                TextInfo {
+                    .hash = { .hash = std::move(hash) },
+                    .references = std::move(refs.others),
+                }
+            };
+        },
+        [&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
+            return ContentAddressWithReferences {
+                FixedOutputInfo {
+                    .hash = {
+                        .method = m2,
+                        .hash = std::move(hash),
+                    },
+                    .references = std::move(refs),
+                }
+            };
+        },
+    }, method.raw);
+}
+
+ContentAddressMethod ContentAddressWithReferences::getMethod() const
+{
+    return std::visit(overloaded {
+        [](const TextInfo & th) -> ContentAddressMethod {
+            return TextIngestionMethod {};
+        },
+        [](const FixedOutputInfo & fsh) -> ContentAddressMethod {
+            return fsh.hash.method;
+        },
+    }, raw);
+}
+
+Hash ContentAddressWithReferences::getHash() const
+{
+    return std::visit(overloaded {
+        [](const TextInfo & th) {
+            return th.hash.hash;
+        },
+        [](const FixedOutputInfo & fsh) {
+            return fsh.hash.hash;
+        },
+    }, raw);
+}
+
 }

src/libstore/content-address.hh

@@ -21,8 +21,14 @@ namespace nix {
  *
  * Somewhat obscure, used by \ref Derivation derivations and
  * `builtins.toFile` currently.
+ *
+ * TextIngestionMethod is identical to FileIngestionMethod::Fixed except that
+ * the former may not have self-references and is tagged `text:${algo}:${hash}`
+ * rather than `fixed:${algo}:${hash}`.  The contents of the store path are
+ * ingested and hashed identically, aside from the slightly different tag and
+ * restriction on self-references.
  */
-struct TextHashMethod : std::monostate { };
+struct TextIngestionMethod : std::monostate { };
 
 /**
  * An enumeration of the main ways we can serialize file system
@@ -46,13 +52,6 @@ enum struct FileIngestionMethod : uint8_t {
  */
 std::string makeFileIngestionPrefix(FileIngestionMethod m);
 
-struct FixedOutputHashMethod {
-    FileIngestionMethod fileIngestionMethod;
-    HashType hashType;
-
-    GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType);
-};
-
 /**
  * An enumeration of all the ways we can serialize file system objects.
  *
@@ -64,8 +63,8 @@ struct FixedOutputHashMethod {
 struct ContentAddressMethod
 {
     typedef std::variant<
-        TextHashMethod,
-        FixedOutputHashMethod
+        TextIngestionMethod,
+        FileIngestionMethod
     > Raw;
 
     Raw raw;
@@ -77,9 +76,36 @@ struct ContentAddressMethod
         : raw(std::forward<decltype(arg)>(arg)...)
     { }
 
-    static ContentAddressMethod parse(std::string_view rawCaMethod);
 
-    std::string render() const;
+    /**
+     * Parse the prefix tag which indicates how the files
+     * were ingested, with the fixed output case not prefixed for back
+     * compat.
+     *
+     * @param [in] m A string that should begin with the prefix.
+     * @param [out] m The remainder of the string after the prefix.
+     */
+    static ContentAddressMethod parsePrefix(std::string_view & m);
+
+    /**
+     * Render the prefix tag which indicates how the files wre ingested.
+     *
+     * The rough inverse of `parsePrefix()`.
+     */
+    std::string renderPrefix() const;
+
+    /**
+     * Parse a content addressing method and hash type.
+     */
+    static std::pair<ContentAddressMethod, HashType> parse(std::string_view rawCaMethod);
+
+    /**
+     * Render a content addressing method and hash type in a
+     * nicer way, prefixing both cases.
+     *
+     * The rough inverse of `parse()`.
+     */
+    std::string render(HashType ht) const;
 };
 
 
@@ -147,8 +173,9 @@ struct ContentAddress
     { }
 
     /**
-     * Compute the content-addressability assertion (ValidPathInfo::ca) for
-     * paths created by Store::makeFixedOutputPath() / Store::addToStore().
+     * Compute the content-addressability assertion
+     * (`ValidPathInfo::ca`) for paths created by
+     * `Store::makeFixedOutputPath()` / `Store::addToStore()`.
      */
     std::string render() const;
 
@@ -156,9 +183,27 @@ struct ContentAddress
 
     static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
 
+    /**
+     * Create a `ContentAddress` from 2 parts:
+     *
+     * @param method Way ingesting the file system data.
+     *
+     * @param hash Hash of ingested file system data.
+     */
+    static ContentAddress fromParts(
+        ContentAddressMethod method, Hash hash) noexcept;
+
+    ContentAddressMethod getMethod() const;
+
     const Hash & getHash() const;
+
+    std::string printMethodAlgo() const;
 };
 
+/**
+ * Render the `ContentAddress` if it exists to a string, return empty
+ * string otherwise.
+ */
 std::string renderContentAddress(std::optional<ContentAddress> ca);
 
 
@@ -244,10 +289,29 @@ struct ContentAddressWithReferences
     { }
 
     /**
-     * Create a ContentAddressWithReferences from a mere ContentAddress, by
-     * assuming no references in all cases.
+     * Create a `ContentAddressWithReferences` from a mere
+     * `ContentAddress`, by claiming no references.
      */
-    static ContentAddressWithReferences withoutRefs(const ContentAddress &);
+    static ContentAddressWithReferences withoutRefs(const ContentAddress &) noexcept;
+
+    /**
+     * Create a `ContentAddressWithReferences` from 3 parts:
+     *
+     * @param method Way ingesting the file system data.
+     *
+     * @param hash Hash of ingested file system data.
+     *
+     * @param refs References to other store objects or oneself.
+     *
+     * Do note that not all combinations are supported; `nullopt` is
+     * returns for invalid combinations.
+     */
+    static std::optional<ContentAddressWithReferences> fromPartsOpt(
+        ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept;
+
+    ContentAddressMethod getMethod() const;
+
+    Hash getHash() const;
 };
 
 }

src/libstore/daemon.cc

@@ -263,7 +263,7 @@ static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int cli
 {
     std::vector<DerivedPath> reqs;
     if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
-        reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
+        reqs = WorkerProto<std::vector<DerivedPath>>::read(store, from);
     } else {
         for (auto & s : readStrings<Strings>(from))
             reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
@@ -287,7 +287,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryValidPaths: {
-        auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto paths = WorkerProto<StorePathSet>::read(*store, from);
 
         SubstituteFlag substitute = NoSubstitute;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
@@ -300,7 +300,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         }
         auto res = store->queryValidPaths(paths, substitute);
         logger->stopWork();
-        worker_proto::write(*store, to, res);
+        workerProtoWrite(*store, to, res);
         break;
     }
 
@@ -316,11 +316,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePaths: {
-        auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto paths = WorkerProto<StorePathSet>::read(*store, from);
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        worker_proto::write(*store, to, res);
+        workerProtoWrite(*store, to, res);
         break;
     }
 
@@ -349,7 +349,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        worker_proto::write(*store, to, paths);
+        workerProtoWrite(*store, to, paths);
         break;
     }
 
@@ -367,7 +367,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
-        worker_proto::write(*store, to, outputs);
+        workerProtoWrite(*store, to, outputs);
         break;
     }
 
@@ -393,7 +393,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
             auto name = readString(from);
             auto camStr = readString(from);
-            auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            auto refs = WorkerProto<StorePathSet>::read(*store, from);
             bool repairBool;
             from >> repairBool;
             auto repair = RepairFlag{repairBool};
@@ -401,18 +401,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             logger->startWork();
             auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
-                ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr);
+                auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr);
+                auto hashType = hashType_; // work around clang bug
                 FramedSource source(from);
                 // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
                 return std::visit(overloaded {
-                    [&](const TextHashMethod &) {
+                    [&](const TextIngestionMethod &) {
+                        if (hashType != htSHA256)
+                            throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
+                                name, printHashType(hashType));
                         // We could stream this by changing Store
                         std::string contents = source.drain();
                         auto path = store->addTextToStore(name, contents, refs, repair);
                         return store->queryPathInfo(path);
                     },
-                    [&](const FixedOutputHashMethod & fohm) {
-                        auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
+                    [&](const FileIngestionMethod & fim) {
+                        auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
                         return store->queryPathInfo(path);
                     },
                 }, contentAddressMethod.raw);
@@ -491,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopAddTextToStore: {
         std::string suffix = readString(from);
         std::string s = readString(from);
-        auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        auto refs = WorkerProto<StorePathSet>::read(*store, from);
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -563,7 +567,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto results = store->buildPathsWithResults(drvs, mode);
         logger->stopWork();
 
-        worker_proto::write(*store, to, results);
+        workerProtoWrite(*store, to, results);
 
         break;
     }
@@ -640,7 +644,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             DrvOutputs builtOutputs;
             for (auto & [output, realisation] : res.builtOutputs)
                 builtOutputs.insert_or_assign(realisation.id, realisation);
-            worker_proto::write(*store, to, builtOutputs);
+            workerProtoWrite(*store, to, builtOutputs);
         }
         break;
     }
@@ -705,7 +709,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopCollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -775,7 +779,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            worker_proto::write(*store, to, i->second.references);
+            workerProtoWrite(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -786,11 +790,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         SubstitutablePathInfos infos;
         StorePathCAMap pathsMap = {};
         if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
-            auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            auto paths = WorkerProto<StorePathSet>::read(*store, from);
             for (auto & path : paths)
                 pathsMap.emplace(path, std::nullopt);
         } else
-            pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
+            pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
         logger->startWork();
         store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
@@ -798,7 +802,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            worker_proto::write(*store, to, i.second.references);
+            workerProtoWrite(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -808,7 +812,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        worker_proto::write(*store, to, paths);
+        workerProtoWrite(*store, to, paths);
         break;
     }
 
@@ -880,7 +884,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+        info.references = WorkerProto<StorePathSet>::read(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = ContentAddress::parseOpt(readString(from));
@@ -931,9 +935,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        worker_proto::write(*store, to, willBuild);
-        worker_proto::write(*store, to, willSubstitute);
-        worker_proto::write(*store, to, unknown);
+        workerProtoWrite(*store, to, willBuild);
+        workerProtoWrite(*store, to, willSubstitute);
+        workerProtoWrite(*store, to, unknown);
         to << downloadSize << narSize;
         break;
     }
@@ -946,7 +950,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             store->registerDrvOutput(Realisation{
                 .id = outputId, .outPath = outputPath});
         } else {
-            auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
+            auto realisation = WorkerProto<Realisation>::read(*store, from);
             store->registerDrvOutput(realisation);
         }
         logger->stopWork();
@@ -961,11 +965,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
             std::set<StorePath> outPaths;
             if (info) outPaths.insert(info->outPath);
-            worker_proto::write(*store, to, outPaths);
+            workerProtoWrite(*store, to, outPaths);
         } else {
             std::set<Realisation> realisations;
             if (info) realisations.insert(*info);
-            worker_proto::write(*store, to, realisations);
+            workerProtoWrite(*store, to, realisations);
         }
         break;
     }
@@ -1041,7 +1045,7 @@ void processConnection(
         auto temp = trusted
             ? store->isTrustedClient()
             : std::optional { NotTrusted };
-        worker_proto::write(*store, to, temp);
+        workerProtoWrite(*store, to, temp);
     }
 
     /* Send startup error messages to the client. */

src/libstore/derivations.cc

@@ -1,7 +1,9 @@
 #include "derivations.hh"
+#include "downstream-placeholder.hh"
 #include "store-api.hh"
 #include "globals.hh"
 #include "util.hh"
+#include "split.hh"
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
 #include <boost/container/small_vector.hpp>
@@ -35,9 +37,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
 
 StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
 {
-    return store.makeFixedOutputPath(
+    return store.makeFixedOutputPathFromCA(
         outputPathName(drvName, outputName),
-        { hash, {} });
+        ContentAddressWithReferences::withoutRefs(ca));
 }
 
 
@@ -211,29 +213,27 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 
 static DerivationOutput parseDerivationOutput(const Store & store,
-    std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
+    std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
 {
     if (hashAlgo != "") {
-        auto method = FileIngestionMethod::Flat;
-        if (hashAlgo.substr(0, 2) == "r:") {
-            method = FileIngestionMethod::Recursive;
-            hashAlgo = hashAlgo.substr(2);
-        }
+        ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
+        if (method == TextIngestionMethod {})
+            experimentalFeatureSettings.require(Xp::DynamicDerivations);
         const auto hashType = parseHashType(hashAlgo);
-        if (hash == "impure") {
+        if (hashS == "impure") {
             experimentalFeatureSettings.require(Xp::ImpureDerivations);
             assert(pathS == "");
             return DerivationOutput::Impure {
                 .method = std::move(method),
                 .hashType = std::move(hashType),
             };
-        } else if (hash != "") {
+        } else if (hashS != "") {
             validatePath(pathS);
+            auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
             return DerivationOutput::CAFixed {
-                .hash = FixedOutputHash {
-                    .method = std::move(method),
-                    .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
-                },
+                .ca = ContentAddress::fromParts(
+                    std::move(method),
+                    std::move(hash)),
             };
         } else {
             experimentalFeatureSettings.require(Xp::CaDerivations);
@@ -393,12 +393,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
             },
             [&](const DerivationOutput::CAFixed & dof) {
                 s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
-                s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
-                s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
+                s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
+                s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false));
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
+                s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType));
                 s += ','; printUnquotedString(s, "");
             },
             [&](const DerivationOutput::Deferred &) {
@@ -409,7 +409,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
             [&](const DerivationOutputImpure & doi) {
                 // FIXME
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
+                s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
                 s += ','; printUnquotedString(s, "impure");
             }
         }, i.second.raw());
@@ -626,8 +626,8 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         for (const auto & i : drv.outputs) {
             auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
             auto hash = hashString(htSHA256, "fixed:out:"
-                + dof.hash.printMethodAlgo() + ":"
-                + dof.hash.hash.to_string(Base16, false) + ":"
+                + dof.ca.printMethodAlgo() + ":"
+                + dof.ca.getHash().to_string(Base16, false) + ":"
                 + store.printStorePath(dof.path(store, drv.name, i.first)));
             outputHashes.insert_or_assign(i.first, std::move(hash));
         }
@@ -749,7 +749,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
+    drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -777,12 +777,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
             [&](const DerivationOutput::CAFixed & dof) {
                 out << store.printStorePath(dof.path(store, drv.name, i.first))
-                    << dof.hash.printMethodAlgo()
-                    << dof.hash.hash.to_string(Base16, false);
+                    << dof.ca.printMethodAlgo()
+                    << dof.ca.getHash().to_string(Base16, false);
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 out << ""
-                    << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
+                    << (dof.method.renderPrefix() + printHashType(dof.hashType))
                     << "";
             },
             [&](const DerivationOutput::Deferred &) {
@@ -792,12 +792,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
             [&](const DerivationOutput::Impure & doi) {
                 out << ""
-                    << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
+                    << (doi.method.renderPrefix() + printHashType(doi.hashType))
                     << "impure";
             },
         }, i.second.raw());
     }
-    worker_proto::write(store, out, drv.inputSrcs);
+    workerProtoWrite(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
@@ -811,13 +811,7 @@ std::string hashPlaceholder(const std::string_view outputName)
     return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
 }
 
-std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
-{
-    auto drvNameWithExtension = drvPath.name();
-    auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
-    auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
-    return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
-}
+
 
 
 static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
@@ -881,7 +875,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(
         for (auto & outputName : inputOutputs) {
             if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
                 inputRewrites.emplace(
-                    downstreamPlaceholder(store, inputDrv, outputName),
+                    DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
                     store.printStorePath(*actualPath));
                 resolved.inputSrcs.insert(*actualPath);
             } else {
@@ -942,7 +936,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
                 envHasRightPath(doia.path, i.first);
             },
             [&](const DerivationOutput::CAFixed & dof) {
-                StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} });
+                auto path = dof.path(store, drvName, i.first);
                 envHasRightPath(path, i.first);
             },
             [&](const DerivationOutput::CAFloating &) {
@@ -971,15 +965,16 @@ nlohmann::json DerivationOutput::toJSON(
         },
         [&](const DerivationOutput::CAFixed & dof) {
             res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
-            res["hashAlgo"] = dof.hash.printMethodAlgo();
-            res["hash"] = dof.hash.hash.to_string(Base16, false);
+            res["hashAlgo"] = dof.ca.printMethodAlgo();
+            res["hash"] = dof.ca.getHash().to_string(Base16, false);
+            // FIXME print refs?
         },
         [&](const DerivationOutput::CAFloating & dof) {
-            res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType);
+            res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType);
         },
         [&](const DerivationOutput::Deferred &) {},
         [&](const DerivationOutput::Impure & doi) {
-            res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType);
+            res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
             res["impure"] = true;
         },
     }, raw());
@@ -998,15 +993,15 @@ DerivationOutput DerivationOutput::fromJSON(
     for (const auto & [key, _] : json)
         keys.insert(key);
 
-    auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> {
+    auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> {
         std::string hashAlgo = json["hashAlgo"];
-        auto method = FileIngestionMethod::Flat;
-        if (hashAlgo.substr(0, 2) == "r:") {
-            method = FileIngestionMethod::Recursive;
-            hashAlgo = hashAlgo.substr(2);
-        }
-        auto hashType = parseHashType(hashAlgo);
-        return { method, hashType };
+        // remaining to parse, will be mutated by parsers
+        std::string_view s = hashAlgo;
+        ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
+        if (method == TextIngestionMethod {})
+            xpSettings.require(Xp::DynamicDerivations);
+        auto hashType = parseHashType(s);
+        return { std::move(method), std::move(hashType) };
     };
 
     if (keys == (std::set<std::string_view> { "path" })) {
@@ -1018,10 +1013,9 @@ DerivationOutput DerivationOutput::fromJSON(
     else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
         auto [method, hashType] = methodAlgo();
         auto dof = DerivationOutput::CAFixed {
-            .hash = {
-                .method = method,
-                .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
-            },
+            .ca = ContentAddress::fromParts(
+                std::move(method),
+                Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)),
         };
         if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
             throw Error("Path doesn't match derivation output");
@@ -1032,8 +1026,8 @@ DerivationOutput DerivationOutput::fromJSON(
         xpSettings.require(Xp::CaDerivations);
         auto [method, hashType] = methodAlgo();
         return DerivationOutput::CAFloating {
-            .method = method,
-            .hashType = hashType,
+            .method = std::move(method),
+            .hashType = std::move(hashType),
         };
     }
 
@@ -1045,7 +1039,7 @@ DerivationOutput DerivationOutput::fromJSON(
         xpSettings.require(Xp::ImpureDerivations);
         auto [method, hashType] = methodAlgo();
         return DerivationOutput::Impure {
-            .method = method,
+            .method = std::move(method),
             .hashType = hashType,
         };
     }

src/libstore/derivations.hh

@@ -6,6 +6,7 @@
 #include "hash.hh"
 #include "content-address.hh"
 #include "repair-flag.hh"
+#include "derived-path.hh"
 #include "sync.hh"
 #include "comparator.hh"
 
@@ -36,9 +37,11 @@ struct DerivationOutputInputAddressed
 struct DerivationOutputCAFixed
 {
     /**
-     * hash used for expected hash computation
+     * Method and hash used for expected hash computation.
+     *
+     * References are not allowed by fiat.
      */
-    FixedOutputHash hash;
+    ContentAddress ca;
 
     /**
      * Return the \ref StorePath "store path" corresponding to this output
@@ -48,7 +51,7 @@ struct DerivationOutputCAFixed
      */
     StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 
-    GENERATE_CMP(DerivationOutputCAFixed, me->hash);
+    GENERATE_CMP(DerivationOutputCAFixed, me->ca);
 };
 
 /**
@@ -61,7 +64,7 @@ struct DerivationOutputCAFloating
     /**
      * How the file system objects will be serialized for hashing
      */
-    FileIngestionMethod method;
+    ContentAddressMethod method;
 
     /**
      * How the serialization will be hashed
@@ -88,7 +91,7 @@ struct DerivationOutputImpure
     /**
      * How the file system objects will be serialized for hashing
      */
-    FileIngestionMethod method;
+    ContentAddressMethod method;
 
     /**
      * How the serialization will be hashed
@@ -343,12 +346,14 @@ struct Derivation : BasicDerivation
         Store & store,
         const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
 
-    /* Check that the derivation is valid and does not present any
-       illegal states.
-
-       This is mainly a matter of checking the outputs, where our C++
-       representation supports all sorts of combinations we do not yet
-       allow. */
+    /**
+     * Check that the derivation is valid and does not present any
+     * illegal states.
+     *
+     * This is mainly a matter of checking the outputs, where our C++
+     * representation supports all sorts of combinations we do not yet
+     * allow.
+     */
     void checkInvariants(Store & store, const StorePath & drvPath) const;
 
     Derivation() = default;
@@ -491,17 +496,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
  */
 std::string hashPlaceholder(const std::string_view outputName);
 
-/**
- * This creates an opaque and almost certainly unique string
- * deterministically from a derivation path and output name.
- *
- * It is used as a placeholder to allow derivations to refer to
- * content-addressed paths whose content --- and thus the path
- * themselves --- isn't yet known. This occurs when a derivation has a
- * dependency which is a CA derivation.
- */
-std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
-
 extern const Hash impureOutputHash;
 
 }

src/libstore/downstream-placeholder.cc

@@ -0,0 +1,39 @@
+#include "downstream-placeholder.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+std::string DownstreamPlaceholder::render() const
+{
+    return "/" + hash.to_string(Base32, false);
+}
+
+
+DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
+    const StorePath & drvPath,
+    std::string_view outputName)
+{
+    auto drvNameWithExtension = drvPath.name();
+    auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
+    auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
+    return DownstreamPlaceholder {
+        hashString(htSHA256, clearText)
+    };
+}
+
+DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
+    const DownstreamPlaceholder & placeholder,
+    std::string_view outputName,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    xpSettings.require(Xp::DynamicDerivations);
+    auto compressed = compressHash(placeholder.hash, 20);
+    auto clearText = "nix-computed-output:"
+        + compressed.to_string(Base32, false)
+        + ":" + std::string { outputName };
+    return DownstreamPlaceholder {
+        hashString(htSHA256, clearText)
+    };
+}
+
+}

src/libstore/downstream-placeholder.hh

@@ -0,0 +1,75 @@
+#pragma once
+///@file
+
+#include "hash.hh"
+#include "path.hh"
+
+namespace nix {
+
+/**
+ * Downstream Placeholders are opaque and almost certainly unique values
+ * used to allow derivations to refer to store objects which are yet to
+ * be built and for we do not yet have store paths for.
+ *
+ * They correspond to `DerivedPaths` that are not `DerivedPath::Opaque`,
+ * except for the cases involving input addressing or fixed outputs
+ * where we do know a store path for the derivation output in advance.
+ *
+ * Unlike `DerivationPath`, however, `DownstreamPlaceholder` is
+ * purposefully opaque and obfuscated. This is so they are hard to
+ * create by accident, and so substituting them (once we know what the
+ * path to store object is) is unlikely to capture other stuff it
+ * shouldn't.
+ *
+ * We use them with `Derivation`: the `render()` method is called to
+ * render an opaque string which can be used in the derivation, and the
+ * resolving logic can substitute those strings for store paths when
+ * resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`.
+ */
+class DownstreamPlaceholder
+{
+    /**
+     * `DownstreamPlaceholder` is just a newtype of `Hash`.
+     * This its only field.
+     */
+    Hash hash;
+
+    /**
+     * Newtype constructor
+     */
+    DownstreamPlaceholder(Hash hash) : hash(hash) { }
+
+public:
+    /**
+     * This creates an opaque and almost certainly unique string
+     * deterministically from the placeholder.
+     */
+    std::string render() const;
+
+    /**
+     * Create a placeholder for an unknown output of a content-addressed
+     * derivation.
+     *
+     * The derivation itself is known (we have a store path for it), but
+     * the output doesn't yet have a known store path.
+     */
+    static DownstreamPlaceholder unknownCaOutput(
+        const StorePath & drvPath,
+        std::string_view outputName);
+
+    /**
+     * Create a placehold for the output of an unknown derivation.
+     *
+     * The derivation is not yet known because it is a dynamic
+     * derivaiton --- it is itself an output of another derivation ---
+     * and we just have (another) placeholder for it.
+     *
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
+     */
+    static DownstreamPlaceholder unknownDerivation(
+        const DownstreamPlaceholder & drvPlaceholder,
+        std::string_view outputName,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+};
+
+}

src/libstore/export-import.cc

@@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    worker_proto::write(*this, teeSink, info->references);
+    workerProtoWrite(*this, teeSink, info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
 
-        auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
+        auto references = WorkerProto<StorePathSet>::read(*this, source);
         auto deriver = readString(source);
         auto narHash = hashString(htSHA256, saved.s);
 

src/libstore/globals.cc

@@ -183,7 +183,7 @@ bool Settings::isWSL1()
 Path Settings::getDefaultSSLCertFile()
 {
     for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
-        if (pathExists(fn)) return fn;
+        if (pathAccessible(fn)) return fn;
     return "";
 }
 

src/libstore/globals.hh

@@ -159,6 +159,15 @@ public:
         )",
         {"build-max-jobs"}};
 
+    Setting<unsigned int> maxSubstitutionJobs{
+        this, 16, "max-substitution-jobs",
+        R"(
+          This option defines the maximum number of substitution jobs that Nix
+          will try to run in parallel. The default is `16`. The minimum value
+          one can choose is `1` and lower values will be interpreted as `1`.
+        )",
+        {"substitution-max-jobs"}};
+
     Setting<unsigned int> buildCores{
         this,
         getDefaultCores(),
@@ -887,12 +896,11 @@ public:
         this, {}, "hashed-mirrors",
         R"(
           A list of web servers used by `builtins.fetchurl` to obtain files by
-          hash. The default is `http://tarballs.nixos.org/`. Given a hash type
-          *ht* and a base-16 hash *h*, Nix will try to download the file from
-          *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
-          they have disappeared from their original URI. For example, given
-          the default mirror `http://tarballs.nixos.org/`, when building the
-          derivation
+          hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
+          download the file from *hashed-mirror*/*ht*/*h*. This allows files to
+          be downloaded even if they have disappeared from their original URI.
+          For example, given an example mirror `http://tarballs.nixos.org/`,
+          when building the derivation
 
           ```nix
           builtins.fetchurl {
@@ -972,7 +980,7 @@ public:
         this, false, "use-xdg-base-directories",
         R"(
           If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`.
-          The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/installation/env-variables.md).
+          The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/command-ref/env-common.md).
 
           [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
 
@@ -986,6 +994,18 @@ public:
           | `~/.nix-profile`  | `$XDG_STATE_HOME/nix/profile`  |
           | `~/.nix-defexpr`  | `$XDG_STATE_HOME/nix/defexpr`  |
           | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` |
+
+          If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option.
+          If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`.
+          This can be achieved with the following shell commands:
+
+          ```sh
+          nix_state_home=${XDG_STATE_HOME-$HOME/.local/state}/nix
+          mkdir -p $nix_state_home
+          mv $HOME/.nix-profile $nix_state_home/profile
+          mv $HOME/.nix-defexpr $nix_state_home/defexpr
+          mv $HOME/.nix-channels $nix_state_home/channels
+          ```
         )"
     };
 };

src/libstore/legacy-ssh-store.cc

@@ -146,7 +146,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -180,7 +180,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            worker_proto::write(*this, conn->to, info.references);
+            workerProtoWrite(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            worker_proto::write(*this, conn->to, info.references);
+            workerProtoWrite(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -294,7 +294,7 @@ public:
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
             conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
-            auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+            auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
             for (auto && [output, realisation] : builtOutputs)
                 status.builtOutputs.insert_or_assign(
                     std::move(output.outputName),
@@ -344,6 +344,17 @@ public:
     virtual ref<FSAccessor> getFSAccessor() override
     { unsupported("getFSAccessor"); }
 
+    /**
+     * The default instance would schedule the work on the client side, but
+     * for consistency with `buildPaths` and `buildDerivation` it should happen
+     * on the remote side.
+     *
+     * We make this fail for now so we can add implement this properly later
+     * without it being a breaking change.
+     */
+    void repairPath(const StorePath & path) override
+    { unsupported("repairPath"); }
+
     void computeFSClosure(const StorePathSet & paths,
         StorePathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false) override
@@ -358,10 +369,10 @@ public:
         conn->to
             << cmdQueryClosure
             << includeOutputs;
-        worker_proto::write(*this, conn->to, paths);
+        workerProtoWrite(*this, conn->to, paths);
         conn->to.flush();
 
-        for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
+        for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
             out.insert(i);
     }
 
@@ -374,10 +385,10 @@ public:
             << cmdQueryValidPaths
             << false // lock
             << maybeSubstitute;
-        worker_proto::write(*this, conn->to, paths);
+        workerProtoWrite(*this, conn->to, paths);
         conn->to.flush();
 
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 
     void connect() override

src/libstore/local-store.hh

@@ -240,8 +240,6 @@ public:
 
     void vacuumDB();
 
-    void repairPath(const StorePath & path) override;
-
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
     /**

src/libstore/local.mk

@@ -57,12 +57,6 @@ $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
 $(d)/build.cc:
 
-%.gen.hh: %
-	@echo 'R"foo(' >> $@.tmp
-	$(trace-gen) cat $< >> $@.tmp
-	@echo ')foo"' >> $@.tmp
-	@mv $@.tmp $@
-
 clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
 $(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))

src/libstore/misc.cc

@@ -83,14 +83,15 @@ void Store::computeFSClosure(const StorePath & startPath,
 }
 
 
-std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
+const ContentAddress * getDerivationCA(const BasicDerivation & drv)
 {
     auto out = drv.outputs.find("out");
-    if (out != drv.outputs.end()) {
-        if (const auto * v = std::get_if<DerivationOutput::CAFixed>(&out->second.raw()))
-            return v->hash;
+    if (out == drv.outputs.end())
+        return nullptr;
+    if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second)) {
+        return &dof->ca;
     }
-    return std::nullopt;
+    return nullptr;
 }
 
 void Store::queryMissing(const std::vector<DerivedPath> & targets,
@@ -140,7 +141,13 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
         if (drvState_->lock()->done) return;
 
         SubstitutablePathInfos infos;
-        querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
+        auto * cap = getDerivationCA(*drv);
+        querySubstitutablePathInfos({
+            {
+                outPath,
+                cap ? std::optional { *cap } : std::nullopt,
+            },
+        }, infos);
 
         if (infos.empty()) {
             drvState_->lock()->done = true;

src/libstore/path-info.cc

@@ -1,5 +1,6 @@
 #include "path-info.hh"
 #include "worker-protocol.hh"
+#include "store-api.hh"
 
 namespace nix {
 
@@ -131,7 +132,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
     auto narHash = Hash::parseAny(readString(source), htSHA256);
     ValidPathInfo info(path, narHash);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
-    info.references = worker_proto::read(store, source, Phantom<StorePathSet> {});
+    info.references = WorkerProto<StorePathSet>::read(store, source);
     source >> info.registrationTime >> info.narSize;
     if (format >= 16) {
         source >> info.ultimate;
@@ -152,7 +153,7 @@ void ValidPathInfo::write(
         sink << store.printStorePath(path);
     sink << (deriver ? store.printStorePath(*deriver) : "")
          << narHash.to_string(Base16, false);
-    worker_proto::write(store, sink, references);
+    workerProtoWrite(store, sink, references);
     sink << registrationTime << narSize;
     if (format >= 16) {
         sink << ultimate

src/libstore/path.cc

@@ -9,8 +9,8 @@ static void checkName(std::string_view path, std::string_view name)
     if (name.empty())
         throw BadStorePath("store path '%s' has an empty name", path);
     if (name.size() > StorePath::MaxPathLen)
-        throw BadStorePath("store path '%s' has a name longer than '%d characters",
-            StorePath::MaxPathLen, path);
+        throw BadStorePath("store path '%s' has a name longer than %d characters",
+            path, StorePath::MaxPathLen);
     // See nameRegexStr for the definition
     for (auto c : name)
         if (!((c >= '0' && c <= '9')

src/libstore/realisation.cc

@@ -136,6 +136,19 @@ size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const
     return good;
 }
 
+
+SingleDrvOutputs filterDrvOutputs(const OutputsSpec& wanted, SingleDrvOutputs&& outputs)
+{
+    SingleDrvOutputs ret = std::move(outputs);
+    for (auto it = ret.begin(); it != ret.end(); ) {
+        if (!wanted.contains(it->first))
+            it = ret.erase(it);
+        else
+            ++it;
+    }
+    return ret;
+}
+
 StorePath RealisedPath::path() const {
     return std::visit([](auto && arg) { return arg.getPath(); }, raw);
 }

src/libstore/realisation.hh

@@ -12,6 +12,7 @@
 namespace nix {
 
 class Store;
+struct OutputsSpec;
 
 /**
  * A general `Realisation` key.
@@ -93,6 +94,14 @@ typedef std::map<std::string, Realisation> SingleDrvOutputs;
  */
 typedef std::map<DrvOutput, Realisation> DrvOutputs;
 
+/**
+ * Filter a SingleDrvOutputs to include only specific output names
+ *
+ * Moves the `outputs` input.
+ */
+SingleDrvOutputs filterDrvOutputs(const OutputsSpec&, SingleDrvOutputs&&);
+
+
 struct OpaquePath {
     StorePath path;
 

src/libstore/remote-store.cc

@@ -18,189 +18,6 @@
 
 namespace nix {
 
-namespace worker_proto {
-
-std::string read(const Store & store, Source & from, Phantom<std::string> _)
-{
-    return readString(from);
-}
-
-void write(const Store & store, Sink & out, const std::string & str)
-{
-    out << str;
-}
-
-
-StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
-{
-    return store.parseStorePath(readString(from));
-}
-
-void write(const Store & store, Sink & out, const StorePath & storePath)
-{
-    out << store.printStorePath(storePath);
-}
-
-
-std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _)
-{
-    auto temp = readNum<uint8_t>(from);
-    switch (temp) {
-        case 0:
-            return std::nullopt;
-        case 1:
-            return { Trusted };
-        case 2:
-            return { NotTrusted };
-        default:
-            throw Error("Invalid trusted status from remote");
-    }
-}
-
-void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
-{
-    if (!optTrusted)
-        out << (uint8_t)0;
-    else {
-        switch (*optTrusted) {
-        case Trusted:
-            out << (uint8_t)1;
-            break;
-        case NotTrusted:
-            out << (uint8_t)2;
-            break;
-        default:
-            assert(false);
-        };
-    }
-}
-
-
-ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
-{
-    return ContentAddress::parse(readString(from));
-}
-
-void write(const Store & store, Sink & out, const ContentAddress & ca)
-{
-    out << renderContentAddress(ca);
-}
-
-
-DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
-{
-    auto s = readString(from);
-    return DerivedPath::parseLegacy(store, s);
-}
-
-void write(const Store & store, Sink & out, const DerivedPath & req)
-{
-    out << req.to_string_legacy(store);
-}
-
-
-Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
-{
-    std::string rawInput = readString(from);
-    return Realisation::fromJSON(
-        nlohmann::json::parse(rawInput),
-        "remote-protocol"
-    );
-}
-
-void write(const Store & store, Sink & out, const Realisation & realisation)
-{
-    out << realisation.toJSON().dump();
-}
-
-
-DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
-{
-    return DrvOutput::parse(readString(from));
-}
-
-void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
-{
-    out << drvOutput.to_string();
-}
-
-
-KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _)
-{
-    auto path = worker_proto::read(store, from, Phantom<DerivedPath> {});
-    auto br = worker_proto::read(store, from, Phantom<BuildResult> {});
-    return KeyedBuildResult {
-        std::move(br),
-        /* .path = */ std::move(path),
-    };
-}
-
-void write(const Store & store, Sink & to, const KeyedBuildResult & res)
-{
-    worker_proto::write(store, to, res.path);
-    worker_proto::write(store, to, static_cast<const BuildResult &>(res));
-}
-
-
-BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
-{
-    BuildResult res;
-    res.status = (BuildResult::Status) readInt(from);
-    from
-        >> res.errorMsg
-        >> res.timesBuilt
-        >> res.isNonDeterministic
-        >> res.startTime
-        >> res.stopTime;
-    auto builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {});
-    for (auto && [output, realisation] : builtOutputs)
-        res.builtOutputs.insert_or_assign(
-            std::move(output.outputName),
-            std::move(realisation));
-    return res;
-}
-
-void write(const Store & store, Sink & to, const BuildResult & res)
-{
-    to
-        << res.status
-        << res.errorMsg
-        << res.timesBuilt
-        << res.isNonDeterministic
-        << res.startTime
-        << res.stopTime;
-    DrvOutputs builtOutputs;
-    for (auto & [output, realisation] : res.builtOutputs)
-        builtOutputs.insert_or_assign(realisation.id, realisation);
-    worker_proto::write(store, to, builtOutputs);
-}
-
-
-std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
-{
-    auto s = readString(from);
-    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
-}
-
-void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
-{
-    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
-}
-
-
-std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
-{
-    return ContentAddress::parseOpt(readString(from));
-}
-
-void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
-{
-    out << (caOpt ? renderContentAddress(*caOpt) : "");
-}
-
-}
-
-
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : RemoteStoreConfig(params)
@@ -283,7 +100,7 @@ void RemoteStore::initConnection(Connection & conn)
         }
 
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
-            conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {});
+            conn.remoteTrustsUs = WorkerProto<std::optional<TrustedFlag>>::read(*this, conn.from);
         } else {
             // We don't know the answer; protocol to old.
             conn.remoteTrustsUs = std::nullopt;
@@ -410,12 +227,12 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         return res;
     } else {
         conn->to << wopQueryValidPaths;
-        worker_proto::write(*this, conn->to, paths);
+        workerProtoWrite(*this, conn->to, paths);
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
             conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
         }
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 }
 
@@ -425,7 +242,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
     auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -442,9 +259,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths;
-        worker_proto::write(*this, conn->to, paths);
+        workerProtoWrite(*this, conn->to, paths);
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        return WorkerProto<StorePathSet>::read(*this, conn->from);
     }
 }
 
@@ -466,7 +283,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
             infos.insert_or_assign(i.first, std::move(info));
@@ -479,9 +296,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             StorePathSet paths;
             for (auto & path : pathsMap)
                 paths.insert(path.first);
-            worker_proto::write(*this, conn->to, paths);
+            workerProtoWrite(*this, conn->to, paths);
         } else
-            worker_proto::write(*this, conn->to, pathsMap);
+            workerProtoWrite(*this, conn->to, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
@@ -489,7 +306,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+            info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -532,7 +349,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
     auto conn(getConnection());
     conn->to << wopQueryReferrers << printStorePath(path);
     conn.processStderr();
-    for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
+    for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
         referrers.insert(i);
 }
 
@@ -542,7 +359,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
     auto conn(getConnection());
     conn->to << wopQueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -554,7 +371,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
     auto conn(getConnection());
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    return WorkerProto<StorePathSet>::read(*this, conn->from);
 }
 
 
@@ -564,7 +381,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
         auto conn(getConnection());
         conn->to << wopQueryDerivationOutputMap << printStorePath(path);
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
+        return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
     } else {
         // Fallback for old daemon versions.
         // For floating-CA derivations (and their co-dependencies) this is an
@@ -597,6 +414,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
     Source & dump,
     std::string_view name,
     ContentAddressMethod caMethod,
+    HashType hashType,
     const StorePathSet & references,
     RepairFlag repair)
 {
@@ -608,8 +426,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
         conn->to
             << wopAddToStore
             << name
-            << caMethod.render();
-        worker_proto::write(*this, conn->to, references);
+            << caMethod.render(hashType);
+        workerProtoWrite(*this, conn->to, references);
         conn->to << repair;
 
         // The dump source may invoke the store, so we need to make some room.
@@ -628,26 +446,29 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
         if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
 
         std::visit(overloaded {
-            [&](const TextHashMethod & thm) -> void {
+            [&](const TextIngestionMethod & thm) -> void {
+                if (hashType != htSHA256)
+                    throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
+                        name, printHashType(hashType));
                 std::string s = dump.drain();
                 conn->to << wopAddTextToStore << name << s;
-                worker_proto::write(*this, conn->to, references);
+                workerProtoWrite(*this, conn->to, references);
                 conn.processStderr();
             },
-            [&](const FixedOutputHashMethod & fohm) -> void {
+            [&](const FileIngestionMethod & fim) -> void {
                 conn->to
                     << wopAddToStore
                     << name
-                    << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
-                    << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
-                    << printHashType(fohm.hashType);
+                    << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+                    << (fim == FileIngestionMethod::Recursive ? 1 : 0)
+                    << printHashType(hashType);
 
                 try {
                     conn->to.written = 0;
                     connections->incCapacity();
                     {
                         Finally cleanup([&]() { connections->decCapacity(); });
-                        if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
+                        if (fim == FileIngestionMethod::Recursive) {
                             dump.drainInto(conn->to);
                         } else {
                             std::string contents = dump.drain();
@@ -678,7 +499,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
       FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
 {
-    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
+    return addCAToStore(dump, name, method, hashType, references, repair)->path;
 }
 
 
@@ -697,7 +518,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            worker_proto::write(*this, sink, info.references);
+            workerProtoWrite(*this, sink, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -707,7 +528,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        auto importedPaths = WorkerProto<StorePathSet>::read(*this, conn->from);
         assert(importedPaths.size() <= 1);
     }
 
@@ -716,7 +537,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash.to_string(Base16, false);
-        worker_proto::write(*this, conn->to, info.references);
+        workerProtoWrite(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
@@ -778,7 +599,7 @@ StorePath RemoteStore::addTextToStore(
     RepairFlag repair)
 {
     StringSource source(s);
-    return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
+    return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path;
 }
 
 void RemoteStore::registerDrvOutput(const Realisation & info)
@@ -789,7 +610,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
         conn->to << info.id.to_string();
         conn->to << std::string(info.outPath.to_string());
     } else {
-        worker_proto::write(*this, conn->to, info);
+        workerProtoWrite(*this, conn->to, info);
     }
     conn.processStderr();
 }
@@ -811,14 +632,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
 
         auto real = [&]() -> std::shared_ptr<const Realisation> {
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
-                auto outPaths = worker_proto::read(
-                    *this, conn->from, Phantom<std::set<StorePath>> {});
+                auto outPaths = WorkerProto<std::set<StorePath>>::read(
+                    *this, conn->from);
                 if (outPaths.empty())
                     return nullptr;
                 return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
             } else {
-                auto realisations = worker_proto::read(
-                    *this, conn->from, Phantom<std::set<Realisation>> {});
+                auto realisations = WorkerProto<std::set<Realisation>>::read(
+                    *this, conn->from);
                 if (realisations.empty())
                     return nullptr;
                 return std::make_shared<const Realisation>(*realisations.begin());
@@ -832,7 +653,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
 static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
 {
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
-        worker_proto::write(store, conn->to, reqs);
+        workerProtoWrite(store, conn->to, reqs);
     } else {
         Strings ss;
         for (auto & p : reqs) {
@@ -902,7 +723,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
         writeDerivedPaths(*this, conn, paths);
         conn->to << buildMode;
         conn.processStderr();
-        return worker_proto::read(*this, conn->from, Phantom<std::vector<KeyedBuildResult>> {});
+        return WorkerProto<std::vector<KeyedBuildResult>>::read(*this, conn->from);
     } else {
         // Avoid deadlock.
         conn_.reset();
@@ -985,7 +806,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
         conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
     }
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
-        auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+        auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
         for (auto && [output, realisation] : builtOutputs)
             res.builtOutputs.insert_or_assign(
                 std::move(output.outputName),
@@ -1044,7 +865,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     conn->to
         << wopCollectGarbage << options.action;
-    worker_proto::write(*this, conn->to, options.pathsToDelete);
+    workerProtoWrite(*this, conn->to, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -1103,9 +924,9 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
         conn->to << wopQueryMissing;
         writeDerivedPaths(*this, conn, targets);
         conn.processStderr();
-        willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        willBuild = WorkerProto<StorePathSet>::read(*this, conn->from);
+        willSubstitute = WorkerProto<StorePathSet>::read(*this, conn->from);
+        unknown = WorkerProto<StorePathSet>::read(*this, conn->from);
         conn->from >> downloadSize >> narSize;
         return;
     }

src/libstore/remote-store.hh

@@ -78,6 +78,7 @@ public:
         Source & dump,
         std::string_view name,
         ContentAddressMethod caMethod,
+        HashType hashType,
         const StorePathSet & references,
         RepairFlag repair);
 
@@ -136,6 +137,17 @@ public:
 
     bool verifyStore(bool checkContents, RepairFlag repair) override;
 
+    /**
+     * The default instance would schedule the work on the client side, but
+     * for consistency with `buildPaths` and `buildDerivation` it should happen
+     * on the remote side.
+     *
+     * We make this fail for now so we can add implement this properly later
+     * without it being a breaking change.
+     */
+    void repairPath(const StorePath & path) override
+    { unsupported("repairPath"); }
+
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
     void queryMissing(const std::vector<DerivedPath> & targets,

src/libstore/ssh.cc

@@ -41,6 +41,11 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
     args.push_back("-oLocalCommand=echo started");
 }
 
+bool SSHMaster::isMasterRunning() {
+    auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true});
+    return res.first == 0;
+}
+
 std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
 {
     Path socketPath = startMaster();
@@ -97,7 +102,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
     // Wait for the SSH connection to be established,
     // So that we don't overwrite the password prompt with our progress bar.
-    if (!fakeSSH && !useMaster) {
+    if (!fakeSSH && !useMaster && !isMasterRunning()) {
         std::string reply;
         try {
             reply = readLine(out.readSide.get());
@@ -133,6 +138,8 @@ Path SSHMaster::startMaster()
     logger->pause();
     Finally cleanup = [&]() { logger->resume(); };
 
+    bool wasMasterRunning = isMasterRunning();
+
     state->sshMaster = startProcess([&]() {
         restoreProcessContext();
 
@@ -152,13 +159,15 @@ Path SSHMaster::startMaster()
 
     out.writeSide = -1;
 
-    std::string reply;
-    try {
-        reply = readLine(out.readSide.get());
-    } catch (EndOfFile & e) { }
+    if (!wasMasterRunning) {
+        std::string reply;
+        try {
+            reply = readLine(out.readSide.get());
+        } catch (EndOfFile & e) { }
 
-    if (reply != "started")
-        throw Error("failed to start SSH master connection to '%s'", host);
+        if (reply != "started")
+            throw Error("failed to start SSH master connection to '%s'", host);
+    }
 
     return state->socketPath;
 }

src/libstore/ssh.hh

@@ -28,6 +28,7 @@ private:
     Sync<State> state_;
 
     void addCommonSSHOpts(Strings & args);
+    bool isMasterRunning();
 
 public:
 

src/libstore/store-api.hh

@@ -679,8 +679,7 @@ public:
      * Repair the contents of the given path by redownloading it using
      * a substituter (if available).
      */
-    virtual void repairPath(const StorePath & path)
-    { unsupported("repairPath"); }
+    virtual void repairPath(const StorePath & path);
 
     /**
      * Add signatures to the specified store path. The signatures are
@@ -1022,7 +1021,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
  */
 std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
 
-std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
+const ContentAddress * getDerivationCA(const BasicDerivation & drv);
 
 std::map<DrvOutput, StorePath> drvOutputReferences(
     Store & store,

src/libstore/tests/derivation.cc

@@ -26,6 +26,14 @@ class CaDerivationTest : public DerivationTest
     }
 };
 
+class DynDerivationTest : public DerivationTest
+{
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
+    }
+};
+
 class ImpureDerivationTest : public DerivationTest
 {
     void SetUp() override
@@ -66,20 +74,47 @@ TEST_JSON(DerivationTest, inputAddressed,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(DerivationTest, caFixed,
+TEST_JSON(DerivationTest, caFixedFlat,
+    R"({
+        "hashAlgo": "sha256",
+        "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+        "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
+    })",
+    (DerivationOutput::CAFixed {
+        .ca = FixedOutputHash {
+            .method = FileIngestionMethod::Flat,
+            .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
+        },
+    }),
+    "drv-name", "output-name")
+
+TEST_JSON(DerivationTest, caFixedNAR,
     R"({
         "hashAlgo": "r:sha256",
         "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
         "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
     })",
     (DerivationOutput::CAFixed {
-        .hash = {
+        .ca = FixedOutputHash {
             .method = FileIngestionMethod::Recursive,
             .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
         },
     }),
     "drv-name", "output-name")
 
+TEST_JSON(DynDerivationTest, caFixedText,
+    R"({
+        "hashAlgo": "text:sha256",
+        "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+        "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
+    })",
+    (DerivationOutput::CAFixed {
+        .ca = TextHash {
+            .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
+        },
+    }),
+    "drv-name", "output-name")
+
 TEST_JSON(CaDerivationTest, caFloating,
     R"({
         "hashAlgo": "r:sha256"

src/libstore/tests/derived-path.cc

@@ -27,11 +27,13 @@ Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
 
 Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
 {
-    switch (*gen::inRange<uint8_t>(0, 1)) {
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
     case 0:
         return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>());
-    default:
+    case 1:
         return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>());
+    default:
+        assert(false);
     }
 }
 

src/libstore/tests/downstream-placeholder.cc

@@ -0,0 +1,33 @@
+#include <gtest/gtest.h>
+
+#include "downstream-placeholder.hh"
+
+namespace nix {
+
+TEST(DownstreamPlaceholder, unknownCaOutput) {
+    ASSERT_EQ(
+        DownstreamPlaceholder::unknownCaOutput(
+            StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
+            "out").render(),
+        "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz");
+}
+
+TEST(DownstreamPlaceholder, unknownDerivation) {
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+    mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
+
+    ASSERT_EQ(
+        DownstreamPlaceholder::unknownDerivation(
+            DownstreamPlaceholder::unknownCaOutput(
+                StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" },
+                "out"),
+            "out",
+            mockXpSettings).render(),
+        "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w");
+}
+
+}

src/libstore/tests/outputs-spec.cc

@@ -206,15 +206,17 @@ using namespace nix;
 
 Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
 {
-    switch (*gen::inRange<uint8_t>(0, 1)) {
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<OutputsSpec::Raw>)) {
     case 0:
         return gen::just((OutputsSpec) OutputsSpec::All { });
-    default:
+    case 1:
         return gen::just((OutputsSpec) OutputsSpec::Names {
             *gen::nonEmpty(gen::container<StringSet>(gen::map(
                 gen::arbitrary<StorePathName>(),
                 [](StorePathName n) { return n.name; }))),
         });
+    default:
+        assert(false);
     }
 }
 

src/libstore/worker-protocol.cc

@@ -0,0 +1,192 @@
+#include "serialise.hh"
+#include "util.hh"
+#include "path-with-outputs.hh"
+#include "store-api.hh"
+#include "build-result.hh"
+#include "worker-protocol.hh"
+#include "archive.hh"
+#include "derivations.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+std::string WorkerProto<std::string>::read(const Store & store, Source & from)
+{
+    return readString(from);
+}
+
+void WorkerProto<std::string>::write(const Store & store, Sink & out, const std::string & str)
+{
+    out << str;
+}
+
+
+StorePath WorkerProto<StorePath>::read(const Store & store, Source & from)
+{
+    return store.parseStorePath(readString(from));
+}
+
+void WorkerProto<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
+{
+    out << store.printStorePath(storePath);
+}
+
+
+std::optional<TrustedFlag> WorkerProto<std::optional<TrustedFlag>>::read(const Store & store, Source & from)
+{
+    auto temp = readNum<uint8_t>(from);
+    switch (temp) {
+        case 0:
+            return std::nullopt;
+        case 1:
+            return { Trusted };
+        case 2:
+            return { NotTrusted };
+        default:
+            throw Error("Invalid trusted status from remote");
+    }
+}
+
+void WorkerProto<std::optional<TrustedFlag>>::write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
+{
+    if (!optTrusted)
+        out << (uint8_t)0;
+    else {
+        switch (*optTrusted) {
+        case Trusted:
+            out << (uint8_t)1;
+            break;
+        case NotTrusted:
+            out << (uint8_t)2;
+            break;
+        default:
+            assert(false);
+        };
+    }
+}
+
+
+ContentAddress WorkerProto<ContentAddress>::read(const Store & store, Source & from)
+{
+    return ContentAddress::parse(readString(from));
+}
+
+void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
+{
+    out << renderContentAddress(ca);
+}
+
+
+DerivedPath WorkerProto<DerivedPath>::read(const Store & store, Source & from)
+{
+    auto s = readString(from);
+    return DerivedPath::parseLegacy(store, s);
+}
+
+void WorkerProto<DerivedPath>::write(const Store & store, Sink & out, const DerivedPath & req)
+{
+    out << req.to_string_legacy(store);
+}
+
+
+Realisation WorkerProto<Realisation>::read(const Store & store, Source & from)
+{
+    std::string rawInput = readString(from);
+    return Realisation::fromJSON(
+        nlohmann::json::parse(rawInput),
+        "remote-protocol"
+    );
+}
+
+void WorkerProto<Realisation>::write(const Store & store, Sink & out, const Realisation & realisation)
+{
+    out << realisation.toJSON().dump();
+}
+
+
+DrvOutput WorkerProto<DrvOutput>::read(const Store & store, Source & from)
+{
+    return DrvOutput::parse(readString(from));
+}
+
+void WorkerProto<DrvOutput>::write(const Store & store, Sink & out, const DrvOutput & drvOutput)
+{
+    out << drvOutput.to_string();
+}
+
+
+KeyedBuildResult WorkerProto<KeyedBuildResult>::read(const Store & store, Source & from)
+{
+    auto path = WorkerProto<DerivedPath>::read(store, from);
+    auto br = WorkerProto<BuildResult>::read(store, from);
+    return KeyedBuildResult {
+        std::move(br),
+        /* .path = */ std::move(path),
+    };
+}
+
+void WorkerProto<KeyedBuildResult>::write(const Store & store, Sink & to, const KeyedBuildResult & res)
+{
+    workerProtoWrite(store, to, res.path);
+    workerProtoWrite(store, to, static_cast<const BuildResult &>(res));
+}
+
+
+BuildResult WorkerProto<BuildResult>::read(const Store & store, Source & from)
+{
+    BuildResult res;
+    res.status = (BuildResult::Status) readInt(from);
+    from
+        >> res.errorMsg
+        >> res.timesBuilt
+        >> res.isNonDeterministic
+        >> res.startTime
+        >> res.stopTime;
+    auto builtOutputs = WorkerProto<DrvOutputs>::read(store, from);
+    for (auto && [output, realisation] : builtOutputs)
+        res.builtOutputs.insert_or_assign(
+            std::move(output.outputName),
+            std::move(realisation));
+    return res;
+}
+
+void WorkerProto<BuildResult>::write(const Store & store, Sink & to, const BuildResult & res)
+{
+    to
+        << res.status
+        << res.errorMsg
+        << res.timesBuilt
+        << res.isNonDeterministic
+        << res.startTime
+        << res.stopTime;
+    DrvOutputs builtOutputs;
+    for (auto & [output, realisation] : res.builtOutputs)
+        builtOutputs.insert_or_assign(realisation.id, realisation);
+    workerProtoWrite(store, to, builtOutputs);
+}
+
+
+std::optional<StorePath> WorkerProto<std::optional<StorePath>>::read(const Store & store, Source & from)
+{
+    auto s = readString(from);
+    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
+}
+
+void WorkerProto<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+{
+    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
+}
+
+
+std::optional<ContentAddress> WorkerProto<std::optional<ContentAddress>>::read(const Store & store, Source & from)
+{
+    return ContentAddress::parseOpt(readString(from));
+}
+
+void WorkerProto<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
+{
+    out << (caOpt ? renderContentAddress(*caOpt) : "");
+}
+
+}

src/libstore/worker-protocol.hh

@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "store-api.hh"
 #include "serialise.hh"
 
 namespace nix {
@@ -79,41 +78,81 @@ typedef enum {
 class Store;
 struct Source;
 
+// items being serialized
+struct DerivedPath;
+struct DrvOutput;
+struct Realisation;
+struct BuildResult;
+struct KeyedBuildResult;
+enum TrustedFlag : bool;
+
+
 /**
- * Used to guide overloading
+ * Data type for canonical pairs of serializers for the worker protocol.
  *
  * See https://en.cppreference.com/w/cpp/language/adl for the broader
  * concept of what is going on here.
  */
 template<typename T>
-struct Phantom {};
+struct WorkerProto {
+    static T read(const Store & store, Source & from);
+    static void write(const Store & store, Sink & out, const T & t);
+};
 
+/**
+ * Wrapper function around `WorkerProto<T>::write` that allows us to
+ * infer the type instead of having to write it down explicitly.
+ */
+template<typename T>
+void workerProtoWrite(const Store & store, Sink & out, const T & t)
+{
+    WorkerProto<T>::write(store, out, t);
+}
 
-namespace worker_proto {
-/* FIXME maybe move more stuff inside here */
+/**
+ * Declare a canonical serializer pair for the worker protocol.
+ *
+ * We specialize the struct merely to indicate that we are implementing
+ * the function for the given type.
+ *
+ * Some sort of `template<...>` must be used with the caller for this to
+ * be legal specialization syntax. See below for what that looks like in
+ * practice.
+ */
+#define MAKE_WORKER_PROTO(T) \
+    struct WorkerProto< T > { \
+        static T read(const Store & store, Source & from); \
+        static void write(const Store & store, Sink & out, const T & t); \
+    };
 
-#define MAKE_WORKER_PROTO(TEMPLATE, T) \
-    TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \
-    TEMPLATE void write(const Store & store, Sink & out, const T & str)
+template<>
+MAKE_WORKER_PROTO(std::string);
+template<>
+MAKE_WORKER_PROTO(StorePath);
+template<>
+MAKE_WORKER_PROTO(ContentAddress);
+template<>
+MAKE_WORKER_PROTO(DerivedPath);
+template<>
+MAKE_WORKER_PROTO(Realisation);
+template<>
+MAKE_WORKER_PROTO(DrvOutput);
+template<>
+MAKE_WORKER_PROTO(BuildResult);
+template<>
+MAKE_WORKER_PROTO(KeyedBuildResult);
+template<>
+MAKE_WORKER_PROTO(std::optional<TrustedFlag>);
 
-MAKE_WORKER_PROTO(, std::string);
-MAKE_WORKER_PROTO(, StorePath);
-MAKE_WORKER_PROTO(, ContentAddress);
-MAKE_WORKER_PROTO(, DerivedPath);
-MAKE_WORKER_PROTO(, Realisation);
-MAKE_WORKER_PROTO(, DrvOutput);
-MAKE_WORKER_PROTO(, BuildResult);
-MAKE_WORKER_PROTO(, KeyedBuildResult);
-MAKE_WORKER_PROTO(, std::optional<TrustedFlag>);
+template<typename T>
+MAKE_WORKER_PROTO(std::vector<T>);
+template<typename T>
+MAKE_WORKER_PROTO(std::set<T>);
 
-MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
-MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
-
-#define X_ template<typename K, typename V>
-#define Y_ std::map<K, V>
-MAKE_WORKER_PROTO(X_, Y_);
+template<typename K, typename V>
+#define X_ std::map<K, V>
+MAKE_WORKER_PROTO(X_);
 #undef X_
-#undef Y_
 
 /**
  * These use the empty string for the null case, relying on the fact
@@ -129,72 +168,72 @@ MAKE_WORKER_PROTO(X_, Y_);
  * worker protocol harder to implement in other languages where such
  * specializations may not be allowed.
  */
-MAKE_WORKER_PROTO(, std::optional<StorePath>);
-MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
+template<>
+MAKE_WORKER_PROTO(std::optional<StorePath>);
+template<>
+MAKE_WORKER_PROTO(std::optional<ContentAddress>);
 
 template<typename T>
-std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
+std::vector<T> WorkerProto<std::vector<T>>::read(const Store & store, Source & from)
 {
     std::vector<T> resSet;
     auto size = readNum<size_t>(from);
     while (size--) {
-        resSet.push_back(read(store, from, Phantom<T> {}));
+        resSet.push_back(WorkerProto<T>::read(store, from));
     }
     return resSet;
 }
 
 template<typename T>
-void write(const Store & store, Sink & out, const std::vector<T> & resSet)
+void WorkerProto<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
 {
     out << resSet.size();
     for (auto & key : resSet) {
-        write(store, out, key);
+        WorkerProto<T>::write(store, out, key);
     }
 }
 
 template<typename T>
-std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
+std::set<T> WorkerProto<std::set<T>>::read(const Store & store, Source & from)
 {
     std::set<T> resSet;
     auto size = readNum<size_t>(from);
     while (size--) {
-        resSet.insert(read(store, from, Phantom<T> {}));
+        resSet.insert(WorkerProto<T>::read(store, from));
     }
     return resSet;
 }
 
 template<typename T>
-void write(const Store & store, Sink & out, const std::set<T> & resSet)
+void WorkerProto<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
 {
     out << resSet.size();
     for (auto & key : resSet) {
-        write(store, out, key);
+        WorkerProto<T>::write(store, out, key);
     }
 }
 
 template<typename K, typename V>
-std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
+std::map<K, V> WorkerProto<std::map<K, V>>::read(const Store & store, Source & from)
 {
     std::map<K, V> resMap;
     auto size = readNum<size_t>(from);
     while (size--) {
-        auto k = read(store, from, Phantom<K> {});
-        auto v = read(store, from, Phantom<V> {});
+        auto k = WorkerProto<K>::read(store, from);
+        auto v = WorkerProto<V>::read(store, from);
         resMap.insert_or_assign(std::move(k), std::move(v));
     }
     return resMap;
 }
 
 template<typename K, typename V>
-void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
+void WorkerProto<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
 {
     out << resMap.size();
     for (auto & i : resMap) {
-        write(store, out, i.first);
-        write(store, out, i.second);
+        WorkerProto<K>::write(store, out, i.first);
+        WorkerProto<V>::write(store, out, i.second);
     }
 }
 
 }
-
-}

src/libutil/canon-path.cc

@@ -13,6 +13,11 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
     : path(absPath((Path) raw, root.abs()))
 { }
 
+CanonPath CanonPath::fromCwd(std::string_view path)
+{
+    return CanonPath(unchecked_t(), absPath((Path) path));
+}
+
 std::optional<CanonPath> CanonPath::parent() const
 {
     if (isRoot()) return std::nullopt;

src/libutil/canon-path.hh

@@ -46,6 +46,8 @@ public:
         : path(std::move(path))
     { }
 
+    static CanonPath fromCwd(std::string_view path = ".");
+
     static CanonPath root;
 
     /**

src/libutil/experimental-features.cc

@@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
+constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",
@@ -50,6 +50,8 @@ constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
             or other impure derivations can rely on impure derivations. Finally,
             an impure derivation cannot also be
             [content-addressed](#xp-feature-ca-derivations).
+
+            This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime).
         )",
     },
     {
@@ -199,6 +201,26 @@ constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
             networking.
         )",
     },
+    {
+        .tag = Xp::DynamicDerivations,
+        .name = "dynamic-derivations",
+        .description = R"(
+            Allow the use of a few things related to dynamic derivations:
+
+              - "text hashing" derivation outputs, so we can build .drv
+                files.
+
+              - dependencies in derivations on the outputs of
+                derivations that are themselves derivations outputs.
+        )",
+    },
+    {
+        .tag = Xp::ParseTomlTimestamps,
+        .name = "parse-toml-timestamps",
+        .description = R"(
+            Allow parsing of timestamps in builtins.fromTOML.
+        )",
+    },
 }};
 
 static_assert(
@@ -233,7 +255,7 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag)
     return xpFeatureDetails[(size_t)tag].name;
 }
 
-nlohmann::json documentExperimentalFeatures() 
+nlohmann::json documentExperimentalFeatures()
 {
     StringMap res;
     for (auto & xpFeature : xpFeatureDetails)

src/libutil/experimental-features.hh

@@ -29,6 +29,8 @@ enum struct ExperimentalFeature
     Cgroups,
     DiscardReferences,
     DaemonTrustOverride,
+    DynamicDerivations,
+    ParseTomlTimestamps,
 };
 
 /**

src/libutil/tarfile.cc

@@ -17,7 +17,7 @@ static ssize_t callback_read(struct archive * archive, void * _self, const void
     *buffer = self->buffer.data();
 
     try {
-        return self->source->read((char *) self->buffer.data(), 4096);
+        return self->source->read((char *) self->buffer.data(), self->buffer.size());
     } catch (EndOfFile &) {
         return 0;
     } catch (std::exception & err) {
@@ -39,7 +39,7 @@ void TarArchive::check(int err, const std::string & reason)
         throw Error(reason, archive_error_string(this->archive));
 }
 
-TarArchive::TarArchive(Source & source, bool raw) : buffer(4096)
+TarArchive::TarArchive(Source & source, bool raw) : buffer(65536)
 {
     this->archive = archive_read_new();
     this->source = &source;

src/libutil/tarfile.hh

@@ -24,6 +24,7 @@ struct TarArchive {
 
     ~TarArchive();
 };
+
 void unpackTarfile(Source & source, const Path & destDir);
 
 void unpackTarfile(const Path & tarFile, const Path & destDir);

src/libutil/tests/tests.cc

@@ -202,7 +202,7 @@ namespace nix {
     }
 
     TEST(pathExists, bogusPathDoesNotExist) {
-        ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes"));
+        ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes"));
     }
 
     /* ----------------------------------------------------------------------------

src/libutil/util.cc

@@ -266,6 +266,17 @@ bool pathExists(const Path & path)
     return false;
 }
 
+bool pathAccessible(const Path & path)
+{
+    try {
+        return pathExists(path);
+    } catch (SysError & e) {
+        // swallow EPERM
+        if (e.errNo == EPERM) return false;
+        throw;
+    }
+}
+
 
 Path readLink(const Path & path)
 {
@@ -1141,9 +1152,9 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 }
 
 std::string runProgram(Path program, bool searchPath, const Strings & args,
-    const std::optional<std::string> & input)
+    const std::optional<std::string> & input, bool isInteractive)
 {
-    auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
+    auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive});
 
     if (!statusOk(res.first))
         throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
@@ -1193,6 +1204,16 @@ void runProgram2(const RunOptions & options)
     // case), so we can't use it if we alter the environment
     processOptions.allowVfork = !options.environment;
 
+    std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
+    if (options.isInteractive) {
+        logger->pause();
+        resumeLoggerDefer.emplace(
+            []() {
+                logger->resume();
+            }
+        );
+    }
+
     /* Fork. */
     Pid pid = startProcess([&]() {
         if (options.environment)

src/libutil/util.hh

@@ -120,6 +120,14 @@ struct stat lstat(const Path & path);
  */
 bool pathExists(const Path & path);
 
+/**
+ * A version of pathExists that returns false on a permission error.
+ * Useful for inferring default paths across directories that might not
+ * be readable.
+ * @return true iff the given path can be accessed and exists
+ */
+bool pathAccessible(const Path & path);
+
 /**
  * Read the contents (target) of a symbolic link.  The result is not
  * in any way canonicalised.
@@ -415,7 +423,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
  */
 std::string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings(),
-    const std::optional<std::string> & input = {});
+    const std::optional<std::string> & input = {}, bool isInteractive = false);
 
 struct RunOptions
 {
@@ -430,6 +438,7 @@ struct RunOptions
     Source * standardIn = nullptr;
     Sink * standardOut = nullptr;
     bool mergeStderrToStdout = false;
+    bool isInteractive = false;
 };
 
 std::pair<int, std::string> runProgram(RunOptions && options);

src/nix-build/nix-build.cc

@@ -84,7 +84,6 @@ static void main_nix_build(int argc, char * * argv)
     auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO);
     Strings attrPaths;
     Strings left;
-    RepairFlag repair = NoRepair;
     BuildMode buildMode = bmNormal;
     bool readStdin = false;
 
@@ -169,11 +168,6 @@ static void main_nix_build(int argc, char * * argv)
         else if (*arg == "--dry-run")
             dryRun = true;
 
-        else if (*arg == "--repair") {
-            repair = Repair;
-            buildMode = bmRepair;
-        }
-
         else if (*arg == "--run-env") // obsolete
             runEnv = true;
 
@@ -249,7 +243,8 @@ static void main_nix_build(int argc, char * * argv)
     auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store;
 
     auto state = std::make_unique<EvalState>(myArgs.searchPath, evalStore, store);
-    state->repair = repair;
+    state->repair = myArgs.repair;
+    if (myArgs.repair) buildMode = bmRepair;
 
     auto autoArgs = myArgs.getAutoArgs(*state);
 
@@ -289,7 +284,7 @@ static void main_nix_build(int argc, char * * argv)
     else
         for (auto i : left) {
             if (fromArgs)
-                exprs.push_back(state->parseExprFromString(std::move(i), absPath(".")));
+                exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath(CanonPath::fromCwd())));
             else {
                 auto absolute = i;
                 try {
@@ -385,7 +380,9 @@ static void main_nix_build(int argc, char * * argv)
         if (!shell) {
 
             try {
-                auto expr = state->parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
+                auto expr = state->parseExprFromString(
+                    "(import <nixpkgs> {}).bashInteractive",
+                    state->rootPath(CanonPath::fromCwd()));
 
                 Value v;
                 state->eval(expr, v);