Merge remote-tracking branch 'upstream/master' into best-effort-supplementary-groups

src/libcmd/installable-attr-path.cc

@@ -46,7 +46,15 @@ std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state)
 
 DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
 {
-    auto v = toValue(*state).first;
+    auto [v, pos] = toValue(*state);
+
+    if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
+        *v,
+        pos,
+        fmt("while evaluating the attribute '%s'", attrPath)))
+    {
+        return { *derivedPathWithInfo };
+    }
 
     Bindings & autoArgs = *cmd.getAutoArgs(*state);
 

src/libcmd/installable-flake.cc

@@ -95,31 +95,13 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
         // FIXME: use eval cache?
         auto v = attr->forceValue();
 
-        if (v.type() == nPath) {
-            auto storePath = v.path().fetchToStore(state->store);
-            return {{
-                .path = DerivedPath::Opaque {
-                    .path = std::move(storePath),
-                },
-                .info = make_ref<ExtraPathInfo>(),
-            }};
+        if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
+            v,
+            noPos,
+            fmt("while evaluating the flake output attribute '%s'", attrPath)))
+        {
+            return { *derivedPathWithInfo };
         }
-
-        else if (v.type() == nString) {
-            NixStringContext context;
-            auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath));
-            auto storePath = state->store->maybeParseStorePath(s);
-            if (storePath && context.count(NixStringContextElem::Opaque { .path = *storePath })) {
-                return {{
-                    .path = DerivedPath::Opaque {
-                        .path = std::move(*storePath),
-                    },
-                    .info = make_ref<ExtraPathInfo>(),
-                }};
-            } else
-                throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s);
-        }
-
         else
             throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
     }
@@ -234,7 +216,7 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
         }
     }
 
-    return InstallableValue::nixpkgsFlakeRef();
+    return defaultNixpkgsFlakeRef();
 }
 
 }

src/libcmd/installable-flake.hh

@@ -67,9 +67,22 @@ struct InstallableFlake : InstallableValue
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
-    FlakeRef nixpkgsFlakeRef() const override;
+    FlakeRef nixpkgsFlakeRef() const;
 };
 
+/**
+ * Default flake ref for referring to Nixpkgs. For flakes that don't
+ * have their own Nixpkgs input, or other installables.
+ *
+ * It is a layer violation for Nix to know about Nixpkgs; currently just
+ * `nix develop` does. Be wary of using this /
+ * `InstallableFlake::nixpkgsFlakeRef` more places.
+ */
+static inline FlakeRef defaultNixpkgsFlakeRef()
+{
+    return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
+}
+
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake);

src/libcmd/installable-value.cc

@@ -41,4 +41,26 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
     return ref { castedInstallable };
 }
 
+std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
+{
+    if (v.type() == nPath) {
+        auto storePath = v.path().fetchToStore(state->store);
+        return {{
+            .path = DerivedPath::Opaque {
+                .path = std::move(storePath),
+            },
+            .info = make_ref<ExtraPathInfo>(),
+        }};
+    }
+
+    else if (v.type() == nString) {
+        return {{
+            .path = state->coerceToDerivedPath(pos, v, errorCtx),
+            .info = make_ref<ExtraPathInfo>(),
+        }};
+    }
+
+    else return std::nullopt;
+}
+
 }

src/libcmd/installable-value.hh

@@ -96,13 +96,26 @@ struct InstallableValue : Installable
 
     UnresolvedApp toApp(EvalState & state);
 
-    virtual FlakeRef nixpkgsFlakeRef() const
-    {
-        return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
-    }
-
     static InstallableValue & require(Installable & installable);
     static ref<InstallableValue> require(ref<Installable> installable);
+
+protected:
+
+    /**
+     * Handles either a plain path, or a string with a single string
+     * context elem in the right format. The latter case is handled by
+     * `EvalState::coerceToDerivedPath()`; see it for details.
+     *
+     * @param v Value that is hopefully a string or path per the above.
+     *
+     * @param pos Position of value to aid with diagnostics.
+     *
+     * @param errorCtx Arbitrary message for use in potential error message when something is wrong with `v`.
+     *
+     * @result A derived path (with empty info, for now) if the value
+     * matched the above criteria.
+     */
+    std::optional<DerivedPathWithInfo> trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx);
 };
 
 }

src/libexpr/eval.cc

@@ -94,7 +94,6 @@ RootValue allocRootValue(Value * v)
 #endif
 }
 
-
 void Value::print(const SymbolTable & symbols, std::ostream & str,
     std::set<const void *> * seen) const
 {
@@ -1048,6 +1047,27 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
 }
 
 
+void EvalState::mkOutputString(
+    Value & value,
+    const StorePath & drvPath,
+    const std::string outputName,
+    std::optional<StorePath> optOutputPath)
+{
+    value.mkString(
+        optOutputPath
+            ? store->printStorePath(*std::move(optOutputPath))
+            /* Downstream we would substitute this for an actual path once
+               we build the floating CA derivation */
+            : downstreamPlaceholder(*store, drvPath, outputName),
+        NixStringContext {
+            NixStringContextElem::Built {
+                .drvPath = drvPath,
+                .output = outputName,
+            }
+        });
+}
+
+
 /* Create a thunk for the delayed computation of the given expression
    in the given environment.  But if the expression is a variable,
    then look it up right away.  This significantly reduces the number
@@ -2298,6 +2318,80 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
 }
 
 
+std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
+{
+    NixStringContext context;
+    auto s = forceString(v, context, pos, errorCtx);
+    auto csize = context.size();
+    if (csize != 1)
+        error(
+            "string '%s' has %d entries in its context. It should only have exactly one entry",
+            s, csize)
+            .withTrace(pos, errorCtx).debugThrow<EvalError>();
+    auto derivedPath = std::visit(overloaded {
+        [&](NixStringContextElem::Opaque && o) -> DerivedPath {
+            return DerivedPath::Opaque {
+                .path = std::move(o.path),
+            };
+        },
+        [&](NixStringContextElem::DrvDeep &&) -> DerivedPath {
+            error(
+                "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
+                s).withTrace(pos, errorCtx).debugThrow<EvalError>();
+        },
+        [&](NixStringContextElem::Built && b) -> DerivedPath {
+            return DerivedPath::Built {
+                .drvPath = std::move(b.drvPath),
+                .outputs = OutputsSpec::Names { std::move(b.output) },
+            };
+        },
+    }, ((NixStringContextElem &&) *context.begin()).raw());
+    return {
+        std::move(derivedPath),
+        std::move(s),
+    };
+}
+
+
+DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
+{
+    auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx);
+    auto s = s_;
+    std::visit(overloaded {
+        [&](const DerivedPath::Opaque & o) {
+            auto sExpected = store->printStorePath(o.path);
+            if (s != sExpected)
+                error(
+                    "path string '%s' has context with the different path '%s'",
+                    s, sExpected)
+                    .withTrace(pos, errorCtx).debugThrow<EvalError>();
+        },
+        [&](const DerivedPath::Built & b) {
+            // TODO need derived path with single output to make this
+            // total. Will add as part of RFC 92 work and then this is
+            // cleaned up.
+            auto output = *std::get<OutputsSpec::Names>(b.outputs).begin();
+
+            auto drv = store->readDerivation(b.drvPath);
+            auto i = drv.outputs.find(output);
+            if (i == drv.outputs.end())
+                throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output);
+            auto optOutputPath = i->second.path(*store, drv.name, output);
+            // This is testing for the case of CA derivations
+            auto sExpected = optOutputPath
+                ? store->printStorePath(*optOutputPath)
+                : downstreamPlaceholder(*store, b.drvPath, output);
+            if (s != sExpected)
+                error(
+                    "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
+                    s, output, store->printStorePath(b.drvPath), sExpected)
+                    .withTrace(pos, errorCtx).debugThrow<EvalError>();
+        }
+    }, derivedPath.raw());
+    return derivedPath;
+}
+
+
 bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
 {
     forceValue(v1, noPos);

src/libexpr/eval.hh

@@ -21,6 +21,7 @@ namespace nix {
 class Store;
 class EvalState;
 class StorePath;
+struct DerivedPath;
 enum RepairFlag : bool;
 
 
@@ -473,6 +474,28 @@ public:
      */
     StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
 
+    /**
+     * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only.
+     */
+    std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
+
+    /**
+     * Coerce to `DerivedPath`.
+     *
+     * Must be a string which is either a literal store path or a
+     * "placeholder (see `downstreamPlaceholder()`).
+     *
+     * Even more importantly, the string context must be exactly one
+     * element, which is either a `NixStringContextElem::Opaque` or
+     * `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep`
+     * is not permitted).
+     *
+     * The string is parsed based on the context --- the context is the
+     * source of truth, and ultimately tells us what we want, and then
+     * we ensure the string corresponds to it.
+     */
+    DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
+
 public:
 
     /**
@@ -576,12 +599,37 @@ public:
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, PosIdx pos);
 
-    /* Create a string representing a store path.
-
-       The string is the printed store path with a context containing a single
-       `Opaque` element of that store path. */
+    /**
+     * Create a string representing a store path.
+     *
+     * The string is the printed store path with a context containing a single
+     * `NixStringContextElem::Opaque` element of that store path.
+     */
     void mkStorePathString(const StorePath & storePath, Value & v);
 
+    /**
+     * Create a string representing a `DerivedPath::Built`.
+     *
+     * The string is the printed store path with a context containing a single
+     * `NixStringContextElem::Built` element of the drv path and output name.
+     *
+     * @param value Value we are settings
+     *
+     * @param drvPath Path the drv whose output we are making a string for
+     *
+     * @param outputName Name of the output
+     *
+     * @param optOutputPath Optional output path for that string. Must
+     * be passed if and only if output store object is input-addressed.
+     * Will be printed to form string if passed, otherwise a placeholder
+     * will be used (see `downstreamPlaceholder()`).
+     */
+    void mkOutputString(
+        Value & value,
+        const StorePath & drvPath,
+        const std::string outputName,
+        std::optional<StorePath> optOutputPath);
+
     void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
     /**

src/libexpr/primops.cc

@@ -129,40 +129,31 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
     }
 }
 
-/* Add and attribute to the given attribute map from the output name to
-   the output path, or a placeholder.
-
-   Where possible the path is used, but for floating CA derivations we
-   may not know it. For sake of determinism we always assume we don't
-   and instead put in a place holder. In either case, however, the
-   string context will contain the drv path and output name, so
-   downstream derivations will have the proper dependency, and in
-   addition, before building, the placeholder will be rewritten to be
-   the actual path.
-
-   The 'drv' and 'drvPath' outputs must correspond. */
+/**
+ * Add and attribute to the given attribute map from the output name to
+ * the output path, or a placeholder.
+ *
+ * Where possible the path is used, but for floating CA derivations we
+ * may not know it. For sake of determinism we always assume we don't
+ * and instead put in a place holder. In either case, however, the
+ * string context will contain the drv path and output name, so
+ * downstream derivations will have the proper dependency, and in
+ * addition, before building, the placeholder will be rewritten to be
+ * the actual path.
+ *
+ * The 'drv' and 'drvPath' outputs must correspond.
+ */
 static void mkOutputString(
     EvalState & state,
     BindingsBuilder & attrs,
     const StorePath & drvPath,
-    const BasicDerivation & drv,
     const std::pair<std::string, DerivationOutput> & o)
 {
-    auto optOutputPath = o.second.path(*state.store, drv.name, o.first);
-    attrs.alloc(o.first).mkString(
-        optOutputPath
-            ? state.store->printStorePath(*optOutputPath)
-            /* Downstream we would substitute this for an actual path once
-               we build the floating CA derivation */
-            /* FIXME: we need to depend on the basic derivation, not
-               derivation */
-            : downstreamPlaceholder(*state.store, drvPath, o.first),
-        NixStringContext {
-            NixStringContextElem::Built {
-                .drvPath = drvPath,
-                .output = o.first,
-            }
-        });
+    state.mkOutputString(
+        attrs.alloc(o.first),
+        drvPath,
+        o.first,
+        o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first));
 }
 
 /* Load and evaluate an expression from path specified by the
@@ -193,7 +184,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
         state.mkList(outputsVal, drv.outputs.size());
 
         for (const auto & [i, o] : enumerate(drv.outputs)) {
-            mkOutputString(state, attrs, *storePath, drv, o);
+            mkOutputString(state, attrs, *storePath, o);
             (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first);
         }
 
@@ -1100,7 +1091,7 @@ drvName, Bindings * attrs, Value & v)
     bool isImpure = false;
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
-    std::optional<FileIngestionMethod> ingestionMethod;
+    std::optional<ContentAddressMethod> ingestionMethod;
 
     StringSet outputs;
     outputs.insert("out");
@@ -1113,7 +1104,10 @@ drvName, Bindings * attrs, Value & v)
         auto handleHashMode = [&](const std::string_view s) {
             if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
-            else
+            else if (s == "text") {
+                experimentalFeatureSettings.require(Xp::DynamicDerivations);
+                ingestionMethod = TextIngestionMethod {};
+            } else
                 state.debugThrowLastTrace(EvalError({
                     .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
                     .errPos = state.positions[noPos]
@@ -1280,11 +1274,16 @@ drvName, Bindings * attrs, Value & v)
         }));
 
     /* Check whether the derivation name is valid. */
-    if (isDerivation(drvName))
+    if (isDerivation(drvName) &&
+        !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
+          outputs.size() == 1 &&
+          *(outputs.begin()) == "out"))
+    {
         state.debugThrowLastTrace(EvalError({
-            .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
+            .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension),
             .errPos = state.positions[noPos]
         }));
+    }
 
     if (outputHash) {
         /* Handle fixed-output derivations.
@@ -1300,21 +1299,15 @@ drvName, Bindings * attrs, Value & v)
         auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
 
         auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
-        auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
-            .hash = {
-                .method = method,
-                .hash = h,
-            },
-            .references = {},
-        });
-        drv.env["out"] = state.store->printStorePath(outPath);
-        drv.outputs.insert_or_assign("out",
-            DerivationOutput::CAFixed {
-                .hash = FixedOutputHash {
-                    .method = method,
-                    .hash = std::move(h),
-                },
-            });
+
+        DerivationOutput::CAFixed dof {
+            .ca = ContentAddress::fromParts(
+                std::move(method),
+                std::move(h)),
+        };
+
+        drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
+        drv.outputs.insert_or_assign("out", std::move(dof));
     }
 
     else if (contentAddressed || isImpure) {
@@ -1332,13 +1325,13 @@ drvName, Bindings * attrs, Value & v)
             if (isImpure)
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::Impure {
-                        .method = method,
+                        .method = method.raw,
                         .hashType = ht,
                     });
             else
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::CAFloating {
-                        .method = method,
+                        .method = method.raw,
                         .hashType = ht,
                     });
         }
@@ -1403,7 +1396,7 @@ drvName, Bindings * attrs, Value & v)
         NixStringContextElem::DrvDeep { .drvPath = drvPath },
     });
     for (auto & i : drv.outputs)
-        mkOutputString(state, result, drvPath, drv, i);
+        mkOutputString(state, result, drvPath, i);
 
     v.mkAttrs(result);
 }

src/libexpr/print.cc

@@ -1,4 +1,5 @@
 #include "print.hh"
+#include <unordered_set>
 
 namespace nix {
 
@@ -25,11 +26,26 @@ printLiteralBool(std::ostream & str, bool boolean)
     return str;
 }
 
+// Returns `true' is a string is a reserved keyword which requires quotation
+// when printing attribute set field names.
+//
+// This list should generally be kept in sync with `./lexer.l'.
+// You can test if a keyword needs to be added by running:
+//   $ nix eval --expr '{ <KEYWORD> = 1; }'
+// For example `or' doesn't need to be quoted.
+bool isReservedKeyword(const std::string_view str)
+{
+    static const std::unordered_set<std::string_view> reservedKeywords = {
+        "if", "then", "else", "assert", "with", "let", "in", "rec", "inherit"
+    };
+    return reservedKeywords.contains(str);
+}
+
 std::ostream &
 printIdentifier(std::ostream & str, std::string_view s) {
     if (s.empty())
         str << "\"\"";
-    else if (s == "if") // FIXME: handle other keywords
+    else if (isReservedKeyword(s))
         str << '"' << s << '"';
     else {
         char c = s[0];
@@ -50,10 +66,10 @@ printIdentifier(std::ostream & str, std::string_view s) {
     return str;
 }
 
-// FIXME: keywords
 static bool isVarName(std::string_view s)
 {
     if (s.size() == 0) return false;
+    if (isReservedKeyword(s)) return false;
     char c = s[0];
     if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false;
     for (auto & i : s)

src/libexpr/print.hh

@@ -35,6 +35,12 @@ namespace nix {
      */
     std::ostream & printAttributeName(std::ostream & o, std::string_view s);
 
+    /**
+     * Returns `true' is a string is a reserved keyword which requires quotation
+     * when printing attribute set field names.
+     */
+    bool isReservedKeyword(const std::string_view str);
+
     /**
      * Print a string as an identifier in the Nix expression language syntax.
      *

src/libexpr/tests/derived-path.cc

@@ -0,0 +1,65 @@
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+#include <rapidcheck/gtest.h>
+
+#include "tests/derived-path.hh"
+#include "tests/libexpr.hh"
+
+namespace nix {
+
+// Testing of trivial expressions
+class DerivedPathExpressionTest : public LibExprTest {};
+
+// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is
+// no a real fixture.
+//
+// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args
+TEST_F(DerivedPathExpressionTest, force_init)
+{
+}
+
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathExpressionTest,
+    prop_opaque_path_round_trip,
+    (const DerivedPath::Opaque & o))
+{
+    auto * v = state.allocValue();
+    state.mkStorePathString(o.path, *v);
+    auto d = state.coerceToDerivedPath(noPos, *v, "");
+    RC_ASSERT(DerivedPath { o } == d);
+}
+
+// TODO use DerivedPath::Built for parameter once it supports a single output
+// path only.
+
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathExpressionTest,
+    prop_built_path_placeholder_round_trip,
+    (const StorePath & drvPath, const StorePathName & outputName))
+{
+    auto * v = state.allocValue();
+    state.mkOutputString(*v, drvPath, outputName.name, std::nullopt);
+    auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
+    DerivedPath::Built b {
+        .drvPath = drvPath,
+        .outputs = OutputsSpec::Names { outputName.name },
+    };
+    RC_ASSERT(DerivedPath { b } == d);
+}
+
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathExpressionTest,
+    prop_built_path_out_path_round_trip,
+    (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath))
+{
+    auto * v = state.allocValue();
+    state.mkOutputString(*v, drvPath, outputName.name, outPath);
+    auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
+    DerivedPath::Built b {
+        .drvPath = drvPath,
+        .outputs = OutputsSpec::Names { outputName.name },
+    };
+    RC_ASSERT(DerivedPath { b } == d);
+}
+
+} /* namespace nix */

src/libexpr/tests/value/context.cc

@@ -95,13 +95,15 @@ Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitra
 
 Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
 {
-    switch (*gen::inRange<uint8_t>(0, 2)) {
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {
     case 0:
         return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>());
     case 1:
         return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>());
-    default:
+    case 2:
         return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>());
+    default:
+        assert(false);
     }
 }
 

src/libstore/build/derivation-goal.cc

@@ -274,11 +274,13 @@ void DerivationGoal::haveDerivation()
                         )
                     )
                 );
-            else
+            else {
+                auto * cap = getDerivationCA(*drv);
                 addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
                     status.known->path,
                     buildMode == bmRepair ? Repair : NoRepair,
-                    getDerivationCA(*drv))));
+                    cap ? std::optional { *cap } : std::nullopt)));
+            }
         }
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -1020,43 +1022,33 @@ void DerivationGoal::resolvedFinished()
 
         StorePathSet outputPaths;
 
-        // `wantedOutputs` might merely indicate “all the outputs”
-        auto realWantedOutputs = std::visit(overloaded {
-            [&](const OutputsSpec::All &) {
-                return resolvedDrv.outputNames();
-            },
-            [&](const OutputsSpec::Names & names) {
-                return static_cast<std::set<std::string>>(names);
-            },
-        }, wantedOutputs.raw());
-
-        for (auto & wantedOutput : realWantedOutputs) {
-            auto initialOutput = get(initialOutputs, wantedOutput);
-            auto resolvedHash = get(resolvedHashes, wantedOutput);
+        for (auto & outputName : resolvedDrv.outputNames()) {
+            auto initialOutput = get(initialOutputs, outputName);
+            auto resolvedHash = get(resolvedHashes, outputName);
             if ((!initialOutput) || (!resolvedHash))
                 throw Error(
                     "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
-                    worker.store.printStorePath(drvPath), wantedOutput);
+                    worker.store.printStorePath(drvPath), outputName);
 
             auto realisation = [&]{
-              auto take1 = get(resolvedResult.builtOutputs, wantedOutput);
+              auto take1 = get(resolvedResult.builtOutputs, outputName);
               if (take1) return *take1;
 
               /* The above `get` should work. But sateful tracking of
                  outputs in resolvedResult, this can get out of sync with the
                  store, which is our actual source of truth. For now we just
                  check the store directly if it fails. */
-              auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput });
+              auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName });
               if (take2) return *take2;
 
               throw Error(
                   "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
-                  worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
+                  worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
             }();
 
             if (drv->type().isPure()) {
                 auto newRealisation = realisation;
-                newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
+                newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
                 newRealisation.signatures.clear();
                 if (!drv->type().isFixed())
                     newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
@@ -1064,7 +1056,7 @@ void DerivationGoal::resolvedFinished()
                 worker.store.registerDrvOutput(newRealisation);
             }
             outputPaths.insert(realisation.outPath);
-            builtOutputs.emplace(wantedOutput, realisation);
+            builtOutputs.emplace(outputName, realisation);
         }
 
         runPostBuildHook(
@@ -1406,7 +1398,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
                 );
             }
         }
-        if (info.wanted && info.known && info.known->isValid())
+        if (info.known && info.known->isValid())
             validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
     }
 
@@ -1457,8 +1449,9 @@ void DerivationGoal::done(
     mcRunningBuilds.reset();
 
     if (buildResult.success()) {
-        assert(!builtOutputs.empty());
-        buildResult.builtOutputs = std::move(builtOutputs);
+        auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs));
+        assert(!wantedBuiltOutputs.empty());
+        buildResult.builtOutputs = std::move(wantedBuiltOutputs);
         if (status == BuildResult::Built)
             worker.doneBuilds++;
     } else {

src/libstore/build/derivation-goal.hh

@@ -306,15 +306,13 @@ struct DerivationGoal : public Goal
      * Update 'initialOutputs' to determine the current status of the
      * outputs of the derivation. Also returns a Boolean denoting
      * whether all outputs are valid and non-corrupt, and a
-     * 'SingleDrvOutputs' structure containing the valid and wanted
-     * outputs.
+     * 'SingleDrvOutputs' structure containing the valid outputs.
      */
     std::pair<bool, SingleDrvOutputs> checkPathValidity();
 
     /**
      * Aborts if any output is not valid or corrupt, and otherwise
-     * returns a 'SingleDrvOutputs' structure containing the wanted
-     * outputs.
+     * returns a 'SingleDrvOutputs' structure containing all outputs.
      */
     SingleDrvOutputs assertPathValidity();
 
@@ -335,6 +333,8 @@ struct DerivationGoal : public Goal
     void waiteeDone(GoalPtr waitee, ExitCode result) override;
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
+
+    JobCategory jobCategory() override { return JobCategory::Build; };
 };
 
 MakeError(NotDeterministic, BuildError);

src/libstore/build/drv-output-substitution-goal.hh

@@ -21,7 +21,7 @@ class Worker;
 class DrvOutputSubstitutionGoal : public Goal {
 
     /**
-     * The drv output we're trying to substitue
+     * The drv output we're trying to substitute
      */
     DrvOutput id;
 
@@ -72,6 +72,8 @@ public:
 
     void work() override;
     void handleEOF(int fd) override;
+
+    JobCategory jobCategory() override { return JobCategory::Substitution; };
 };
 
 }

src/libstore/build/goal.hh

@@ -34,6 +34,17 @@ typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
  */
 typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
 
+/**
+ * Used as a hint to the worker on how to schedule a particular goal. For example,
+ * builds are typically CPU- and memory-bound, while substitutions are I/O bound.
+ * Using this information, the worker might decide to schedule more or fewer goals
+ * of each category in parallel.
+ */
+enum struct JobCategory {
+    Build,
+    Substitution,
+};
+
 struct Goal : public std::enable_shared_from_this<Goal>
 {
     typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
@@ -150,6 +161,8 @@ public:
     void amDone(ExitCode result, std::optional<Error> ex = {});
 
     virtual void cleanup() { }
+
+    virtual JobCategory jobCategory() = 0;
 };
 
 void addToWeakGoals(WeakGoals & goals, GoalPtr p);

src/libstore/build/local-derivation-goal.cc

@@ -2421,37 +2421,51 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 throw BuildError(
                     "output path %1% without valid stats info",
                     actualPath);
-            if (outputHash.method == FileIngestionMethod::Flat) {
+            if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
+                outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
+            {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
                     throw BuildError(
                         "output path '%1%' should be a non-executable regular file "
-                        "since recursive hashing is not enabled (outputHashMode=flat)",
+                        "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
                         actualPath);
             }
             rewriteOutput();
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
             HashModuloSink caSink { outputHash.hashType, oldHashPart };
-            switch (outputHash.method) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(actualPath, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(actualPath, caSink);
-                break;
-            }
+            std::visit(overloaded {
+                [&](const TextIngestionMethod &) {
+                    readFile(actualPath, caSink);
+                },
+                [&](const FileIngestionMethod & m2) {
+                    switch (m2) {
+                    case FileIngestionMethod::Recursive:
+                        dumpPath(actualPath, caSink);
+                        break;
+                    case FileIngestionMethod::Flat:
+                        readFile(actualPath, caSink);
+                        break;
+                    }
+                },
+            }, outputHash.method.raw);
             auto got = caSink.finish().first;
+
+            auto optCA = ContentAddressWithReferences::fromPartsOpt(
+                outputHash.method,
+                std::move(got),
+                rewriteRefs());
+            if (!optCA) {
+                // TODO track distinct failure modes separately (at the time of
+                // writing there is just one but `nullopt` is unclear) so this
+                // message can't get out of sync.
+                throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
+            }
             ValidPathInfo newInfo0 {
                 worker.store,
                 outputPathName(drv->name, outputName),
-                FixedOutputInfo {
-                    .hash = {
-                        .method = outputHash.method,
-                        .hash = got,
-                    },
-                    .references = rewriteRefs(),
-                },
+                *std::move(optCA),
                 Hash::dummy,
             };
             if (*scratchPath != newInfo0.path) {
@@ -2498,13 +2512,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             },
 
             [&](const DerivationOutput::CAFixed & dof) {
+                auto wanted = dof.ca.getHash();
+
                 auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
-                    .method = dof.hash.method,
-                    .hashType = dof.hash.hash.type,
+                    .method = dof.ca.getMethod(),
+                    .hashType = wanted.type,
                 });
 
                 /* Check wanted hash */
-                const Hash & wanted = dof.hash.hash;
                 assert(newInfo0.ca);
                 auto got = newInfo0.ca->getHash();
                 if (wanted != got) {
@@ -2517,6 +2532,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                             wanted.to_string(SRI, true),
                             got.to_string(SRI, true)));
                 }
+                if (!newInfo0.references.empty())
+                    delayedException = std::make_exception_ptr(
+                        BuildError("illegal path references in fixed-output derivation '%s'",
+                            worker.store.printStorePath(drvPath)));
+
                 return newInfo0;
             },
 
@@ -2696,8 +2716,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             signRealisation(thisRealisation);
             worker.store.registerDrvOutput(thisRealisation);
         }
-        if (wantedOutputs.contains(outputName))
-            builtOutputs.emplace(outputName, thisRealisation);
+        builtOutputs.emplace(outputName, thisRealisation);
     }
 
     return builtOutputs;

src/libstore/build/substitution-goal.cc

@@ -200,11 +200,10 @@ void PathSubstitutionGoal::tryToRun()
 {
     trace("trying to run");
 
-    /* Make sure that we are allowed to start a build.  Note that even
-       if maxBuildJobs == 0 (no local builds allowed), we still allow
-       a substituter to run.  This is because substitutions cannot be
-       distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
+    /* Make sure that we are allowed to start a substitution.  Note that even
+       if maxSubstitutionJobs == 0, we still allow a substituter to run. This
+       prevents infinite waiting. */
+    if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
         worker.waitForBuildSlot(shared_from_this());
         return;
     }

src/libstore/build/substitution-goal.hh

@@ -115,6 +115,8 @@ public:
     void handleEOF(int fd) override;
 
     void cleanup() override;
+
+    JobCategory jobCategory() override { return JobCategory::Substitution; };
 };
 
 }

src/libstore/build/worker.cc

@@ -18,6 +18,7 @@ Worker::Worker(Store & store, Store & evalStore)
 {
     /* Debugging: prevent recursive workers. */
     nrLocalBuilds = 0;
+    nrSubstitutions = 0;
     lastWokenUp = steady_time_point::min();
     permanentFailure = false;
     timedOut = false;
@@ -176,6 +177,12 @@ unsigned Worker::getNrLocalBuilds()
 }
 
 
+unsigned Worker::getNrSubstitutions()
+{
+    return nrSubstitutions;
+}
+
+
 void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     bool inBuildSlot, bool respectTimeouts)
 {
@@ -187,7 +194,10 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     child.inBuildSlot = inBuildSlot;
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
-    if (inBuildSlot) nrLocalBuilds++;
+    if (inBuildSlot) {
+        if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
+        else nrLocalBuilds++;
+    }
 }
 
 
@@ -198,8 +208,13 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
     if (i == children.end()) return;
 
     if (i->inBuildSlot) {
-        assert(nrLocalBuilds > 0);
-        nrLocalBuilds--;
+        if (goal->jobCategory() == JobCategory::Substitution) {
+            assert(nrSubstitutions > 0);
+            nrSubstitutions--;
+        } else {
+            assert(nrLocalBuilds > 0);
+            nrLocalBuilds--;
+        }
     }
 
     children.erase(i);
@@ -220,7 +235,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
 void Worker::waitForBuildSlot(GoalPtr goal)
 {
     debug("wait for build slot");
-    if (getNrLocalBuilds() < settings.maxBuildJobs)
+    bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
+    if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
+        (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))
         wakeUp(goal); /* we can do it right away */
     else
         addToWeakGoals(wantingToBuild, goal);

src/libstore/build/worker.hh

@@ -88,11 +88,16 @@ private:
     std::list<Child> children;
 
     /**
-     * Number of build slots occupied.  This includes local builds and
-     * substitutions but not remote builds via the build hook.
+     * Number of build slots occupied.  This includes local builds but does not
+     * include substitutions or remote builds via the build hook.
      */
     unsigned int nrLocalBuilds;
 
+    /**
+     * Number of substitution slots occupied.
+     */
+    unsigned int nrSubstitutions;
+
     /**
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
@@ -220,12 +225,16 @@ public:
     void wakeUp(GoalPtr goal);
 
     /**
-     * Return the number of local build and substitution processes
-     * currently running (but not remote builds via the build
-     * hook).
+     * Return the number of local build processes currently running (but not
+     * remote builds via the build hook).
      */
     unsigned int getNrLocalBuilds();
 
+    /**
+     * Return the number of substitution processes currently running.
+     */
+    unsigned int getNrSubstitutions();
+
     /**
      * Registers a running child process.  `inBuildSlot` means that
      * the process counts towards the jobs limit.

src/libstore/content-address.cc

@@ -21,6 +21,27 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
     }
 }
 
+std::string ContentAddressMethod::renderPrefix() const
+{
+    return std::visit(overloaded {
+        [](TextIngestionMethod) -> std::string { return "text:"; },
+        [](FileIngestionMethod m2) {
+             /* Not prefixed for back compat with things that couldn't produce text before. */
+            return makeFileIngestionPrefix(m2);
+        },
+    }, raw);
+}
+
+ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
+{
+    ContentAddressMethod method = FileIngestionMethod::Flat;
+    if (splitPrefix(m, "r:"))
+        method = FileIngestionMethod::Recursive;
+    else if (splitPrefix(m, "text:"))
+        method = TextIngestionMethod {};
+    return method;
+}
+
 std::string ContentAddress::render() const
 {
     return std::visit(overloaded {
@@ -36,14 +57,14 @@ std::string ContentAddress::render() const
     }, raw);
 }
 
-std::string ContentAddressMethod::render() const
+std::string ContentAddressMethod::render(HashType ht) const
 {
     return std::visit(overloaded {
-        [](const TextHashMethod & th) {
-            return std::string{"text:"} + printHashType(htSHA256);
+        [&](const TextIngestionMethod & th) {
+            return std::string{"text:"} + printHashType(ht);
         },
-        [](const FixedOutputHashMethod & fshm) {
-            return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
+        [&](const FileIngestionMethod & fim) {
+            return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
         }
     }, raw);
 }
@@ -51,7 +72,7 @@ std::string ContentAddressMethod::render() const
 /**
  * Parses content address strings up to the hash.
  */
-static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
+static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
 {
     std::string_view wholeInput { rest };
 
@@ -75,46 +96,47 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
     if (prefix == "text") {
         // No parsing of the ingestion method, "text" only support flat.
         HashType hashType = parseHashType_();
-        if (hashType != htSHA256)
-            throw Error("text content address hash should use %s, but instead uses %s",
-                printHashType(htSHA256), printHashType(hashType));
-        return TextHashMethod {};
+        return {
+            TextIngestionMethod {},
+            std::move(hashType),
+        };
     } else if (prefix == "fixed") {
         // Parse method
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
         HashType hashType = parseHashType_();
-        return FixedOutputHashMethod {
-            .fileIngestionMethod = method,
-            .hashType = std::move(hashType),
+        return {
+            std::move(method),
+            std::move(hashType),
         };
     } else
         throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
 }
 
-ContentAddress ContentAddress::parse(std::string_view rawCa) {
+ContentAddress ContentAddress::parse(std::string_view rawCa)
+{
     auto rest = rawCa;
 
-    ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
+    auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest);
+    auto hashType = hashType_; // work around clang bug
 
-    return std::visit(
-        overloaded {
-            [&](TextHashMethod & thm) {
-                return ContentAddress(TextHash {
-                    .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
-                });
-            },
-            [&](FixedOutputHashMethod & fohMethod) {
-                return ContentAddress(FixedOutputHash {
-                    .method = fohMethod.fileIngestionMethod,
-                    .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
-                });
-            },
-        }, caMethod.raw);
+    return std::visit(overloaded {
+        [&](TextIngestionMethod &) {
+            return ContentAddress(TextHash {
+                .hash = Hash::parseNonSRIUnprefixed(rest, hashType)
+            });
+        },
+        [&](FileIngestionMethod & fim) {
+            return ContentAddress(FixedOutputHash {
+                .method = fim,
+                .hash = Hash::parseNonSRIUnprefixed(rest, hashType),
+            });
+        },
+    }, caMethod.raw);
 }
 
-ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod)
+std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
 {
     std::string asPrefix = std::string{caMethod} + ":";
     // parseContentAddressMethodPrefix takes its argument by reference
@@ -134,6 +156,36 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
     return ca ? ca->render() : "";
 }
 
+ContentAddress ContentAddress::fromParts(
+    ContentAddressMethod method, Hash hash) noexcept
+{
+    return std::visit(overloaded {
+        [&](TextIngestionMethod _) -> ContentAddress {
+            return TextHash {
+                .hash = std::move(hash),
+            };
+        },
+        [&](FileIngestionMethod m2) -> ContentAddress {
+            return FixedOutputHash {
+                .method = std::move(m2),
+                .hash = std::move(hash),
+            };
+        },
+    }, method.raw);
+}
+
+ContentAddressMethod ContentAddress::getMethod() const
+{
+    return std::visit(overloaded {
+        [](const TextHash & th) -> ContentAddressMethod {
+            return TextIngestionMethod {};
+        },
+        [](const FixedOutputHash & fsh) -> ContentAddressMethod {
+            return fsh.method;
+        },
+    }, raw);
+}
+
 const Hash & ContentAddress::getHash() const
 {
     return std::visit(overloaded {
@@ -146,6 +198,12 @@ const Hash & ContentAddress::getHash() const
     }, raw);
 }
 
+std::string ContentAddress::printMethodAlgo() const
+{
+    return getMethod().renderPrefix()
+        + printHashType(getHash().type);
+}
+
 bool StoreReferences::empty() const
 {
     return !self && others.empty();
@@ -156,7 +214,8 @@ size_t StoreReferences::size() const
     return (self ? 1 : 0) + others.size();
 }
 
-ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) {
+ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept
+{
     return std::visit(overloaded {
         [&](const TextHash & h) -> ContentAddressWithReferences {
             return TextInfo {
@@ -173,4 +232,56 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
     }, ca.raw);
 }
 
+std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
+    ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept
+{
+    return std::visit(overloaded {
+        [&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> {
+            if (refs.self)
+                return std::nullopt;
+            return ContentAddressWithReferences {
+                TextInfo {
+                    .hash = { .hash = std::move(hash) },
+                    .references = std::move(refs.others),
+                }
+            };
+        },
+        [&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
+            return ContentAddressWithReferences {
+                FixedOutputInfo {
+                    .hash = {
+                        .method = m2,
+                        .hash = std::move(hash),
+                    },
+                    .references = std::move(refs),
+                }
+            };
+        },
+    }, method.raw);
+}
+
+ContentAddressMethod ContentAddressWithReferences::getMethod() const
+{
+    return std::visit(overloaded {
+        [](const TextInfo & th) -> ContentAddressMethod {
+            return TextIngestionMethod {};
+        },
+        [](const FixedOutputInfo & fsh) -> ContentAddressMethod {
+            return fsh.hash.method;
+        },
+    }, raw);
+}
+
+Hash ContentAddressWithReferences::getHash() const
+{
+    return std::visit(overloaded {
+        [](const TextInfo & th) {
+            return th.hash.hash;
+        },
+        [](const FixedOutputInfo & fsh) {
+            return fsh.hash.hash;
+        },
+    }, raw);
+}
+
 }

src/libstore/content-address.hh

@@ -21,8 +21,14 @@ namespace nix {
  *
  * Somewhat obscure, used by \ref Derivation derivations and
  * `builtins.toFile` currently.
+ *
+ * TextIngestionMethod is identical to FileIngestionMethod::Fixed except that
+ * the former may not have self-references and is tagged `text:${algo}:${hash}`
+ * rather than `fixed:${algo}:${hash}`.  The contents of the store path are
+ * ingested and hashed identically, aside from the slightly different tag and
+ * restriction on self-references.
  */
-struct TextHashMethod : std::monostate { };
+struct TextIngestionMethod : std::monostate { };
 
 /**
  * An enumeration of the main ways we can serialize file system
@@ -46,13 +52,6 @@ enum struct FileIngestionMethod : uint8_t {
  */
 std::string makeFileIngestionPrefix(FileIngestionMethod m);
 
-struct FixedOutputHashMethod {
-    FileIngestionMethod fileIngestionMethod;
-    HashType hashType;
-
-    GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType);
-};
-
 /**
  * An enumeration of all the ways we can serialize file system objects.
  *
@@ -64,8 +63,8 @@ struct FixedOutputHashMethod {
 struct ContentAddressMethod
 {
     typedef std::variant<
-        TextHashMethod,
-        FixedOutputHashMethod
+        TextIngestionMethod,
+        FileIngestionMethod
     > Raw;
 
     Raw raw;
@@ -77,9 +76,36 @@ struct ContentAddressMethod
         : raw(std::forward<decltype(arg)>(arg)...)
     { }
 
-    static ContentAddressMethod parse(std::string_view rawCaMethod);
 
-    std::string render() const;
+    /**
+     * Parse the prefix tag which indicates how the files
+     * were ingested, with the fixed output case not prefixed for back
+     * compat.
+     *
+     * @param [in] m A string that should begin with the prefix.
+     * @param [out] m The remainder of the string after the prefix.
+     */
+    static ContentAddressMethod parsePrefix(std::string_view & m);
+
+    /**
+     * Render the prefix tag which indicates how the files wre ingested.
+     *
+     * The rough inverse of `parsePrefix()`.
+     */
+    std::string renderPrefix() const;
+
+    /**
+     * Parse a content addressing method and hash type.
+     */
+    static std::pair<ContentAddressMethod, HashType> parse(std::string_view rawCaMethod);
+
+    /**
+     * Render a content addressing method and hash type in a
+     * nicer way, prefixing both cases.
+     *
+     * The rough inverse of `parse()`.
+     */
+    std::string render(HashType ht) const;
 };
 
 
@@ -147,8 +173,9 @@ struct ContentAddress
     { }
 
     /**
-     * Compute the content-addressability assertion (ValidPathInfo::ca) for
-     * paths created by Store::makeFixedOutputPath() / Store::addToStore().
+     * Compute the content-addressability assertion
+     * (`ValidPathInfo::ca`) for paths created by
+     * `Store::makeFixedOutputPath()` / `Store::addToStore()`.
      */
     std::string render() const;
 
@@ -156,9 +183,27 @@ struct ContentAddress
 
     static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
 
+    /**
+     * Create a `ContentAddress` from 2 parts:
+     *
+     * @param method Way ingesting the file system data.
+     *
+     * @param hash Hash of ingested file system data.
+     */
+    static ContentAddress fromParts(
+        ContentAddressMethod method, Hash hash) noexcept;
+
+    ContentAddressMethod getMethod() const;
+
     const Hash & getHash() const;
+
+    std::string printMethodAlgo() const;
 };
 
+/**
+ * Render the `ContentAddress` if it exists to a string, return empty
+ * string otherwise.
+ */
 std::string renderContentAddress(std::optional<ContentAddress> ca);
 
 
@@ -244,10 +289,29 @@ struct ContentAddressWithReferences
     { }
 
     /**
-     * Create a ContentAddressWithReferences from a mere ContentAddress, by
-     * assuming no references in all cases.
+     * Create a `ContentAddressWithReferences` from a mere
+     * `ContentAddress`, by claiming no references.
      */
-    static ContentAddressWithReferences withoutRefs(const ContentAddress &);
+    static ContentAddressWithReferences withoutRefs(const ContentAddress &) noexcept;
+
+    /**
+     * Create a `ContentAddressWithReferences` from 3 parts:
+     *
+     * @param method Way ingesting the file system data.
+     *
+     * @param hash Hash of ingested file system data.
+     *
+     * @param refs References to other store objects or oneself.
+     *
+     * Do note that not all combinations are supported; `nullopt` is
+     * returns for invalid combinations.
+     */
+    static std::optional<ContentAddressWithReferences> fromPartsOpt(
+        ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept;
+
+    ContentAddressMethod getMethod() const;
+
+    Hash getHash() const;
 };
 
 }

src/libstore/daemon.cc

@@ -401,18 +401,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             logger->startWork();
             auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
-                ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr);
+                auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr);
+                auto hashType = hashType_; // work around clang bug
                 FramedSource source(from);
                 // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
                 return std::visit(overloaded {
-                    [&](const TextHashMethod &) {
+                    [&](const TextIngestionMethod &) {
+                        if (hashType != htSHA256)
+                            throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
+                                name, printHashType(hashType));
                         // We could stream this by changing Store
                         std::string contents = source.drain();
                         auto path = store->addTextToStore(name, contents, refs, repair);
                         return store->queryPathInfo(path);
                     },
-                    [&](const FixedOutputHashMethod & fohm) {
-                        auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
+                    [&](const FileIngestionMethod & fim) {
+                        auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
                         return store->queryPathInfo(path);
                     },
                 }, contentAddressMethod.raw);

src/libstore/derivations.cc

@@ -2,6 +2,7 @@
 #include "store-api.hh"
 #include "globals.hh"
 #include "util.hh"
+#include "split.hh"
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
 #include <boost/container/small_vector.hpp>
@@ -35,9 +36,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
 
 StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
 {
-    return store.makeFixedOutputPath(
+    return store.makeFixedOutputPathFromCA(
         outputPathName(drvName, outputName),
-        { hash, {} });
+        ContentAddressWithReferences::withoutRefs(ca));
 }
 
 
@@ -211,29 +212,27 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 
 static DerivationOutput parseDerivationOutput(const Store & store,
-    std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
+    std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
 {
     if (hashAlgo != "") {
-        auto method = FileIngestionMethod::Flat;
-        if (hashAlgo.substr(0, 2) == "r:") {
-            method = FileIngestionMethod::Recursive;
-            hashAlgo = hashAlgo.substr(2);
-        }
+        ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
+        if (method == TextIngestionMethod {})
+            experimentalFeatureSettings.require(Xp::DynamicDerivations);
         const auto hashType = parseHashType(hashAlgo);
-        if (hash == "impure") {
+        if (hashS == "impure") {
             experimentalFeatureSettings.require(Xp::ImpureDerivations);
             assert(pathS == "");
             return DerivationOutput::Impure {
                 .method = std::move(method),
                 .hashType = std::move(hashType),
             };
-        } else if (hash != "") {
+        } else if (hashS != "") {
             validatePath(pathS);
+            auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
             return DerivationOutput::CAFixed {
-                .hash = FixedOutputHash {
-                    .method = std::move(method),
-                    .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
-                },
+                .ca = ContentAddress::fromParts(
+                    std::move(method),
+                    std::move(hash)),
             };
         } else {
             experimentalFeatureSettings.require(Xp::CaDerivations);
@@ -393,12 +392,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
             },
             [&](const DerivationOutput::CAFixed & dof) {
                 s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
-                s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
-                s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
+                s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
+                s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false));
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
+                s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType));
                 s += ','; printUnquotedString(s, "");
             },
             [&](const DerivationOutput::Deferred &) {
@@ -409,7 +408,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
             [&](const DerivationOutputImpure & doi) {
                 // FIXME
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
+                s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
                 s += ','; printUnquotedString(s, "impure");
             }
         }, i.second.raw());
@@ -626,8 +625,8 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         for (const auto & i : drv.outputs) {
             auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
             auto hash = hashString(htSHA256, "fixed:out:"
-                + dof.hash.printMethodAlgo() + ":"
-                + dof.hash.hash.to_string(Base16, false) + ":"
+                + dof.ca.printMethodAlgo() + ":"
+                + dof.ca.getHash().to_string(Base16, false) + ":"
                 + store.printStorePath(dof.path(store, drv.name, i.first)));
             outputHashes.insert_or_assign(i.first, std::move(hash));
         }
@@ -777,12 +776,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
             [&](const DerivationOutput::CAFixed & dof) {
                 out << store.printStorePath(dof.path(store, drv.name, i.first))
-                    << dof.hash.printMethodAlgo()
-                    << dof.hash.hash.to_string(Base16, false);
+                    << dof.ca.printMethodAlgo()
+                    << dof.ca.getHash().to_string(Base16, false);
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 out << ""
-                    << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
+                    << (dof.method.renderPrefix() + printHashType(dof.hashType))
                     << "";
             },
             [&](const DerivationOutput::Deferred &) {
@@ -792,7 +791,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
             [&](const DerivationOutput::Impure & doi) {
                 out << ""
-                    << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
+                    << (doi.method.renderPrefix() + printHashType(doi.hashType))
                     << "impure";
             },
         }, i.second.raw());
@@ -942,7 +941,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
                 envHasRightPath(doia.path, i.first);
             },
             [&](const DerivationOutput::CAFixed & dof) {
-                StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} });
+                auto path = dof.path(store, drvName, i.first);
                 envHasRightPath(path, i.first);
             },
             [&](const DerivationOutput::CAFloating &) {
@@ -971,15 +970,16 @@ nlohmann::json DerivationOutput::toJSON(
         },
         [&](const DerivationOutput::CAFixed & dof) {
             res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
-            res["hashAlgo"] = dof.hash.printMethodAlgo();
-            res["hash"] = dof.hash.hash.to_string(Base16, false);
+            res["hashAlgo"] = dof.ca.printMethodAlgo();
+            res["hash"] = dof.ca.getHash().to_string(Base16, false);
+            // FIXME print refs?
         },
         [&](const DerivationOutput::CAFloating & dof) {
-            res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType);
+            res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType);
         },
         [&](const DerivationOutput::Deferred &) {},
         [&](const DerivationOutput::Impure & doi) {
-            res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType);
+            res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
             res["impure"] = true;
         },
     }, raw());
@@ -998,15 +998,15 @@ DerivationOutput DerivationOutput::fromJSON(
     for (const auto & [key, _] : json)
         keys.insert(key);
 
-    auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> {
+    auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> {
         std::string hashAlgo = json["hashAlgo"];
-        auto method = FileIngestionMethod::Flat;
-        if (hashAlgo.substr(0, 2) == "r:") {
-            method = FileIngestionMethod::Recursive;
-            hashAlgo = hashAlgo.substr(2);
-        }
-        auto hashType = parseHashType(hashAlgo);
-        return { method, hashType };
+        // remaining to parse, will be mutated by parsers
+        std::string_view s = hashAlgo;
+        ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
+        if (method == TextIngestionMethod {})
+            xpSettings.require(Xp::DynamicDerivations);
+        auto hashType = parseHashType(s);
+        return { std::move(method), std::move(hashType) };
     };
 
     if (keys == (std::set<std::string_view> { "path" })) {
@@ -1018,10 +1018,9 @@ DerivationOutput DerivationOutput::fromJSON(
     else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
         auto [method, hashType] = methodAlgo();
         auto dof = DerivationOutput::CAFixed {
-            .hash = {
-                .method = method,
-                .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
-            },
+            .ca = ContentAddress::fromParts(
+                std::move(method),
+                Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)),
         };
         if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
             throw Error("Path doesn't match derivation output");
@@ -1032,8 +1031,8 @@ DerivationOutput DerivationOutput::fromJSON(
         xpSettings.require(Xp::CaDerivations);
         auto [method, hashType] = methodAlgo();
         return DerivationOutput::CAFloating {
-            .method = method,
-            .hashType = hashType,
+            .method = std::move(method),
+            .hashType = std::move(hashType),
         };
     }
 
@@ -1045,7 +1044,7 @@ DerivationOutput DerivationOutput::fromJSON(
         xpSettings.require(Xp::ImpureDerivations);
         auto [method, hashType] = methodAlgo();
         return DerivationOutput::Impure {
-            .method = method,
+            .method = std::move(method),
             .hashType = hashType,
         };
     }

src/libstore/derivations.hh

@@ -36,9 +36,11 @@ struct DerivationOutputInputAddressed
 struct DerivationOutputCAFixed
 {
     /**
-     * hash used for expected hash computation
+     * Method and hash used for expected hash computation.
+     *
+     * References are not allowed by fiat.
      */
-    FixedOutputHash hash;
+    ContentAddress ca;
 
     /**
      * Return the \ref StorePath "store path" corresponding to this output
@@ -48,7 +50,7 @@ struct DerivationOutputCAFixed
      */
     StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 
-    GENERATE_CMP(DerivationOutputCAFixed, me->hash);
+    GENERATE_CMP(DerivationOutputCAFixed, me->ca);
 };
 
 /**
@@ -61,7 +63,7 @@ struct DerivationOutputCAFloating
     /**
      * How the file system objects will be serialized for hashing
      */
-    FileIngestionMethod method;
+    ContentAddressMethod method;
 
     /**
      * How the serialization will be hashed
@@ -88,7 +90,7 @@ struct DerivationOutputImpure
     /**
      * How the file system objects will be serialized for hashing
      */
-    FileIngestionMethod method;
+    ContentAddressMethod method;
 
     /**
      * How the serialization will be hashed
@@ -343,12 +345,14 @@ struct Derivation : BasicDerivation
         Store & store,
         const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
 
-    /* Check that the derivation is valid and does not present any
-       illegal states.
-
-       This is mainly a matter of checking the outputs, where our C++
-       representation supports all sorts of combinations we do not yet
-       allow. */
+    /**
+     * Check that the derivation is valid and does not present any
+     * illegal states.
+     *
+     * This is mainly a matter of checking the outputs, where our C++
+     * representation supports all sorts of combinations we do not yet
+     * allow.
+     */
     void checkInvariants(Store & store, const StorePath & drvPath) const;
 
     Derivation() = default;

src/libstore/globals.hh

@@ -159,6 +159,15 @@ public:
         )",
         {"build-max-jobs"}};
 
+    Setting<unsigned int> maxSubstitutionJobs{
+        this, 16, "max-substitution-jobs",
+        R"(
+          This option defines the maximum number of substitution jobs that Nix
+          will try to run in parallel. The default is `16`. The minimum value
+          one can choose is `1` and lower values will be interpreted as `1`.
+        )",
+        {"substitution-max-jobs"}};
+
     Setting<unsigned int> buildCores{
         this,
         getDefaultCores(),
@@ -991,7 +1000,7 @@ public:
         this, false, "use-xdg-base-directories",
         R"(
           If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`.
-          The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/installation/env-variables.md).
+          The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/command-ref/env-common.md).
 
           [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
 

src/libstore/local.mk

@@ -57,12 +57,6 @@ $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
 $(d)/build.cc:
 
-%.gen.hh: %
-	@echo 'R"foo(' >> $@.tmp
-	$(trace-gen) cat $< >> $@.tmp
-	@echo ')foo"' >> $@.tmp
-	@mv $@.tmp $@
-
 clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
 $(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))

src/libstore/misc.cc

@@ -83,14 +83,15 @@ void Store::computeFSClosure(const StorePath & startPath,
 }
 
 
-std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
+const ContentAddress * getDerivationCA(const BasicDerivation & drv)
 {
     auto out = drv.outputs.find("out");
-    if (out != drv.outputs.end()) {
-        if (const auto * v = std::get_if<DerivationOutput::CAFixed>(&out->second.raw()))
-            return v->hash;
+    if (out == drv.outputs.end())
+        return nullptr;
+    if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second)) {
+        return &dof->ca;
     }
-    return std::nullopt;
+    return nullptr;
 }
 
 void Store::queryMissing(const std::vector<DerivedPath> & targets,
@@ -140,7 +141,13 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
         if (drvState_->lock()->done) return;
 
         SubstitutablePathInfos infos;
-        querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
+        auto * cap = getDerivationCA(*drv);
+        querySubstitutablePathInfos({
+            {
+                outPath,
+                cap ? std::optional { *cap } : std::nullopt,
+            },
+        }, infos);
 
         if (infos.empty()) {
             drvState_->lock()->done = true;

src/libstore/realisation.cc

@@ -136,6 +136,19 @@ size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const
     return good;
 }
 
+
+SingleDrvOutputs filterDrvOutputs(const OutputsSpec& wanted, SingleDrvOutputs&& outputs)
+{
+    SingleDrvOutputs ret = std::move(outputs);
+    for (auto it = ret.begin(); it != ret.end(); ) {
+        if (!wanted.contains(it->first))
+            it = ret.erase(it);
+        else
+            ++it;
+    }
+    return ret;
+}
+
 StorePath RealisedPath::path() const {
     return std::visit([](auto && arg) { return arg.getPath(); }, raw);
 }

src/libstore/realisation.hh

@@ -12,6 +12,7 @@
 namespace nix {
 
 class Store;
+struct OutputsSpec;
 
 /**
  * A general `Realisation` key.
@@ -93,6 +94,14 @@ typedef std::map<std::string, Realisation> SingleDrvOutputs;
  */
 typedef std::map<DrvOutput, Realisation> DrvOutputs;
 
+/**
+ * Filter a SingleDrvOutputs to include only specific output names
+ *
+ * Moves the `outputs` input.
+ */
+SingleDrvOutputs filterDrvOutputs(const OutputsSpec&, SingleDrvOutputs&&);
+
+
 struct OpaquePath {
     StorePath path;
 

src/libstore/remote-store.cc

@@ -597,6 +597,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
     Source & dump,
     std::string_view name,
     ContentAddressMethod caMethod,
+    HashType hashType,
     const StorePathSet & references,
     RepairFlag repair)
 {
@@ -608,7 +609,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
         conn->to
             << wopAddToStore
             << name
-            << caMethod.render();
+            << caMethod.render(hashType);
         worker_proto::write(*this, conn->to, references);
         conn->to << repair;
 
@@ -628,26 +629,29 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
         if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
 
         std::visit(overloaded {
-            [&](const TextHashMethod & thm) -> void {
+            [&](const TextIngestionMethod & thm) -> void {
+                if (hashType != htSHA256)
+                    throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
+                        name, printHashType(hashType));
                 std::string s = dump.drain();
                 conn->to << wopAddTextToStore << name << s;
                 worker_proto::write(*this, conn->to, references);
                 conn.processStderr();
             },
-            [&](const FixedOutputHashMethod & fohm) -> void {
+            [&](const FileIngestionMethod & fim) -> void {
                 conn->to
                     << wopAddToStore
                     << name
-                    << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
-                    << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
-                    << printHashType(fohm.hashType);
+                    << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+                    << (fim == FileIngestionMethod::Recursive ? 1 : 0)
+                    << printHashType(hashType);
 
                 try {
                     conn->to.written = 0;
                     connections->incCapacity();
                     {
                         Finally cleanup([&]() { connections->decCapacity(); });
-                        if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
+                        if (fim == FileIngestionMethod::Recursive) {
                             dump.drainInto(conn->to);
                         } else {
                             std::string contents = dump.drain();
@@ -678,7 +682,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
       FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
 {
-    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
+    return addCAToStore(dump, name, method, hashType, references, repair)->path;
 }
 
 
@@ -778,7 +782,7 @@ StorePath RemoteStore::addTextToStore(
     RepairFlag repair)
 {
     StringSource source(s);
-    return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
+    return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path;
 }
 
 void RemoteStore::registerDrvOutput(const Realisation & info)

src/libstore/remote-store.hh

@@ -78,6 +78,7 @@ public:
         Source & dump,
         std::string_view name,
         ContentAddressMethod caMethod,
+        HashType hashType,
         const StorePathSet & references,
         RepairFlag repair);
 

src/libstore/store-api.hh

@@ -1022,7 +1022,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
  */
 std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
 
-std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
+const ContentAddress * getDerivationCA(const BasicDerivation & drv);
 
 std::map<DrvOutput, StorePath> drvOutputReferences(
     Store & store,

src/libstore/tests/derivation.cc

@@ -26,6 +26,14 @@ class CaDerivationTest : public DerivationTest
     }
 };
 
+class DynDerivationTest : public DerivationTest
+{
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
+    }
+};
+
 class ImpureDerivationTest : public DerivationTest
 {
     void SetUp() override
@@ -66,20 +74,47 @@ TEST_JSON(DerivationTest, inputAddressed,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(DerivationTest, caFixed,
+TEST_JSON(DerivationTest, caFixedFlat,
+    R"({
+        "hashAlgo": "sha256",
+        "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+        "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
+    })",
+    (DerivationOutput::CAFixed {
+        .ca = FixedOutputHash {
+            .method = FileIngestionMethod::Flat,
+            .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
+        },
+    }),
+    "drv-name", "output-name")
+
+TEST_JSON(DerivationTest, caFixedNAR,
     R"({
         "hashAlgo": "r:sha256",
         "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
         "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
     })",
     (DerivationOutput::CAFixed {
-        .hash = {
+        .ca = FixedOutputHash {
             .method = FileIngestionMethod::Recursive,
             .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
         },
     }),
     "drv-name", "output-name")
 
+TEST_JSON(DynDerivationTest, caFixedText,
+    R"({
+        "hashAlgo": "text:sha256",
+        "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+        "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
+    })",
+    (DerivationOutput::CAFixed {
+        .ca = TextHash {
+            .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
+        },
+    }),
+    "drv-name", "output-name")
+
 TEST_JSON(CaDerivationTest, caFloating,
     R"({
         "hashAlgo": "r:sha256"

src/libstore/tests/derived-path.cc

@@ -27,11 +27,13 @@ Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
 
 Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
 {
-    switch (*gen::inRange<uint8_t>(0, 1)) {
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
     case 0:
         return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>());
-    default:
+    case 1:
         return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>());
+    default:
+        assert(false);
     }
 }
 

src/libstore/tests/outputs-spec.cc

@@ -206,15 +206,17 @@ using namespace nix;
 
 Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
 {
-    switch (*gen::inRange<uint8_t>(0, 1)) {
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<OutputsSpec::Raw>)) {
     case 0:
         return gen::just((OutputsSpec) OutputsSpec::All { });
-    default:
+    case 1:
         return gen::just((OutputsSpec) OutputsSpec::Names {
             *gen::nonEmpty(gen::container<StringSet>(gen::map(
                 gen::arbitrary<StorePathName>(),
                 [](StorePathName n) { return n.name; }))),
         });
+    default:
+        assert(false);
     }
 }
 

src/libutil/experimental-features.cc

@@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
+constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",
@@ -199,6 +199,16 @@ constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
             networking.
         )",
     },
+    {
+        .tag = Xp::DynamicDerivations,
+        .name = "dynamic-derivations",
+        .description = R"(
+            Allow the use of a few things related to dynamic derivations:
+
+              - "text hashing" derivation outputs, so we can build .drv
+                files.
+        )",
+    },
 }};
 
 static_assert(

src/libutil/experimental-features.hh

@@ -29,6 +29,7 @@ enum struct ExperimentalFeature
     Cgroups,
     DiscardReferences,
     DaemonTrustOverride,
+    DynamicDerivations,
 };
 
 /**

src/nix/develop.cc

@@ -252,7 +252,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
     throw Error("get-env.sh failed to produce an environment");
 }
 
-struct Common : InstallableValueCommand, MixProfile
+struct Common : InstallableCommand, MixProfile
 {
     std::set<std::string> ignoreVars{
         "BASHOPTS",
@@ -374,7 +374,7 @@ struct Common : InstallableValueCommand, MixProfile
         return res;
     }
 
-    StorePath getShellOutPath(ref<Store> store, ref<InstallableValue> installable)
+    StorePath getShellOutPath(ref<Store> store, ref<Installable> installable)
     {
         auto path = installable->getStorePath();
         if (path && hasSuffix(path->to_string(), "-env"))
@@ -393,7 +393,7 @@ struct Common : InstallableValueCommand, MixProfile
     }
 
     std::pair<BuildEnvironment, std::string>
-    getBuildEnvironment(ref<Store> store, ref<InstallableValue> installable)
+    getBuildEnvironment(ref<Store> store, ref<Installable> installable)
     {
         auto shellOutPath = getShellOutPath(store, installable);
 
@@ -481,7 +481,7 @@ struct CmdDevelop : Common, MixEnvironment
           ;
     }
 
-    void run(ref<Store> store, ref<InstallableValue> installable) override
+    void run(ref<Store> store, ref<Installable> installable) override
     {
         auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable);
 
@@ -538,10 +538,14 @@ struct CmdDevelop : Common, MixEnvironment
             nixpkgsLockFlags.inputOverrides = {};
             nixpkgsLockFlags.inputUpdates = {};
 
+            auto nixpkgs = defaultNixpkgsFlakeRef();
+            if (auto * i = dynamic_cast<const InstallableFlake *>(&*installable))
+                nixpkgs = i->nixpkgsFlakeRef();
+
             auto bashInstallable = make_ref<InstallableFlake>(
                 this,
                 state,
-                installable->nixpkgsFlakeRef(),
+                std::move(nixpkgs),
                 "bashInteractive",
                 DefaultOutputs(),
                 Strings{},
@@ -605,7 +609,7 @@ struct CmdPrintDevEnv : Common, MixJSON
 
     Category category() override { return catUtility; }
 
-    void run(ref<Store> store, ref<InstallableValue> installable) override
+    void run(ref<Store> store, ref<Installable> installable) override
     {
         auto buildEnvironment = getBuildEnvironment(store, installable).first;
 

src/nix/local.mk

@@ -32,3 +32,9 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh
 src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh
 
 src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh
+
+src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md
+	@mkdir -p $$(dirname $@)
+	@cp $< $@
+
+src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh

src/nix/profile.md

@@ -7,100 +7,39 @@ profile is a set of packages that can be installed and upgraded
 independently from each other. Nix profiles are versioned, allowing
 them to be rolled back easily.
 
-# Default profile
-
-The default profile used by `nix profile` is `$HOME/.nix-profile`,
-which, if it does not exist, is created as a symlink to
-`/nix/var/nix/profiles/default` if Nix is invoked by the
-`root` user, or `${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/profile` otherwise.
-
-You can specify another profile location using `--profile` *path*.
-
-# Filesystem layout
-
-Profiles are versioned as follows. When using profile *path*, *path*
-is a symlink to *path*`-`*N*, where *N* is the current *version* of
-the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix
-store. For example:
-
-```console
-$ ls -l ~alice/.local/state/nix/profiles/profile*
-lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile -> profile-7-link
-lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /home/alice/.local/state/nix/profiles/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile
-lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /home/alice/.local/state/nix/profiles/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile
-lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile
-```
-
-Each of these symlinks is a root for the Nix garbage collector.
-
-The contents of the store path corresponding to each version of the
-profile is a tree of symlinks to the files of the installed packages,
-e.g.
-
-```console
-$ ll -R ~eelco/.local/state/nix/profiles/profile-7-link/
-/home/eelco/.local/state/nix/profiles/profile-7-link/:
-total 20
-dr-xr-xr-x 2 root root 4096 Jan  1  1970 bin
--r--r--r-- 2 root root 1402 Jan  1  1970 manifest.json
-dr-xr-xr-x 4 root root 4096 Jan  1  1970 share
-
-/home/eelco/.local/state/nix/profiles/profile-7-link/bin:
-total 20
-lrwxrwxrwx 5 root root 79 Jan  1  1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium
-lrwxrwxrwx 7 root root 87 Jan  1  1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify
-lrwxrwxrwx 3 root root 79 Jan  1  1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us
-
-/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications:
-total 12
-lrwxrwxrwx 4 root root 120 Jan  1  1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
-lrwxrwxrwx 7 root root 110 Jan  1  1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop
-lrwxrwxrwx 3 root root 107 Jan  1  1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop
-
-…
-```
-
-The file `manifest.json` records the provenance of the packages that
-are installed in this version of the profile. It looks like this:
-
-```json
-{
-  "version": 1,
-  "elements": [
-    {
-      "active": true,
-      "attrPath": "legacyPackages.x86_64-linux.zoom-us",
-      "originalUrl": "flake:nixpkgs",
-      "storePaths": [
-        "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927"
-      ],
-      "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a"
-    },
-    …
-  ]
-}
-```
-
-Each object in the array `elements` denotes an installed package and
-has the following fields:
-
-* `originalUrl`: The [flake reference](./nix3-flake.md) specified by
-  the user at the time of installation (e.g. `nixpkgs`). This is also
-  the flake reference that will be used by `nix profile upgrade`.
-
-* `uri`: The locked flake reference to which `originalUrl` resolved.
-
-* `attrPath`: The flake output attribute that provided this
-  package. Note that this is not necessarily the attribute that the
-  user specified, but the one resulting from applying the default
-  attribute paths and prefixes; for instance, `hello` might resolve to
-  `packages.x86_64-linux.hello` and the empty string to
-  `packages.x86_64-linux.default`.
-
-* `storePath`: The paths in the Nix store containing the package.
-
-* `active`: Whether the profile contains symlinks to the files of this
-  package. If set to false, the package is kept in the Nix store, but
-  is not "visible" in the profile's symlink tree.
+# Files
+
+)""
+
+#include "doc/files/profiles.md.gen.hh"
+
+R""(
+
+### Profile compatibility
+
+> **Warning**
+>
+> Once you have used [`nix profile`] you can no longer use [`nix-env`] without first deleting `$XDG_STATE_HOME/nix/profiles/profile`
+
+[`nix-env`]: @docroot@/command-ref/nix-env.md
+[`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md
+
+Once you installed a package with [`nix profile`], you get the following error message when using [`nix-env`]:
+
+```console
+$ nix-env -f '<nixpkgs>' -iA 'hello'
+error: nix-env
+profile '/home/alice/.local/state/nix/profiles/profile' is incompatible with 'nix-env'; please use 'nix profile' instead
+```
+
+To migrate back to `nix-env` you can delete your current profile:
+
+> **Warning**
+>
+> This will delete packages that have been installed before, so you may want to back up this information before running the command.
+
+```console
+ $ rm -rf "${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/profile"
+```
 
 )""