Merge remote-tracking branch 'origin/master' into lazy-trees

doc/manual/book.toml

@@ -10,3 +10,12 @@ git-repository-url = "https://github.com/NixOS/nix"
 [preprocessor.anchors]
 renderers = ["html"]
 command = "jq --from-file doc/manual/anchors.jq"
+
+[output.linkcheck]
+# no Internet during the build (in the sandbox)
+follow-web-links = false
+
+# mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in
+# excessive "Potential incomplete link" warnings. No other kind of warning was
+# produced at the time of writing.
+warning-policy = "ignore"

doc/manual/local.mk

@@ -50,11 +50,16 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@
-	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }'
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }'
+	# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
+	$(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g'
+	@mv $@.tmp $@
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \
+	  | sed -e 's^@docroot@^..^g'>> $@.tmp
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix
@@ -67,7 +72,9 @@ $(d)/conf-file.json: $(bindir)/nix
 
 $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \
+	  | sed -e 's^@docroot@^..^g' >> $@.tmp
 	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
@@ -102,6 +109,12 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 	@touch $@
 
 $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
-	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual
+	$(trace-gen) \
+	  set -euo pipefail; \
+	  RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \
+		  | { grep -Fv "because fragment resolution isn't implemented" || :; }
+	@rm -rf $(DESTDIR)$(docdir)/manual
+	@mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual
+	@rm -rf $(DESTDIR)$(docdir)/manual.tmp
 
 endif

doc/manual/src/architecture/architecture.md

@@ -68,7 +68,7 @@ It can also execute build plans to produce new data, which are made available to
 A build plan itself is a series of *build tasks*, together with their build inputs.
 
 > **Important**
-> A build task in Nix is called [derivation](../glossary#gloss-derivation).
+> A build task in Nix is called [derivation](../glossary.md#gloss-derivation).
 
 Each build task has a special build input executed as *build instructions* in order to perform the build.
 The result of a build task can be input to another build task.

doc/manual/src/command-ref/env-common.md

@@ -11,7 +11,7 @@ Most Nix commands interpret the following environment variables:
     expressions using [paths](../language/values.md#type-path)
     enclosed in angle brackets (i.e., `<path>`),
     e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the
-    [`-I` option](./opt-common#opt-I).
+    [`-I` option](./opt-common.md#opt-I).
 
   - [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\
     Normally, the Nix store directory (typically `/nix/store`) is not

doc/manual/src/command-ref/nix-copy-closure.md

@@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`.
   - `--include-outputs`\
     Also copy the outputs of [store derivation]s included in the closure.
 
-    [store derivation]: ../../glossary.md#gloss-store-derivation
+    [store derivation]: ../glossary.md#gloss-store-derivation
 
   - `--use-substitutes` / `-s`\
     Attempt to download missing paths on the target machine using Nix’s

doc/manual/src/contributing/hacking.md

@@ -249,3 +249,36 @@ search/replaced in it for each new build.
 The installer now supports a `--tarball-url-prefix` flag which _may_ have
 solved this need?
 -->
+
+### Checking links in the manual
+
+The build checks for broken internal links.
+This happens late in the process, so `nix build` is not suitable for iterating.
+To build the manual incrementally, run:
+
+```console
+make html -j $NIX_BUILD_CORES
+```
+
+In order to reflect changes to the [Makefile], clear all generated files before re-building:
+
+[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk
+
+```console
+rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES
+```
+
+[`mdbook-linkcheck`] does not implement checking [URI fragments] yet.
+
+[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck
+[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment
+
+#### `@docroot@` variable
+
+`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own.
+
+If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory.
+
+If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it.
+See existing `@docroot@` logic in the [Makefile].
+Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`.

flake.nix

@@ -96,6 +96,7 @@
             buildPackages.flex
             (lib.getBin buildPackages.lowdown-nix)
             buildPackages.mdbook
+            buildPackages.mdbook-linkcheck
             buildPackages.autoconf-archive
             buildPackages.autoreconfHook
             buildPackages.pkg-config

src/libcmd/common-eval-args.cc

@@ -36,8 +36,8 @@ MixEvalArgs::MixEvalArgs()
         .shortName = 'I',
         .description = R"(
   Add *path* to the Nix search path. The Nix search path is
-  initialized from the colon-separated [`NIX_PATH`](./env-common.md#env-NIX_PATH) environment
+  initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment
-  variable, and is used to look up the location of Nix expressions using [paths](../language/values.md#type-path) enclosed in angle
+  variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle
   brackets (i.e., `<nixpkgs>`).
 
   For instance, passing

src/libcmd/installables.hh

@@ -2,7 +2,7 @@
 
 #include "util.hh"
 #include "path.hh"
-#include "path-with-outputs.hh"
+#include "outputs-spec.hh"
 #include "derived-path.hh"
 #include "eval.hh"
 #include "store-api.hh"
@@ -20,7 +20,7 @@ namespace eval_cache { class EvalCache; class AttrCursor; }
 
 struct App
 {
-    std::vector<StorePathWithOutputs> context;
+    std::vector<DerivedPath> context;
     Path program;
     // FIXME: add args, sandbox settings, metadata, ...
 };

src/libexpr/eval-cache.cc

@@ -300,7 +300,7 @@ struct AttrDb
                 NixStringContext context;
                 if (!queryAttribute.isNull(3))
                     for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(cfg, s));
+                        context.push_back(NixStringContextElem::parse(cfg, s));
                 return {{rowId, string_t{queryAttribute.getStr(2), context}}};
             }
             case AttrType::Bool:
@@ -594,7 +594,18 @@ string_t AttrCursor::getStringWithContext()
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 bool valid = true;
                 for (auto & c : s->second) {
-                    if (!root->state.store->isValidPath(c.first)) {
+                    const StorePath & path = std::visit(overloaded {
+                        [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
+                            return d.drvPath;
+                        },
+                        [&](const NixStringContextElem::Built & b) -> const StorePath & {
+                            return b.drvPath;
+                        },
+                        [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
+                            return o.path;
+                        },
+                    }, c.raw());
+                    if (!root->state.store->isValidPath(path)) {
                         valid = false;
                         break;
                     }

src/libexpr/eval.cc

@@ -2045,27 +2045,6 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
 }
 
 
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-NixStringContextElem decodeContext(const Store & store, std::string_view s)
-{
-    if (s.at(0) == '!') {
-        size_t index = s.find("!", 1);
-        return {
-            store.parseStorePath(s.substr(index + 1)),
-            std::string(s.substr(1, index - 1)),
-        };
-    } else
-        return {
-            store.parseStorePath(
-                s.at(0) == '/'
-                ? s
-                : s.substr(1)),
-            "",
-        };
-}
-
-
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -2080,7 +2059,7 @@ NixStringContext Value::getContext(const Store & store)
     assert(internalType == tString);
     if (string.context)
         for (const char * * p = string.context; *p; ++p)
-            res.push_back(decodeContext(store, *p));
+            res.push_back(NixStringContextElem::parse(store, *p));
     return res;
 }
 

src/libexpr/eval.hh

@@ -568,10 +568,6 @@ struct DebugTraceStacker {
 std::string_view showType(ValueType type);
 std::string showType(const Value & v);
 
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-NixStringContextElem decodeContext(const Store & store, std::string_view s);
-
 /* If `path' refers to a directory, then append "/default.nix". */
 SourcePath resolveExprPath(const SourcePath & path);
 

src/libexpr/flake/flakeref.hh

@@ -3,7 +3,7 @@
 #include "types.hh"
 #include "hash.hh"
 #include "fetchers.hh"
-#include "path-with-outputs.hh"
+#include "outputs-spec.hh"
 
 #include <variant>
 

src/libexpr/local.mk

@@ -6,6 +6,7 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := \
   $(wildcard $(d)/*.cc) \
+  $(wildcard $(d)/value/*.cc) \
   $(wildcard $(d)/primops/*.cc) \
   $(wildcard $(d)/flake/*.cc) \
   $(d)/lexer-tab.cc \
@@ -37,6 +38,8 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
+$(foreach i, $(wildcard src/libexpr/value/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 

src/libexpr/primops.cc

@@ -44,16 +44,32 @@ StringMap EvalState::realiseContext(const PathSet & context)
     std::vector<DerivedPath::Built> drvs;
     StringMap res;
 
-    for (auto & i : context) {
+    for (auto & c_ : context) {
-        auto [ctx, outputName] = decodeContext(*store, i);
+        auto ensureValid = [&](const StorePath & p) {
-        auto ctxS = store->printStorePath(ctx);
+            if (!store->isValidPath(p))
-        if (!store->isValidPath(ctx))
+                debugThrowLastTrace(InvalidPathError(store->printStorePath(p)));
-            debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx)));
+        };
-        if (!outputName.empty() && ctx.isDerivation()) {
+        auto c = NixStringContextElem::parse(*store, c_);
-            drvs.push_back({ctx, {outputName}});
+        std::visit(overloaded {
-        } else {
+            [&](const NixStringContextElem::Built & b) {
-            res.insert_or_assign(ctxS, ctxS);
+                drvs.push_back(DerivedPath::Built {
-        }
+                    .drvPath = b.drvPath,
+                    .outputs = std::set { b.output },
+                });
+                ensureValid(b.drvPath);
+            },
+            [&](const NixStringContextElem::Opaque & o) {
+                auto ctxS = store->printStorePath(o.path);
+                res.insert_or_assign(ctxS, ctxS);
+                ensureValid(o.path);
+            },
+            [&](const NixStringContextElem::DrvDeep & d) {
+                /* Treat same as Opaque */
+                auto ctxS = store->printStorePath(d.drvPath);
+                res.insert_or_assign(ctxS, ctxS);
+                ensureValid(d.drvPath);
+            },
+        }, c.raw());
     }
 
     if (drvs.empty()) return {};
@@ -238,6 +254,7 @@ static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info {
 static RegisterPrimOp primop_import({
     .name = "import",
     .args = {"path"},
+    // TODO turn "normal path values" into link below
     .doc = R"(
       Load, parse and return the Nix expression in the file *path*. If
       *path* is a directory, the file ` default.nix ` in that directory
@@ -251,7 +268,7 @@ static RegisterPrimOp primop_import({
       >
       > Unlike some languages, `import` is a regular function in Nix.
       > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*)
-      > are [normal path values](language-values.md).
+      > are normal [path values](@docroot@/language/values.md#type-path).
 
       A Nix expression loaded by `import` must not contain any *free
       variables* (identifiers that are not defined in the Nix expression
@@ -1182,35 +1199,31 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
     /* Everything in the context of the strings in the derivation
        attributes should be added as dependencies of the resulting
        derivation. */
-    for (auto & path : context) {
+    for (auto & c_ : context) {
-
+        auto c = NixStringContextElem::parse(*state.store, c_);
-        /* Paths marked with `=' denote that the path of a derivation
+        std::visit(overloaded {
-           is explicitly passed to the builder.  Since that allows the
+            /* Since this allows the builder to gain access to every
-           builder to gain access to every path in the dependency
+               path in the dependency graph of the derivation (including
-           graph of the derivation (including all outputs), all paths
+               all outputs), all paths in the graph must be added to
-           in the graph must be added to this derivation's list of
+               this derivation's list of inputs to ensure that they are
-           inputs to ensure that they are available when the builder
+               available when the builder runs. */
-           runs. */
+            [&](const NixStringContextElem::DrvDeep & d) {
-        if (path.at(0) == '=') {
+                /* !!! This doesn't work if readOnlyMode is set. */
-            /* !!! This doesn't work if readOnlyMode is set. */
+                StorePathSet refs;
-            StorePathSet refs;
+                state.store->computeFSClosure(d.drvPath, refs);
-            state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
+                for (auto & j : refs) {
-            for (auto & j : refs) {
+                    drv.inputSrcs.insert(j);
-                drv.inputSrcs.insert(j);
+                    if (j.isDerivation())
-                if (j.isDerivation())
+                        drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
-                    drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
+                }
-            }
+            },
-        }
+            [&](const NixStringContextElem::Built & b) {
-
+                drv.inputDrvs[b.drvPath].insert(b.output);
-        /* Handle derivation outputs of the form ‘!<name>!<path>’. */
+            },
-        else if (path.at(0) == '!') {
+            [&](const NixStringContextElem::Opaque & o) {
-            auto ctx = decodeContext(*state.store, path);
+                drv.inputSrcs.insert(o.path);
-            drv.inputDrvs[ctx.first].insert(ctx.second);
+            },
-        }
+        }, c.raw());
-
-        /* Otherwise it's a source file. */
-        else
-            drv.inputSrcs.insert(state.store->parseStorePath(path));
     }
 
     /* Do we have all required attributes? */
@@ -1891,8 +1904,7 @@ static RegisterPrimOp primop_toFile({
       path.  The file has suffix *name*. This file can be used as an
       input to derivations. One application is to write builders
       “inline”. For instance, the following Nix expression combines the
-      [Nix expression for GNU Hello](expression-syntax.md) and its
+      Nix expression for GNU Hello and its build script into one file:
-      [build script](build-script.md) into one file:
 
       ```nix
       { stdenv, fetchurl, perl }:
@@ -1936,7 +1948,7 @@ static RegisterPrimOp primop_toFile({
       ```
 
       Note that `${configFile}` is a
-      [string interpolation](language/values.md#type-string), so the result of the
+      [string interpolation](@docroot@/language/values.md#type-string), so the result of the
       expression `configFile`
       (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be
       spliced into the resulting string.

src/libexpr/primops/context.cc

@@ -37,8 +37,15 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
     auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency");
 
     PathSet context2;
-    for (auto & p : context)
+    for (auto && p : context) {
-        context2.insert(p.at(0) == '=' ? std::string(p, 1) : p);
+        auto c = NixStringContextElem::parse(*state.store, p);
+        if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) {
+            context2.emplace(state.store->printStorePath(ptr->drvPath));
+        } else {
+            /* Can reuse original item */
+            context2.emplace(std::move(p));
+        }
+    }
 
     v.mkString(*s, context2);
 }
@@ -74,34 +81,22 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
     };
     PathSet context;
     state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext");
-    auto contextInfos = std::map<Path, ContextInfo>();
+    auto contextInfos = std::map<StorePath, ContextInfo>();
     for (const auto & p : context) {
         Path drv;
         std::string output;
-        const Path * path = &p;
+        NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p);
-        if (p.at(0) == '=') {
+        std::visit(overloaded {
-            drv = std::string(p, 1);
+            [&](NixStringContextElem::DrvDeep & d) {
-            path = &drv;
+                contextInfos[d.drvPath].allOutputs = true;
-        } else if (p.at(0) == '!') {
+            },
-            NixStringContextElem ctx = decodeContext(*state.store, p);
+            [&](NixStringContextElem::Built & b) {
-            drv = state.store->printStorePath(ctx.first);
+                contextInfos[b.drvPath].outputs.emplace_back(std::move(output));
-            output = ctx.second;
+            },
-            path = &drv;
+            [&](NixStringContextElem::Opaque & o) {
-        }
+                contextInfos[o.path].path = true;
-        auto isPath = drv.empty();
+            },
-        auto isAllOutputs = (!drv.empty()) && output.empty();
+        }, ctx.raw());
-
-        auto iter = contextInfos.find(*path);
-        if (iter == contextInfos.end()) {
-            contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
-        } else {
-            if (isPath)
-                iter->second.path = true;
-            else if (isAllOutputs)
-                iter->second.allOutputs = true;
-            else
-                iter->second.outputs.emplace_back(std::move(output));
-        }
     }
 
     auto attrs = state.buildBindings(contextInfos.size());
@@ -120,7 +115,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
             for (const auto & [i, output] : enumerate(info.second.outputs))
                 (outputsVal.listElems()[i] = state.allocValue())->mkString(output);
         }
-        attrs.alloc(info.first).mkAttrs(infoAttrs);
+        attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
     }
 
     v.mkAttrs(attrs);

src/libexpr/tests/local.mk

@@ -6,7 +6,9 @@ libexpr-tests_DIR := $(d)
 
 libexpr-tests_INSTALL_DIR :=
 
-libexpr-tests_SOURCES := $(wildcard $(d)/*.cc)
+libexpr-tests_SOURCES := \
+    $(wildcard $(d)/*.cc) \
+    $(wildcard $(d)/value/*.cc)
 
 libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
 

src/libexpr/tests/value/context.cc

@@ -0,0 +1,72 @@
+#include "value/context.hh"
+
+#include "libexprtests.hh"
+
+namespace nix {
+
+// Testing of trivial expressions
+struct NixStringContextElemTest : public LibExprTest {
+   const Store & store() const {
+       return *LibExprTest::store;
+   }
+};
+
+TEST_F(NixStringContextElemTest, empty_invalid) {
+    EXPECT_THROW(
+        NixStringContextElem::parse(store(), ""),
+        BadNixStringContextElem);
+}
+
+TEST_F(NixStringContextElemTest, single_bang_invalid) {
+    EXPECT_THROW(
+        NixStringContextElem::parse(store(), "!"),
+        BadNixStringContextElem);
+}
+
+TEST_F(NixStringContextElemTest, double_bang_invalid) {
+    EXPECT_THROW(
+        NixStringContextElem::parse(store(), "!!/"),
+        BadStorePath);
+}
+
+TEST_F(NixStringContextElemTest, eq_slash_invalid) {
+    EXPECT_THROW(
+        NixStringContextElem::parse(store(), "=/"),
+        BadStorePath);
+}
+
+TEST_F(NixStringContextElemTest, slash_invalid) {
+    EXPECT_THROW(
+        NixStringContextElem::parse(store(), "/"),
+        BadStorePath);
+}
+
+TEST_F(NixStringContextElemTest, opaque) {
+    std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
+    auto elem = NixStringContextElem::parse(store(), opaque);
+    auto * p = std::get_if<NixStringContextElem::Opaque>(&elem);
+    ASSERT_TRUE(p);
+    ASSERT_EQ(p->path, store().parseStorePath(opaque));
+    ASSERT_EQ(elem.to_string(store()), opaque);
+}
+
+TEST_F(NixStringContextElemTest, drvDeep) {
+    std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
+    auto elem = NixStringContextElem::parse(store(), drvDeep);
+    auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem);
+    ASSERT_TRUE(p);
+    ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1)));
+    ASSERT_EQ(elem.to_string(store()), drvDeep);
+}
+
+TEST_F(NixStringContextElemTest, built) {
+    std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
+    auto elem = NixStringContextElem::parse(store(), built);
+    auto * p = std::get_if<NixStringContextElem::Built>(&elem);
+    ASSERT_TRUE(p);
+    ASSERT_EQ(p->output, "foo");
+    ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5)));
+    ASSERT_EQ(elem.to_string(store()), built);
+}
+
+}

src/libexpr/value.hh

@@ -3,6 +3,7 @@
 #include <cassert>
 
 #include "symbol-table.hh"
+#include "value/context.hh"
 #include "input-accessor.hh"
 
 #if HAVE_BOEHMGC
@@ -68,8 +69,6 @@ class XMLWriter;
 
 typedef int64_t NixInt;
 typedef double NixFloat;
-typedef std::pair<StorePath, std::string> NixStringContextElem;
-typedef std::vector<NixStringContextElem> NixStringContext;
 
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented

src/libexpr/value/context.cc

@@ -0,0 +1,67 @@
+#include "value/context.hh"
+#include "store-api.hh"
+
+#include <optional>
+
+namespace nix {
+
+NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0)
+{
+    std::string_view s = s0;
+
+    if (s.size() == 0) {
+        throw BadNixStringContextElem(s0,
+            "String context element should never be an empty string");
+    }
+    switch (s.at(0)) {
+    case '!': {
+        s = s.substr(1); // advance string to parse after first !
+        size_t index = s.find("!");
+        // This makes index + 1 safe. Index can be the length (one after index
+        // of last character), so given any valid character index --- a
+        // successful find --- we can add one.
+        if (index == std::string_view::npos) {
+            throw BadNixStringContextElem(s0,
+                "String content element beginning with '!' should have a second '!'");
+        }
+        return NixStringContextElem::Built {
+            .drvPath = store.parseStorePath(s.substr(index + 1)),
+            .output = std::string(s.substr(0, index)),
+        };
+    }
+    case '=': {
+        return NixStringContextElem::DrvDeep {
+            .drvPath = store.parseStorePath(s.substr(1)),
+        };
+    }
+    default: {
+        return NixStringContextElem::Opaque {
+            .path = store.parseStorePath(s),
+        };
+    }
+    }
+}
+
+std::string NixStringContextElem::to_string(const Store & store) const {
+    return std::visit(overloaded {
+        [&](const NixStringContextElem::Built & b) {
+            std::string res;
+            res += '!';
+            res += b.output;
+            res += '!';
+            res += store.printStorePath(b.drvPath);
+            return res;
+        },
+        [&](const NixStringContextElem::DrvDeep & d) {
+            std::string res;
+            res += '=';
+            res += store.printStorePath(d.drvPath);
+            return res;
+        },
+        [&](const NixStringContextElem::Opaque & o) {
+            return store.printStorePath(o.path);
+        },
+    }, raw());
+}
+
+}

src/libexpr/value/context.hh

@@ -0,0 +1,90 @@
+#pragma once
+
+#include "util.hh"
+#include "path.hh"
+
+#include <optional>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+
+class BadNixStringContextElem : public Error
+{
+public:
+    std::string_view raw;
+
+    template<typename... Args>
+    BadNixStringContextElem(std::string_view raw_, const Args & ... args)
+        : Error("")
+    {
+        raw = raw_;
+        auto hf = hintfmt(args...);
+        err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw);
+    }
+};
+
+class Store;
+
+/* Plain opaque path to some store object.
+
+   Encoded as just the path: ‘<path>’.
+*/
+struct NixStringContextElem_Opaque {
+   StorePath path;
+};
+
+/* Path to a derivation and its entire build closure.
+
+   The path doesn't just refer to derivation itself and its closure, but
+   also all outputs of all derivations in that closure (including the
+   root derivation).
+
+   Encoded in the form ‘=<drvPath>’.
+*/
+struct NixStringContextElem_DrvDeep {
+   StorePath drvPath;
+};
+
+/* Derivation output.
+
+   Encoded in the form ‘!<output>!<drvPath>’.
+*/
+struct NixStringContextElem_Built {
+   StorePath drvPath;
+   std::string output;
+};
+
+using _NixStringContextElem_Raw = std::variant<
+    NixStringContextElem_Opaque,
+    NixStringContextElem_DrvDeep,
+    NixStringContextElem_Built
+>;
+
+struct NixStringContextElem : _NixStringContextElem_Raw {
+    using Raw = _NixStringContextElem_Raw;
+    using Raw::Raw;
+
+    using Opaque = NixStringContextElem_Opaque;
+    using DrvDeep = NixStringContextElem_DrvDeep;
+    using Built = NixStringContextElem_Built;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+    inline Raw & raw() {
+        return static_cast<Raw &>(*this);
+    }
+
+    /* Decode a context string, one of:
+       - ‘<path>’
+       - ‘=<path>’
+       - ‘!<name>!<path>’
+      */
+    static NixStringContextElem parse(const Store & store, std::string_view s);
+    std::string to_string(const Store & store) const;
+};
+
+typedef std::vector<NixStringContextElem> NixStringContext;
+
+}

src/libmain/shared.hh

@@ -39,7 +39,6 @@ void printVersion(const std::string & programName);
 void printGCWarning();
 
 class Store;
-struct StorePathWithOutputs;
 
 void printMissing(
     ref<Store> store,

src/libstore/globals.hh

@@ -676,7 +676,7 @@ public:
           - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys)
           - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
           - the [`require-sigs`](#conf-require-sigs) option has been set to `false`
-          - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object)
+          - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object)
         )",
         {"binary-caches"}};
 

src/libstore/outputs-spec.cc

@@ -0,0 +1,61 @@
+#include "outputs-spec.hh"
+#include "nlohmann/json.hpp"
+
+#include <regex>
+
+namespace nix {
+
+std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
+{
+    static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
+
+    std::smatch match;
+    if (!std::regex_match(s, match, regex))
+        return {s, DefaultOutputs()};
+
+    if (match[3].matched)
+        return {match[1], AllOutputs()};
+
+    return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
+}
+
+std::string printOutputsSpec(const OutputsSpec & outputsSpec)
+{
+    if (std::get_if<DefaultOutputs>(&outputsSpec))
+        return "";
+
+    if (std::get_if<AllOutputs>(&outputsSpec))
+        return "^*";
+
+    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        return "^" + concatStringsSep(",", *outputNames);
+
+    assert(false);
+}
+
+void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
+{
+    if (std::get_if<DefaultOutputs>(&outputsSpec))
+        json = nullptr;
+
+    else if (std::get_if<AllOutputs>(&outputsSpec))
+        json = std::vector<std::string>({"*"});
+
+    else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        json = *outputNames;
+}
+
+void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
+{
+    if (json.is_null())
+        outputsSpec = DefaultOutputs();
+    else {
+        auto names = json.get<OutputNames>();
+        if (names == OutputNames({"*"}))
+            outputsSpec = AllOutputs();
+        else
+            outputsSpec = names;
+    }
+}
+
+}

src/libstore/outputs-spec.hh

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <variant>
+
+#include "util.hh"
+
+#include "nlohmann/json_fwd.hpp"
+
+namespace nix {
+
+typedef std::set<std::string> OutputNames;
+
+struct AllOutputs {
+    bool operator < (const AllOutputs & _) const { return false; }
+};
+
+struct DefaultOutputs {
+    bool operator < (const DefaultOutputs & _) const { return false; }
+};
+
+typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
+
+/* Parse a string of the form 'prefix^output1,...outputN' or
+   'prefix^*', returning the prefix and the outputs spec. */
+std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
+
+std::string printOutputsSpec(const OutputsSpec & outputsSpec);
+
+void to_json(nlohmann::json &, const OutputsSpec &);
+void from_json(const nlohmann::json &, OutputsSpec &);
+
+}

src/libstore/path-with-outputs.cc

@@ -1,6 +1,5 @@
 #include "path-with-outputs.hh"
 #include "store-api.hh"
-#include "nlohmann/json.hpp"
 
 #include <regex>
 
@@ -71,57 +70,4 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
     return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
 }
 
-std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
-{
-    static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
-
-    std::smatch match;
-    if (!std::regex_match(s, match, regex))
-        return {s, DefaultOutputs()};
-
-    if (match[3].matched)
-        return {match[1], AllOutputs()};
-
-    return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
-}
-
-std::string printOutputsSpec(const OutputsSpec & outputsSpec)
-{
-    if (std::get_if<DefaultOutputs>(&outputsSpec))
-        return "";
-
-    if (std::get_if<AllOutputs>(&outputsSpec))
-        return "^*";
-
-    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
-        return "^" + concatStringsSep(",", *outputNames);
-
-    assert(false);
-}
-
-void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
-{
-    if (std::get_if<DefaultOutputs>(&outputsSpec))
-        json = nullptr;
-
-    else if (std::get_if<AllOutputs>(&outputsSpec))
-        json = std::vector<std::string>({"*"});
-
-    else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
-        json = *outputNames;
-}
-
-void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
-{
-    if (json.is_null())
-        outputsSpec = DefaultOutputs();
-    else {
-        auto names = json.get<OutputNames>();
-        if (names == OutputNames({"*"}))
-            outputsSpec = AllOutputs();
-        else
-            outputsSpec = names;
-    }
-}
-
 }

src/libstore/path-with-outputs.hh

@@ -1,13 +1,18 @@
 #pragma once
 
-#include <variant>
-
 #include "path.hh"
 #include "derived-path.hh"
 #include "nlohmann/json_fwd.hpp"
 
 namespace nix {
 
+/* This is a deprecated old type just for use by the old CLI, and older
+   versions of the RPC protocols. In new code don't use it; you want
+   `DerivedPath` instead.
+
+   `DerivedPath` is better because it handles more cases, and does so more
+   explicitly without devious punning tricks.
+*/
 struct StorePathWithOutputs
 {
     StorePath path;
@@ -33,25 +38,4 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
 
 StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
 
-typedef std::set<std::string> OutputNames;
-
-struct AllOutputs {
-    bool operator < (const AllOutputs & _) const { return false; }
-};
-
-struct DefaultOutputs {
-    bool operator < (const DefaultOutputs & _) const { return false; }
-};
-
-typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
-
-/* Parse a string of the form 'prefix^output1,...outputN' or
-   'prefix^*', returning the prefix and the outputs spec. */
-std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
-
-std::string printOutputsSpec(const OutputsSpec & outputsSpec);
-
-void to_json(nlohmann::json &, const OutputsSpec &);
-void from_json(const nlohmann::json &, OutputsSpec &);
-
 }

src/libstore/tests/outputs-spec.cc

@@ -1,4 +1,4 @@
-#include "path-with-outputs.hh"
+#include "outputs-spec.hh"
 
 #include <gtest/gtest.h>
 

src/nix/app.cc

@@ -79,9 +79,29 @@ UnresolvedApp Installable::toApp(EvalState & state)
     if (type == "app") {
         auto [program, context] = cursor->getAttr("program")->getStringWithContext();
 
-        std::vector<StorePathWithOutputs> context2;
+        std::vector<DerivedPath> context2;
-        for (auto & [path, name] : context)
+        for (auto & c : context) {
-            context2.push_back({path, {name}});
+            context2.emplace_back(std::visit(overloaded {
+                [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath {
+                    /* We want all outputs of the drv */
+                    return DerivedPath::Built {
+                        .drvPath = d.drvPath,
+                        .outputs = {},
+                    };
+                },
+                [&](const NixStringContextElem::Built & b) -> DerivedPath {
+                    return DerivedPath::Built {
+                        .drvPath = b.drvPath,
+                        .outputs = { b.output },
+                    };
+                },
+                [&](const NixStringContextElem::Opaque & o) -> DerivedPath {
+                    return DerivedPath::Opaque {
+                        .path = o.path,
+                    };
+                },
+            }, c.raw()));
+        }
 
         return UnresolvedApp{App {
             .context = std::move(context2),
@@ -105,7 +125,10 @@ UnresolvedApp Installable::toApp(EvalState & state)
             : DrvName(name).name;
         auto program = outPath + "/bin/" + mainProgram;
         return UnresolvedApp { App {
-            .context = { { drvPath, {outputName} } },
+            .context = { DerivedPath::Built {
+                .drvPath = drvPath,
+                .outputs = {outputName},
+            } },
             .program = program,
         }};
     }
@@ -123,7 +146,7 @@ App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store)
 
     for (auto & ctxElt : unresolved.context)
         installableContext.push_back(
-            std::make_shared<InstallableDerivedPath>(store, ctxElt.toDerivedPath()));
+            std::make_shared<InstallableDerivedPath>(store, ctxElt));
 
     auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext);
     res.program = resolveString(*store, unresolved.program, builtContext);

src/nix/develop.cc

@@ -3,7 +3,7 @@
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"
-#include "path-with-outputs.hh"
+#include "outputs-spec.hh"
 #include "derivations.hh"
 #include "progress-bar.hh"
 #include "run.hh"

src/nix/flake.cc

@@ -7,7 +7,7 @@
 #include "get-drvs.hh"
 #include "store-api.hh"
 #include "derivations.hh"
-#include "path-with-outputs.hh"
+#include "outputs-spec.hh"
 #include "attr-path.hh"
 #include "fetchers.hh"
 #include "registry.hh"
@@ -344,7 +344,7 @@ struct CmdFlakeCheck : FlakeCommand
                 // FIXME
                 auto app = App(*state, v);
                 for (auto & i : app.context) {
-                    auto [drvPathS, outputName] = decodeContext(i);
+                    auto [drvPathS, outputName] = NixStringContextElem::parse(i);
                     store->parseStorePath(drvPathS);
                 }
                 #endif

tests/plugins/local.mk

@@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1
 
 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
 
-libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr -I src/libfetchers -I src/libstore
+libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr -I src/libfetchers