From da64f026dd7b12d72ffbc15752e8b95707fa1f9f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 10 Jan 2023 11:27:19 -0500 Subject: [PATCH 01/11] Make clear that `StorePathWithOutputs` is a deprecated type - Add a comment - Put `OutputsSpec` in a different header (First part of #6815) - Make a few stray uses of it in new code use `DerivedPath` instead. --- src/libcmd/installables.hh | 4 +- src/libexpr/flake/flakeref.hh | 2 +- src/libmain/shared.hh | 1 - src/libstore/outputs-spec.cc | 61 +++++++++++++++++++ src/libstore/outputs-spec.hh | 32 ++++++++++ src/libstore/path-with-outputs.cc | 54 ---------------- src/libstore/path-with-outputs.hh | 30 +++------ .../{path-with-outputs.cc => outputs-spec.cc} | 2 +- src/nix/app.cc | 23 +++++-- src/nix/develop.cc | 2 +- src/nix/flake.cc | 2 +- 11 files changed, 124 insertions(+), 89 deletions(-) create mode 100644 src/libstore/outputs-spec.cc create mode 100644 src/libstore/outputs-spec.hh rename src/libstore/tests/{path-with-outputs.cc => outputs-spec.cc} (97%) diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 250cf7e83..9b92cc4be 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,7 +2,7 @@ #include "util.hh" #include "path.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derived-path.hh" #include "eval.hh" #include "store-api.hh" @@ -20,7 +20,7 @@ namespace eval_cache { class EvalCache; class AttrCursor; } struct App { - std::vector context; + std::vector context; Path program; // FIXME: add args, sandbox settings, metadata, ... }; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a36d852a8..4ec79fb73 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -3,7 +3,7 @@ #include "types.hh" #include "hash.hh" #include "fetchers.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 3c37fd627..1715374a6 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -39,7 +39,6 @@ void printVersion(const std::string & programName); void printGCWarning(); class Store; -struct StorePathWithOutputs; void printMissing( ref store, diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc new file mode 100644 index 000000000..76779d193 --- /dev/null +++ b/src/libstore/outputs-spec.cc @@ -0,0 +1,61 @@ +#include "outputs-spec.hh" +#include "nlohmann/json.hpp" + +#include + +namespace nix { + +std::pair parseOutputsSpec(const std::string & s) +{ + static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))"); + + std::smatch match; + if (!std::regex_match(s, match, regex)) + return {s, DefaultOutputs()}; + + if (match[3].matched) + return {match[1], AllOutputs()}; + + return {match[1], tokenizeString(match[4].str(), ",")}; +} + +std::string printOutputsSpec(const OutputsSpec & outputsSpec) +{ + if (std::get_if(&outputsSpec)) + return ""; + + if (std::get_if(&outputsSpec)) + return "^*"; + + if (auto outputNames = std::get_if(&outputsSpec)) + return "^" + concatStringsSep(",", *outputNames); + + assert(false); +} + +void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec) +{ + if (std::get_if(&outputsSpec)) + json = nullptr; + + else if (std::get_if(&outputsSpec)) + json = std::vector({"*"}); + + else if (auto outputNames = std::get_if(&outputsSpec)) + json = *outputNames; +} + +void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec) +{ + if (json.is_null()) + outputsSpec = DefaultOutputs(); + else { + auto names = json.get(); + if (names == OutputNames({"*"})) + outputsSpec = AllOutputs(); + else + outputsSpec = names; + } +} + +} diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh new file mode 100644 index 000000000..e2cf1d12b --- /dev/null +++ b/src/libstore/outputs-spec.hh @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include "util.hh" + +#include "nlohmann/json_fwd.hpp" + +namespace nix { + +typedef std::set OutputNames; + +struct AllOutputs { + bool operator < (const AllOutputs & _) const { return false; } +}; + +struct DefaultOutputs { + bool operator < (const DefaultOutputs & _) const { return false; } +}; + +typedef std::variant OutputsSpec; + +/* Parse a string of the form 'prefix^output1,...outputN' or + 'prefix^*', returning the prefix and the outputs spec. */ +std::pair parseOutputsSpec(const std::string & s); + +std::string printOutputsSpec(const OutputsSpec & outputsSpec); + +void to_json(nlohmann::json &, const OutputsSpec &); +void from_json(const nlohmann::json &, OutputsSpec &); + +} diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index d6d67ea05..17230ffc4 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,6 +1,5 @@ #include "path-with-outputs.hh" #include "store-api.hh" -#include "nlohmann/json.hpp" #include @@ -71,57 +70,4 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std: return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } -std::pair parseOutputsSpec(const std::string & s) -{ - static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))"); - - std::smatch match; - if (!std::regex_match(s, match, regex)) - return {s, DefaultOutputs()}; - - if (match[3].matched) - return {match[1], AllOutputs()}; - - return {match[1], tokenizeString(match[4].str(), ",")}; -} - -std::string printOutputsSpec(const OutputsSpec & outputsSpec) -{ - if (std::get_if(&outputsSpec)) - return ""; - - if (std::get_if(&outputsSpec)) - return "^*"; - - if (auto outputNames = std::get_if(&outputsSpec)) - return "^" + concatStringsSep(",", *outputNames); - - assert(false); -} - -void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec) -{ - if (std::get_if(&outputsSpec)) - json = nullptr; - - else if (std::get_if(&outputsSpec)) - json = std::vector({"*"}); - - else if (auto outputNames = std::get_if(&outputsSpec)) - json = *outputNames; -} - -void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec) -{ - if (json.is_null()) - outputsSpec = DefaultOutputs(); - else { - auto names = json.get(); - if (names == OutputNames({"*"})) - outputsSpec = AllOutputs(); - else - outputsSpec = names; - } -} - } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 0cb5eb223..ed55cc333 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,13 +1,18 @@ #pragma once -#include - #include "path.hh" #include "derived-path.hh" #include "nlohmann/json_fwd.hpp" namespace nix { +/* This is a deprecated old type just for use by the old CLI, and older + versions of the RPC protocols. In new code don't use it; you want + `DerivedPath` instead. + + `DerivedPath` is better because it handles more cases, and does so more + explicitly without devious punning tricks. +*/ struct StorePathWithOutputs { StorePath path; @@ -33,25 +38,4 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); -typedef std::set OutputNames; - -struct AllOutputs { - bool operator < (const AllOutputs & _) const { return false; } -}; - -struct DefaultOutputs { - bool operator < (const DefaultOutputs & _) const { return false; } -}; - -typedef std::variant OutputsSpec; - -/* Parse a string of the form 'prefix^output1,...outputN' or - 'prefix^*', returning the prefix and the outputs spec. */ -std::pair parseOutputsSpec(const std::string & s); - -std::string printOutputsSpec(const OutputsSpec & outputsSpec); - -void to_json(nlohmann::json &, const OutputsSpec &); -void from_json(const nlohmann::json &, OutputsSpec &); - } diff --git a/src/libstore/tests/path-with-outputs.cc b/src/libstore/tests/outputs-spec.cc similarity index 97% rename from src/libstore/tests/path-with-outputs.cc rename to src/libstore/tests/outputs-spec.cc index 350ea7ffd..d781a930e 100644 --- a/src/libstore/tests/path-with-outputs.cc +++ b/src/libstore/tests/outputs-spec.cc @@ -1,4 +1,4 @@ -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include diff --git a/src/nix/app.cc b/src/nix/app.cc index a8d7e115b..fb149042c 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -79,9 +79,19 @@ UnresolvedApp Installable::toApp(EvalState & state) if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - std::vector context2; - for (auto & [path, name] : context) - context2.push_back({path, {name}}); + std::vector context2; + for (auto & [path, name] : context) { + context2.push_back(name != "" || path.isDerivation() + ? (DerivedPath) DerivedPath::Built { + .drvPath = path, + .outputs = name != "" + ? StringSet { name } + : StringSet { }, + } + : (DerivedPath) DerivedPath::Opaque { + .path = path, + }); + } return UnresolvedApp{App { .context = std::move(context2), @@ -105,7 +115,10 @@ UnresolvedApp Installable::toApp(EvalState & state) : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { - .context = { { drvPath, {outputName} } }, + .context = { DerivedPath::Built { + .drvPath = drvPath, + .outputs = {outputName}, + } }, .program = program, }}; } @@ -123,7 +136,7 @@ App UnresolvedApp::resolve(ref evalStore, ref store) for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared(store, ctxElt.toDerivedPath())); + std::make_shared(store, ctxElt)); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1d90d1dac..6aa675386 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -3,7 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9b4cdf35a..bb020d51e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,7 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" From 5576d5e987e907bf13ae6c7fe79ececce4e86e2d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 3 Jan 2023 11:44:59 -0500 Subject: [PATCH 02/11] Parse string context elements properly Prior to this change, we had a bunch of ad-hoc string manipulation code scattered around. This made it hard to figure out what data model for string contexts is. Now, we still store string contexts most of the time as encoded strings --- I was wary of the performance implications of changing that --- but whenever we parse them we do so only through the `NixStringContextElem::parse` method, which handles all cases. This creates a data type that is very similar to `DerivedPath` but: - Represents the funky `=` case as properly distinct from the others. - Only encodes a single output, no wildcards and no set, for the "built" case. (I would like to deprecate `=`, after which we are in spitting distance of `DerivedPath` and could maybe get away with fewer types, but that is another topic for another day.) --- src/libexpr/eval-cache.cc | 15 ++++- src/libexpr/eval.cc | 23 +------- src/libexpr/eval.hh | 4 -- src/libexpr/local.mk | 3 + src/libexpr/primops.cc | 90 +++++++++++++++++------------- src/libexpr/primops/context.cc | 51 ++++++++--------- src/libexpr/tests/local.mk | 4 +- src/libexpr/tests/value/context.cc | 72 ++++++++++++++++++++++++ src/libexpr/value.hh | 3 +- src/libexpr/value/context.cc | 67 ++++++++++++++++++++++ src/libexpr/value/context.hh | 90 ++++++++++++++++++++++++++++++ src/nix/app.cc | 32 +++++++---- src/nix/flake.cc | 2 +- tests/plugins/local.mk | 2 +- 14 files changed, 347 insertions(+), 111 deletions(-) create mode 100644 src/libexpr/tests/value/context.cc create mode 100644 src/libexpr/value/context.cc create mode 100644 src/libexpr/value/context.hh diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index afe575fee..1219b2471 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -300,7 +300,7 @@ struct AttrDb NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) - context.push_back(decodeContext(cfg, s)); + context.push_back(NixStringContextElem::parse(cfg, s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: @@ -592,7 +592,18 @@ string_t AttrCursor::getStringWithContext() if (auto s = std::get_if(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - if (!root->state.store->isValidPath(c.first)) { + const StorePath & path = std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & { + return d.drvPath; + }, + [&](const NixStringContextElem::Built & b) -> const StorePath & { + return b.drvPath; + }, + [&](const NixStringContextElem::Opaque & o) -> const StorePath & { + return o.path; + }, + }, c.raw()); + if (!root->state.store->isValidPath(path)) { valid = false; break; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 978b0f0e2..277cbb5f9 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2068,27 +2068,6 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string } -/* Decode a context string ‘!!’ into a pair . */ -NixStringContextElem decodeContext(const Store & store, std::string_view s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return { - store.parseStorePath(s.substr(index + 1)), - std::string(s.substr(1, index - 1)), - }; - } else - return { - store.parseStorePath( - s.at(0) == '/' - ? s - : s.substr(1)), - "", - }; -} - - void copyContext(const Value & v, PathSet & context) { if (v.string.context) @@ -2103,7 +2082,7 @@ NixStringContext Value::getContext(const Store & store) assert(internalType == tString); if (string.context) for (const char * * p = string.context; *p; ++p) - res.push_back(decodeContext(store, *p)); + res.push_back(NixStringContextElem::parse(store, *p)); return res; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4e0c4db95..46b8cbaa5 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -551,10 +551,6 @@ struct DebugTraceStacker { std::string_view showType(ValueType type); std::string showType(const Value & v); -/* Decode a context string ‘!!’ into a pair . */ -NixStringContextElem decodeContext(const Store & store, std::string_view s); - /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 016631647..2171e769b 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,6 +6,7 @@ libexpr_DIR := $(d) libexpr_SOURCES := \ $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ $(wildcard $(d)/primops/*.cc) \ $(wildcard $(d)/flake/*.cc) \ $(d)/lexer-tab.cc \ @@ -37,6 +38,8 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe $(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(foreach i, $(wildcard src/libexpr/value/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9ef91cbc5..a433e99f0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -43,16 +43,32 @@ StringMap EvalState::realiseContext(const PathSet & context) std::vector drvs; StringMap res; - for (auto & i : context) { - auto [ctx, outputName] = decodeContext(*store, i); - auto ctxS = store->printStorePath(ctx); - if (!store->isValidPath(ctx)) - debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx))); - if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back({ctx, {outputName}}); - } else { - res.insert_or_assign(ctxS, ctxS); - } + for (auto & c_ : context) { + auto ensureValid = [&](const StorePath & p) { + if (!store->isValidPath(p)) + debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); + }; + auto c = NixStringContextElem::parse(*store, c_); + std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + drvs.push_back(DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = std::set { b.output }, + }); + ensureValid(b.drvPath); + }, + [&](const NixStringContextElem::Opaque & o) { + auto ctxS = store->printStorePath(o.path); + res.insert_or_assign(ctxS, ctxS); + ensureValid(o.path); + }, + [&](const NixStringContextElem::DrvDeep & d) { + /* Treat same as Opaque */ + auto ctxS = store->printStorePath(d.drvPath); + res.insert_or_assign(ctxS, ctxS); + ensureValid(d.drvPath); + }, + }, c.raw()); } if (drvs.empty()) return {}; @@ -1179,35 +1195,31 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & path : context) { - - /* Paths marked with `=' denote that the path of a derivation - is explicitly passed to the builder. Since that allows the - builder to gain access to every path in the dependency - graph of the derivation (including all outputs), all paths - in the graph must be added to this derivation's list of - inputs to ensure that they are available when the builder - runs. */ - if (path.at(0) == '=') { - /* !!! This doesn't work if readOnlyMode is set. */ - StorePathSet refs; - state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); - for (auto & j : refs) { - drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); - } - } - - /* Handle derivation outputs of the form ‘!!’. */ - else if (path.at(0) == '!') { - auto ctx = decodeContext(*state.store, path); - drv.inputDrvs[ctx.first].insert(ctx.second); - } - - /* Otherwise it's a source file. */ - else - drv.inputSrcs.insert(state.store->parseStorePath(path)); + for (auto & c_ : context) { + auto c = NixStringContextElem::parse(*state.store, c_); + std::visit(overloaded { + /* Since this allows the builder to gain access to every + path in the dependency graph of the derivation (including + all outputs), all paths in the graph must be added to + this derivation's list of inputs to ensure that they are + available when the builder runs. */ + [&](const NixStringContextElem::DrvDeep & d) { + /* !!! This doesn't work if readOnlyMode is set. */ + StorePathSet refs; + state.store->computeFSClosure(d.drvPath, refs); + for (auto & j : refs) { + drv.inputSrcs.insert(j); + if (j.isDerivation()) + drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); + } + }, + [&](const NixStringContextElem::Built & b) { + drv.inputDrvs[b.drvPath].insert(b.output); + }, + [&](const NixStringContextElem::Opaque & o) { + drv.inputSrcs.insert(o.path); + }, + }, c.raw()); } /* Do we have all required attributes? */ diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 9fae0b14d..0c65a6b98 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -37,8 +37,15 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); + for (auto && p : context) { + auto c = NixStringContextElem::parse(*state.store, p); + if (auto * ptr = std::get_if(&c)) { + context2.emplace(state.store->printStorePath(ptr->drvPath)); + } else { + /* Can reuse original item */ + context2.emplace(std::move(p)); + } + } v.mkString(*s, context2); } @@ -74,34 +81,22 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, }; PathSet context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); - auto contextInfos = std::map(); + auto contextInfos = std::map(); for (const auto & p : context) { Path drv; std::string output; - const Path * path = &p; - if (p.at(0) == '=') { - drv = std::string(p, 1); - path = &drv; - } else if (p.at(0) == '!') { - NixStringContextElem ctx = decodeContext(*state.store, p); - drv = state.store->printStorePath(ctx.first); - output = ctx.second; - path = &drv; - } - auto isPath = drv.empty(); - auto isAllOutputs = (!drv.empty()) && output.empty(); - - auto iter = contextInfos.find(*path); - if (iter == contextInfos.end()) { - contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); - } else { - if (isPath) - iter->second.path = true; - else if (isAllOutputs) - iter->second.allOutputs = true; - else - iter->second.outputs.emplace_back(std::move(output)); - } + NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p); + std::visit(overloaded { + [&](NixStringContextElem::DrvDeep & d) { + contextInfos[d.drvPath].allOutputs = true; + }, + [&](NixStringContextElem::Built & b) { + contextInfos[b.drvPath].outputs.emplace_back(std::move(output)); + }, + [&](NixStringContextElem::Opaque & o) { + contextInfos[o.path].path = true; + }, + }, ctx.raw()); } auto attrs = state.buildBindings(contextInfos.size()); @@ -120,7 +115,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, for (const auto & [i, output] : enumerate(info.second.outputs)) (outputsVal.listElems()[i] = state.allocValue())->mkString(output); } - attrs.alloc(info.first).mkAttrs(infoAttrs); + attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs); } v.mkAttrs(attrs); diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index b95980cab..e483575a4 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -6,7 +6,9 @@ libexpr-tests_DIR := $(d) libexpr-tests_INSTALL_DIR := -libexpr-tests_SOURCES := $(wildcard $(d)/*.cc) +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc new file mode 100644 index 000000000..d5c9d3bce --- /dev/null +++ b/src/libexpr/tests/value/context.cc @@ -0,0 +1,72 @@ +#include "value/context.hh" + +#include "libexprtests.hh" + +namespace nix { + +// Testing of trivial expressions +struct NixStringContextElemTest : public LibExprTest { + const Store & store() const { + return *LibExprTest::store; + } +}; + +TEST_F(NixStringContextElemTest, empty_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), ""), + BadNixStringContextElem); +} + +TEST_F(NixStringContextElemTest, single_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "!"), + BadNixStringContextElem); +} + +TEST_F(NixStringContextElemTest, double_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "!!/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, eq_slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "=/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(store(), opaque); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store().parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(store()), opaque); +} + +TEST_F(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(store(), drvDeep); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1))); + ASSERT_EQ(elem.to_string(store()), drvDeep); +} + +TEST_F(NixStringContextElemTest, built) { + std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(store(), built); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5))); + ASSERT_EQ(elem.to_string(store()), built); +} + +} diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index f57597cff..7d3f6d700 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,7 @@ #include #include "symbol-table.hh" +#include "value/context.hh" #if HAVE_BOEHMGC #include @@ -67,8 +68,6 @@ class XMLWriter; typedef int64_t NixInt; typedef double NixFloat; -typedef std::pair NixStringContextElem; -typedef std::vector NixStringContext; /* External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc new file mode 100644 index 000000000..61d9c53df --- /dev/null +++ b/src/libexpr/value/context.cc @@ -0,0 +1,67 @@ +#include "value/context.hh" +#include "store-api.hh" + +#include + +namespace nix { + +NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0) +{ + std::string_view s = s0; + + if (s.size() == 0) { + throw BadNixStringContextElem(s0, + "String context element should never be an empty string"); + } + switch (s.at(0)) { + case '!': { + s = s.substr(1); // advance string to parse after first ! + size_t index = s.find("!"); + // This makes index + 1 safe. Index can be the length (one after index + // of last character), so given any valid character index --- a + // successful find --- we can add one. + if (index == std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element beginning with '!' should have a second '!'"); + } + return NixStringContextElem::Built { + .drvPath = store.parseStorePath(s.substr(index + 1)), + .output = std::string(s.substr(0, index)), + }; + } + case '=': { + return NixStringContextElem::DrvDeep { + .drvPath = store.parseStorePath(s.substr(1)), + }; + } + default: { + return NixStringContextElem::Opaque { + .path = store.parseStorePath(s), + }; + } + } +} + +std::string NixStringContextElem::to_string(const Store & store) const { + return std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + std::string res; + res += '!'; + res += b.output; + res += '!'; + res += store.printStorePath(b.drvPath); + return res; + }, + [&](const NixStringContextElem::DrvDeep & d) { + std::string res; + res += '='; + res += store.printStorePath(d.drvPath); + return res; + }, + [&](const NixStringContextElem::Opaque & o) { + return store.printStorePath(o.path); + }, + }, raw()); +} + +} diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh new file mode 100644 index 000000000..d8008c436 --- /dev/null +++ b/src/libexpr/value/context.hh @@ -0,0 +1,90 @@ +#pragma once + +#include "util.hh" +#include "path.hh" + +#include + +#include + +namespace nix { + +class BadNixStringContextElem : public Error +{ +public: + std::string_view raw; + + template + BadNixStringContextElem(std::string_view raw_, const Args & ... args) + : Error("") + { + raw = raw_; + auto hf = hintfmt(args...); + err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw); + } +}; + +class Store; + +/* Plain opaque path to some store object. + + Encoded as just the path: ‘’. +*/ +struct NixStringContextElem_Opaque { + StorePath path; +}; + +/* Path to a derivation and its entire build closure. + + The path doesn't just refer to derivation itself and its closure, but + also all outputs of all derivations in that closure (including the + root derivation). + + Encoded in the form ‘=’. +*/ +struct NixStringContextElem_DrvDeep { + StorePath drvPath; +}; + +/* Derivation output. + + Encoded in the form ‘!!’. +*/ +struct NixStringContextElem_Built { + StorePath drvPath; + std::string output; +}; + +using _NixStringContextElem_Raw = std::variant< + NixStringContextElem_Opaque, + NixStringContextElem_DrvDeep, + NixStringContextElem_Built +>; + +struct NixStringContextElem : _NixStringContextElem_Raw { + using Raw = _NixStringContextElem_Raw; + using Raw::Raw; + + using Opaque = NixStringContextElem_Opaque; + using DrvDeep = NixStringContextElem_DrvDeep; + using Built = NixStringContextElem_Built; + + inline const Raw & raw() const { + return static_cast(*this); + } + inline Raw & raw() { + return static_cast(*this); + } + + /* Decode a context string, one of: + - ‘’ + - ‘=’ + - ‘!!’ + */ + static NixStringContextElem parse(const Store & store, std::string_view s); + std::string to_string(const Store & store) const; +}; + +typedef std::vector NixStringContext; + +} diff --git a/src/nix/app.cc b/src/nix/app.cc index fb149042c..c9637dcf5 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -80,17 +80,27 @@ UnresolvedApp Installable::toApp(EvalState & state) auto [program, context] = cursor->getAttr("program")->getStringWithContext(); std::vector context2; - for (auto & [path, name] : context) { - context2.push_back(name != "" || path.isDerivation() - ? (DerivedPath) DerivedPath::Built { - .drvPath = path, - .outputs = name != "" - ? StringSet { name } - : StringSet { }, - } - : (DerivedPath) DerivedPath::Opaque { - .path = path, - }); + for (auto & c : context) { + context2.emplace_back(std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { + /* We want all outputs of the drv */ + return DerivedPath::Built { + .drvPath = d.drvPath, + .outputs = {}, + }; + }, + [&](const NixStringContextElem::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = { b.output }, + }; + }, + [&](const NixStringContextElem::Opaque & o) -> DerivedPath { + return DerivedPath::Opaque { + .path = o.path, + }; + }, + }, c.raw())); } return UnresolvedApp{App { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index bb020d51e..33ce3f401 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -348,7 +348,7 @@ struct CmdFlakeCheck : FlakeCommand // FIXME auto app = App(*state, v); for (auto & i : app.context) { - auto [drvPathS, outputName] = decodeContext(i); + auto [drvPathS, outputName] = NixStringContextElem::parse(i); store->parseStorePath(drvPathS); } #endif diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk index 82ad99402..8182a6a83 100644 --- a/tests/plugins/local.mk +++ b/tests/plugins/local.mk @@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 -libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr +libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr From be10c09d2350019bbf4075c5e22ddb1f97d3dad0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 08:53:29 +0100 Subject: [PATCH 03/11] manual: Check links mdbook-linkcheck is not consistent about its warning setting. It disables some warnings, but not the warnings about lack of fragment checking support; hence the extra filtering. --- doc/manual/book.toml | 9 +++++++++ doc/manual/local.mk | 5 ++++- flake.nix | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 46ced7ff7..73fb7e75e 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -10,3 +10,12 @@ git-repository-url = "https://github.com/NixOS/nix" [preprocessor.anchors] renderers = ["html"] command = "jq --from-file doc/manual/anchors.jq" + +[output.linkcheck] +# no Internet during the build (in the sandbox) +follow-web-links = false + +# mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in +# excessive "Potential incomplete link" warnings. No other kind of warning was +# produced at the time of writing. +warning-policy = "ignore" diff --git a/doc/manual/local.mk b/doc/manual/local.mk index c0f69e00f..2a32f1a63 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -102,6 +102,9 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli @touch $@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md - $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual + $(trace-gen) \ + set -euo pipefail; \ + RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual 2>&1 \ + | { grep -Fv "because fragment resolution isn't implemented" || :; } endif diff --git a/flake.nix b/flake.nix index 652695989..68011a16b 100644 --- a/flake.nix +++ b/flake.nix @@ -96,6 +96,7 @@ buildPackages.flex (lib.getBin buildPackages.lowdown-nix) buildPackages.mdbook + buildPackages.mdbook-linkcheck buildPackages.autoconf-archive buildPackages.autoreconfHook buildPackages.pkg-config From 34a1e0d29b55ff4f3fc7c923a3f03a65c3ff79a3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 10:09:08 +0100 Subject: [PATCH 04/11] doc/manual: Introduce @docroot@ as a stable base for includable snippets This way the links are clearly within the manual (ie not absolute paths), while allowing snippets to reference the documentation root reliably, regardless of at which base url they're included. --- doc/manual/local.mk | 10 +++++++--- src/libcmd/common-eval-args.cc | 4 ++-- src/libexpr/primops.cc | 2 +- src/libstore/globals.hh | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 2a32f1a63..fdcd131de 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -50,11 +50,14 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + $(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g' + @mv $@.tmp $@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \ + | sed -e 's^@docroot@^..^g'>> $@.tmp @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix @@ -67,7 +70,8 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \ + | sed -e 's^@docroot@^..^g' >> $@.tmp @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 0e321e5e4..908127b4d 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -34,8 +34,8 @@ MixEvalArgs::MixEvalArgs() .shortName = 'I', .description = R"( Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](./env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](../language/values.md#type-path) enclosed in angle + initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment + variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle brackets (i.e., ``). For instance, passing diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9ef91cbc5..5519642b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1917,7 +1917,7 @@ static RegisterPrimOp primop_toFile({ ``` Note that `${configFile}` is a - [string interpolation](language/values.md#type-string), so the result of the + [string interpolation](@docroot@/language/values.md#type-string), so the result of the expression `configFile` (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be spliced into the resulting string. diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f026c8808..7111def92 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -676,7 +676,7 @@ public: - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object) + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-caches"}}; From e79f93571862f62447b7c95d10e797369a0be880 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 10:11:06 +0100 Subject: [PATCH 05/11] doc/manual: Fix broken internal links The targets I could find. --- doc/manual/src/architecture/architecture.md | 2 +- doc/manual/src/command-ref/env-common.md | 2 +- doc/manual/src/command-ref/nix-copy-closure.md | 2 +- src/libexpr/primops.cc | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 2d1b26558..e51958052 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -68,7 +68,7 @@ It can also execute build plans to produce new data, which are made available to A build plan itself is a series of *build tasks*, together with their build inputs. > **Important** -> A build task in Nix is called [derivation](../glossary#gloss-derivation). +> A build task in Nix is called [derivation](../glossary.md#gloss-derivation). Each build task has a special build input executed as *build instructions* in order to perform the build. The result of a build task can be input to another build task. diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 5845bdc43..bb85a6b07 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -11,7 +11,7 @@ Most Nix commands interpret the following environment variables: expressions using [paths](../language/values.md#type-path) enclosed in angle brackets (i.e., ``), e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](./opt-common#opt-I). + [`-I` option](./opt-common.md#opt-I). - [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\ Normally, the Nix store directory (typically `/nix/store`) is not diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index 83e8a2936..cd8e351bb 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`. - `--include-outputs`\ Also copy the outputs of [store derivation]s included in the closure. - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: ../glossary.md#gloss-store-derivation - `--use-substitutes` / `-s`\ Attempt to download missing paths on the target machine using Nix’s diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5519642b1..23ddd607c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -240,6 +240,7 @@ static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { static RegisterPrimOp primop_import({ .name = "import", .args = {"path"}, + // TODO turn "normal path values" into link below .doc = R"( Load, parse and return the Nix expression in the file *path*. If *path* is a directory, the file ` default.nix ` in that directory @@ -253,7 +254,7 @@ static RegisterPrimOp primop_import({ > > Unlike some languages, `import` is a regular function in Nix. > Paths using the angle bracket syntax (e.g., `import` *\*) - > are [normal path values](language-values.md). + > are normal path values. A Nix expression loaded by `import` must not contain any *free variables* (identifiers that are not defined in the Nix expression @@ -1872,8 +1873,7 @@ static RegisterPrimOp primop_toFile({ path. The file has suffix *name*. This file can be used as an input to derivations. One application is to write builders “inline”. For instance, the following Nix expression combines the - [Nix expression for GNU Hello](expression-syntax.md) and its - [build script](build-script.md) into one file: + Nix expression for GNU Hello and its build script into one file: ```nix { stdenv, fetchurl, perl }: From d5c8289f1e39397fcf562fa750719e9e86569041 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 10:29:05 +0100 Subject: [PATCH 06/11] doc/manual: Document hacking on the manual links --- doc/manual/src/contributing/hacking.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index c9da1962f..cd3788efc 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -249,3 +249,22 @@ search/replaced in it for each new build. The installer now supports a `--tarball-url-prefix` flag which _may_ have solved this need? --> + +### Checking the manual links + +The build checks for broken internal links, but this happens late in the process, +so `nix build .` is not suitable for iterating. To check the manual incrementally, run: + +```console +make html -j $NIX_BUILD_CORES +``` + +When iterating on the makefile, run: + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES +``` + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +`mdbook-linkcheck` does not implement fragment checking yet. From fd2af69e600fdf0e06c996df607d560b222f7a38 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 11:04:52 +0100 Subject: [PATCH 07/11] doc/manual: Move the html files back where they were before ... before the link checking "output" was added, bumping the html output into a subdirectory. --- doc/manual/local.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index fdcd131de..7639f510a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -108,7 +108,10 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) \ set -euo pipefail; \ - RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual 2>&1 \ + RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ | { grep -Fv "because fragment resolution isn't implemented" || :; } + @rm -rf $(DESTDIR)$(docdir)/manual + @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual + @rm -rf $(DESTDIR)$(docdir)/manual.tmp endif From fefa3a49ce67998923af0b6c927cb9cea1f4ceaa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 11:21:21 +0100 Subject: [PATCH 08/11] doc/manual: Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 16 +++++++++++----- src/libexpr/primops.cc | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index cd3788efc..fe24294f8 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -250,16 +250,19 @@ The installer now supports a `--tarball-url-prefix` flag which _may_ have solved this need? --> -### Checking the manual links +### Checking links in the manual -The build checks for broken internal links, but this happens late in the process, -so `nix build .` is not suitable for iterating. To check the manual incrementally, run: +The build checks for broken internal links. +This happens late in the process, so `nix build` is not suitable for iterating. +To build the manual incrementally, run: ```console make html -j $NIX_BUILD_CORES ``` -When iterating on the makefile, run: +In order to reflect changes to the [Makefile], clear all generated files before re-building: + +[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk ```console rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES @@ -267,4 +270,7 @@ rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/comman If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. -`mdbook-linkcheck` does not implement fragment checking yet. +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 23ddd607c..fc1524599 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -254,7 +254,7 @@ static RegisterPrimOp primop_import({ > > Unlike some languages, `import` is a regular function in Nix. > Paths using the angle bracket syntax (e.g., `import` *\*) - > are normal path values. + > are normal [path values](@docroot@/language/values.md#type-path). A Nix expression loaded by `import` must not contain any *free variables* (identifiers that are not defined in the Nix expression From da4d4feacf76a829ee7929c3d69f6f3d6ba86f1f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 11:36:01 +0100 Subject: [PATCH 09/11] doc/manual/hacking: Document @docroot@ variable --- doc/manual/local.mk | 3 +++ doc/manual/src/contributing/hacking.md | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 7639f510a..190f0258a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -51,11 +51,13 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g' @mv $@.tmp $@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \ | sed -e 's^@docroot@^..^g'>> $@.tmp @mv $@.tmp $@ @@ -70,6 +72,7 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \ | sed -e 's^@docroot@^..^g' >> $@.tmp @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index fe24294f8..b9b327fba 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -268,9 +268,15 @@ In order to reflect changes to the [Makefile], clear all generated files before rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES ``` -If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. - [`mdbook-linkcheck`] does not implement checking [URI fragments] yet. [`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck [URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment + +#### `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the (probably) generated source file that mentions it. See existing `@docroot@` logic in the [Makefile]. Regular markdown files that are not copied to other markdown files do have a base path of their own and they can use relative paths instead of `@docroot@`. From 6ae4d762d0a1b208eaa47e9bd8960571ac908af7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 13:52:17 +0100 Subject: [PATCH 10/11] doc/manual/src/contributing/hacking.md: Apply suggestion Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index b9b327fba..aeb0d41b3 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -279,4 +279,6 @@ rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/comman If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. -If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the (probably) generated source file that mentions it. See existing `@docroot@` logic in the [Makefile]. Regular markdown files that are not copied to other markdown files do have a base path of their own and they can use relative paths instead of `@docroot@`. +If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. From 7515617ad046a08f2b9d0dd1f552e8841be4971d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Jan 2023 13:49:39 +0100 Subject: [PATCH 11/11] Backport getLine tests from lazy-trees --- src/libutil/tests/tests.cc | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 6e325db98..250e83a38 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -311,6 +311,42 @@ namespace nix { ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); } + /* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + + TEST(getLine, all) { + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } + } + /* ---------------------------------------------------------------------------- * toLower * --------------------------------------------------------------------------*/