diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 3fc1fae42..8af80a02c 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -10,12 +10,14 @@ let type' = optionalString (type != null) " (${type})"; impureNotice = optionalString impure-only '' - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + > **Note** + > + > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). ''; in squash '' -
- ${name}${type'} +
+ ${name}${type'}
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f2e69178f..6c599abcf 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -109,6 +109,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index b8ea977d7..4b0a3a3e5 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -128,7 +128,7 @@ platform. Common solutions include [remote builders] and [binary format emulatio (only supported on NixOS). [remote builders]: ../advanced-topics/distributed-builds.md -[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems +[binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems Given such a setup, executing the build only requires selecting the respective attribute. For example, to compile for `aarch64-linux`: @@ -166,7 +166,7 @@ When Nix is built such that `./configure` is passed any of the `--host`, `--buil -[-]- ``` -For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub) as follows: +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows: | `config.guess` | Nix | |----------------------------|---------------------| diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md new file mode 100644 index 000000000..125a93cfd --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -0,0 +1,29 @@ +# Release 2.17 (2023-07-24) + +* [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand. + +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. + +* Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. + +* Nested dynamic attributes are now merged correctly by the parser. For example: + + ```nix + { + nested = { foo = 1; }; + nested = { ${"ba" + "r"} = 2; }; + } + ``` + + This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: + + ```nix + { nested = { bar = 2; foo = 1; }; } + ``` + + Note that the feature of merging multiple attribute set declarations is of questionable value. + It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. + This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + +* Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 139d07188..c869b5e2f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,8 +1 @@ # Release X.Y (202?-??-??) - -- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand - -* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. - -- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. - Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 41ecbbeb4..c38ea2d2b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -294,10 +294,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index c563a46e9..744e8090b 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "realisation.hh" namespace nix { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 277e77ad5..46fa96d05 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -806,7 +806,7 @@ struct EvalSettings : Config List of directories to be searched for `<...>` file references In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath). + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). )"}; Setting restrictEval{ diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0a1ad9967..217c17382 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -137,6 +137,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, dupAttr(state, ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs.emplace(ad.first, ad.second); } + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); } else { dupAttr(state, attrPath, pos, j->second.pos); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a61e57cc..7ff17b6ee 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1300,9 +1300,10 @@ drvName, Bindings * attrs, Value & v) auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); DerivationOutput::CAFixed dof { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(h)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(h), + }, }; drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); @@ -2164,10 +2165,8 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = *expectedHash, - }, + .method = method, + .hash = *expectedHash, .references = {}, }); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 579a45f92..5e668c629 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -254,10 +254,8 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .hash = { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, - .hash = *expectedHash, - }, + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, .references = {} }); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 2860c1ceb..f86c0604e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -217,10 +217,8 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = *narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = *narHash, .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e42aca6db..a012234e0 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -77,10 +77,8 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Flat, - .hash = hash, - }, + .method = FileIngestionMethod::Flat, + .hash = hash, .references = {}, }, hashString(htSHA256, sink.s), diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index fcd763a9d..b4fea693f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,10 +309,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = nar.first, - }, + .method = method, + .hash = nar.first, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -428,10 +426,8 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -465,8 +461,8 @@ StorePath BinaryCacheStore::addTextToStore( *this, std::string { name }, TextInfo { - { .hash = textHash }, - references, + .hash = textHash, + .references = references, }, nar.first, }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2935b9da9..b7a27490c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,5 +1,5 @@ #include "local-derivation-goal.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -1200,7 +1200,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore { ref next; @@ -1251,11 +1251,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - std::map> queryPartialDerivationOutputMap(const StorePath & path) override + std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); + return next->queryPartialDerivationOutputMap(path, evalStore); } std::optional queryPathFromHashPart(const std::string & hashPart) override @@ -2540,16 +2542,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }, [&](const DerivationOutput::CAFixed & dof) { - auto wanted = dof.ca.getHash(); + auto & wanted = dof.ca.hash; auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.ca.getMethod(), + .method = dof.ca.method, .hashType = wanted.type, }); /* Check wanted hash */ assert(newInfo0.ca); - auto got = newInfo0.ca->getHash(); + auto & got = newInfo0.ca->hash; if (wanted != got) { /* Throw an error after registering the path as valid. */ diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 04f7ac214..080456e18 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,11 +4,6 @@ namespace nix { -std::string FixedOutputHash::printMethodAlgo() const -{ - return makeFileIngestionPrefix(method) + printHashType(hash.type); -} - std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { @@ -42,21 +37,6 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) return method; } -std::string ContentAddress::render() const -{ - return std::visit(overloaded { - [](const TextHash & th) { - return "text:" - + th.hash.to_string(Base32, true); - }, - [](const FixedOutputHash & fsh) { - return "fixed:" - + makeFileIngestionPrefix(fsh.method) - + fsh.hash.to_string(Base32, true); - } - }, raw); -} - std::string ContentAddressMethod::render(HashType ht) const { return std::visit(overloaded { @@ -69,6 +49,20 @@ std::string ContentAddressMethod::render(HashType ht) const }, raw); } +std::string ContentAddress::render() const +{ + return std::visit(overloaded { + [](const TextIngestionMethod &) -> std::string { + return "text:"; + }, + [](const FileIngestionMethod & method) { + return "fixed:" + + makeFileIngestionPrefix(method); + }, + }, method.raw) + + this->hash.to_string(Base32, true); +} + /** * Parses content address strings up to the hash. */ @@ -118,22 +112,12 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; - auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest); - auto hashType = hashType_; // work around clang bug + auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); - return std::visit(overloaded { - [&](TextIngestionMethod &) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, hashType) - }); - }, - [&](FileIngestionMethod & fim) { - return ContentAddress(FixedOutputHash { - .method = fim, - .hash = Hash::parseNonSRIUnprefixed(rest, hashType), - }); - }, - }, caMethod.raw); + return ContentAddress { + .method = std::move(caMethod).raw, + .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + }; } std::pair ContentAddressMethod::parse(std::string_view caMethod) @@ -156,52 +140,10 @@ std::string renderContentAddress(std::optional ca) return ca ? ca->render() : ""; } -ContentAddress ContentAddress::fromParts( - ContentAddressMethod method, Hash hash) noexcept -{ - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddress { - return TextHash { - .hash = std::move(hash), - }; - }, - [&](FileIngestionMethod m2) -> ContentAddress { - return FixedOutputHash { - .method = std::move(m2), - .hash = std::move(hash), - }; - }, - }, method.raw); -} - -ContentAddressMethod ContentAddress::getMethod() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> ContentAddressMethod { - return TextIngestionMethod {}; - }, - [](const FixedOutputHash & fsh) -> ContentAddressMethod { - return fsh.method; - }, - }, raw); -} - -const Hash & ContentAddress::getHash() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> auto & { - return th.hash; - }, - [](const FixedOutputHash & fsh) -> auto & { - return fsh.hash; - }, - }, raw); -} - std::string ContentAddress::printMethodAlgo() const { - return getMethod().renderPrefix() - + printHashType(getHash().type); + return method.renderPrefix() + + printHashType(hash.type); } bool StoreReferences::empty() const @@ -217,19 +159,20 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { return std::visit(overloaded { - [&](const TextHash & h) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { return TextInfo { - .hash = h, + .hash = ca.hash, .references = {}, }; }, - [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { return FixedOutputInfo { - .hash = h, + .method = method, + .hash = ca.hash, .references = {}, }; }, - }, ca.raw); + }, ca.method.raw); } std::optional ContentAddressWithReferences::fromPartsOpt( @@ -241,7 +184,7 @@ std::optional ContentAddressWithReferences::fromPa return std::nullopt; return ContentAddressWithReferences { TextInfo { - .hash = { .hash = std::move(hash) }, + .hash = std::move(hash), .references = std::move(refs.others), } }; @@ -249,10 +192,8 @@ std::optional ContentAddressWithReferences::fromPa [&](FileIngestionMethod m2) -> std::optional { return ContentAddressWithReferences { FixedOutputInfo { - .hash = { - .method = m2, - .hash = std::move(hash), - }, + .method = m2, + .hash = std::move(hash), .references = std::move(refs), } }; @@ -267,7 +208,7 @@ ContentAddressMethod ContentAddressWithReferences::getMethod() const return TextIngestionMethod {}; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.hash.method; + return fsh.method; }, }, raw); } @@ -276,10 +217,10 @@ Hash ContentAddressWithReferences::getHash() const { return std::visit(overloaded { [](const TextInfo & th) { - return th.hash.hash; + return th.hash; }, [](const FixedOutputInfo & fsh) { - return fsh.hash.hash; + return fsh.hash; }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e1e80448b..01b771e52 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -113,37 +113,6 @@ struct ContentAddressMethod * Mini content address */ -/** - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - */ -struct TextHash { - /** - * Hash of the contents of the text/file. - */ - Hash hash; - - GENERATE_CMP(TextHash, me->hash); -}; - -/** - * Used by most store objects that are content-addressed. - */ -struct FixedOutputHash { - /** - * How the file system objects are serialized - */ - FileIngestionMethod method; - /** - * Hash of that serialization - */ - Hash hash; - - std::string printMethodAlgo() const; - - GENERATE_CMP(FixedOutputHash, me->method, me->hash); -}; - /** * We've accumulated several types of content-addressed paths over the * years; fixed-output derivations support multiple hash algorithms and @@ -158,19 +127,17 @@ struct FixedOutputHash { */ struct ContentAddress { - typedef std::variant< - TextHash, - FixedOutputHash - > Raw; + /** + * How the file system objects are serialized + */ + ContentAddressMethod method; - Raw raw; + /** + * Hash of that serialization + */ + Hash hash; - GENERATE_CMP(ContentAddress, me->raw); - - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddress(auto &&... arg) - : raw(std::forward(arg)...) - { } + GENERATE_CMP(ContentAddress, me->method, me->hash); /** * Compute the content-addressability assertion @@ -183,20 +150,6 @@ struct ContentAddress static std::optional parseOpt(std::string_view rawCaOpt); - /** - * Create a `ContentAddress` from 2 parts: - * - * @param method Way ingesting the file system data. - * - * @param hash Hash of ingested file system data. - */ - static ContentAddress fromParts( - ContentAddressMethod method, Hash hash) noexcept; - - ContentAddressMethod getMethod() const; - - const Hash & getHash() const; - std::string printMethodAlgo() const; }; @@ -219,7 +172,8 @@ std::string renderContentAddress(std::optional ca); * References to other store objects are tracked with store paths, self * references however are tracked with a boolean. */ -struct StoreReferences { +struct StoreReferences +{ /** * References to other store objects */ @@ -246,8 +200,13 @@ struct StoreReferences { }; // This matches the additional info that we need for makeTextPath -struct TextInfo { - TextHash hash; +struct TextInfo +{ + /** + * Hash of the contents of the text/file. + */ + Hash hash; + /** * References to other store objects only; self references * disallowed @@ -257,8 +216,18 @@ struct TextInfo { GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo { - FixedOutputHash hash; +struct FixedOutputInfo +{ + /** + * How the file system objects are serialized + */ + FileIngestionMethod method; + + /** + * Hash of that serialization + */ + Hash hash; + /** * References to other store objects or this one. */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ad3dee1a2..8cbf6f044 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -7,6 +7,7 @@ #include "store-cast.hh" #include "gc-store.hh" #include "log-store.hh" +#include "indirect-root-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -675,8 +676,8 @@ static void performOp(TunnelLogger * logger, ref store, Path path = absPath(readString(from)); logger->startWork(); - auto & gcStore = require(*store); - gcStore.addIndirectRoot(path); + auto & indirectRootStore = require(*store); + indirectRootStore.addIndirectRoot(path); logger->stopWork(); to << 1; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6f63685d4..b831b2fe5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -232,9 +232,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, validatePath(pathS); auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(hash)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(hash), + }, }; } else { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -395,7 +396,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -628,7 +629,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto & dof = std::get(i.second.raw()); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.getHash().to_string(Base16, false) + ":" + + dof.ca.hash.to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -780,7 +781,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.getHash().to_string(Base16, false); + << dof.ca.hash.to_string(Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -970,7 +971,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.getHash().to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { @@ -1017,9 +1018,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "path", "hashAlgo", "hash" })) { auto [method, hashType] = methodAlgo(); auto dof = DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)), + .ca = ContentAddress { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) throw Error("Path doesn't match derivation output"); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 6ea80c92e..7a4261ce0 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,7 +3,6 @@ #include "util.hh" #include "path.hh" -#include "realisation.hh" #include "outputs-spec.hh" #include "comparator.hh" diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 2c26c65c4..ab1059fb1 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -71,19 +71,36 @@ struct GCResults }; +/** + * Mix-in class for \ref Store "stores" which expose a notion of garbage + * collection. + * + * Garbage collection will allow deleting paths which are not + * transitively "rooted". + * + * The notion of GC roots actually not part of this class. + * + * - The base `Store` class has `Store::addTempRoot()` because for a store + * that doesn't support garbage collection at all, a temporary GC root is + * safely implementable as no-op. + * + * @todo actually this is not so good because stores are *views*. + * Some views have only a no-op temp roots even though others to the + * same store allow triggering GC. For instance one can't add a root + * over ssh, but that doesn't prevent someone from gc-ing that store + * accesed via SSH locally). + * + * - The derived `LocalFSStore` class has `LocalFSStore::addPermRoot`, + * which is not part of this class because it relies on the notion of + * an ambient file system. There are stores (`ssh-ng://`, for one), + * that *do* support garbage collection but *don't* expose any file + * system, and `LocalFSStore::addPermRoot` thus does not make sense + * for them. + */ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /** - * Add an indirect root, which is merely a symlink to `path` from - * `/nix/var/nix/gcroots/auto/`. `path` is supposed - * to be a symlink to a store path. The garbage collector will - * automatically remove the indirect root when it finds that - * `path` has disappeared. - */ - virtual void addIndirectRoot(const Path & path) = 0; - /** * Find the roots of the garbage collector. Each root is a pair * `(link, storepath)` where `link` is the path of the symlink diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 566dd3963..ea2868c58 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,7 +1,6 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" -#include "local-fs-store.hh" #include "finally.hh" #include @@ -50,7 +49,7 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) +Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) { Path gcRoot(canonPath(_gcRoot)); diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh new file mode 100644 index 000000000..59e45af45 --- /dev/null +++ b/src/libstore/indirect-root-store.hh @@ -0,0 +1,48 @@ +#pragma once +///@file + +#include "local-fs-store.hh" + +namespace nix { + +/** + * Mix-in class for implementing permanent roots as a pair of a direct + * (strong) reference and indirect weak reference to the first + * reference. + * + * See methods for details on the operations it represents. + */ +struct IndirectRootStore : public virtual LocalFSStore +{ + inline static std::string operationName = "Indirect GC roots registration"; + + /** + * Implementation of `LocalFSStore::addPermRoot` where the permanent + * root is a pair of + * + * - The user-facing symlink which all implementations must create + * + * - An additional weak reference known as the "indirect root" that + * points to that symlink. + * + * The garbage collector will automatically remove the indirect root + * when it finds that the symlink has disappeared. + * + * The implementation of this method is concrete, but it delegates + * to `addIndirectRoot()` which is abstract. + */ + Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final; + + /** + * Add an indirect root, which is a weak reference to the + * user-facing symlink created by `addPermRoot()`. + * + * @param path user-facing and user-controlled symlink to a store + * path. + * + * The form this weak-reference takes is implementation-specific. + */ + virtual void addIndirectRoot(const Path & path) = 0; +}; + +} diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 2ee2ef0c8..488109501 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -40,6 +40,7 @@ class LocalFSStore : public virtual LocalFSStoreConfig, public virtual LogStore { public: + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; @@ -49,9 +50,20 @@ public: ref getFSAccessor() override; /** - * Register a permanent GC root. + * Creates symlink from the `gcRoot` to the `storePath` and + * registers the `gcRoot` as a permanent GC root. The `gcRoot` + * symlink lives outside the store and is created and owned by the + * user. + * + * @param gcRoot The location of the symlink. + * + * @param storePath The store object being rooted. The symlink will + * point to `toRealPath(store.printStorePath(storePath))`. + * + * How the permanent GC root corresponding to this symlink is + * managed is implementation-specific. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot); + virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1e6b396a9..21acb3c38 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1028,10 +1028,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path) { - auto path = path_; - auto outputs = retrySQLite>>([&]() { + return retrySQLite>>([&]() { auto state(_state.lock()); std::map> outputs; uint64_t drvId; @@ -1043,21 +1042,6 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - - if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - return outputs; - - auto drv = readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto& [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); - if (realisation) - outputs.insert_or_assign(outputName, realisation->outPath); - else - outputs.insert({outputName, std::nullopt}); - } - - return outputs; } std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) @@ -1255,27 +1239,17 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&info.ca->raw)) { - auto actualFoHash = hashCAPath( - foHash->method, - foHash->hash.type, - info.path - ); - if (foHash->hash != actualFoHash.hash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - foHash->hash.to_string(Base32, true), - actualFoHash.hash.to_string(Base32, true)); - } - } - if (auto textHash = std::get_if(&info.ca->raw)) { - auto actualTextHash = hashString(htSHA256, readFile(realPath)); - if (textHash->hash != actualTextHash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - textHash->hash.to_string(Base32, true), - actualTextHash.to_string(Base32, true)); - } + auto & specified = *info.ca; + auto actualHash = hashCAPath( + specified.method, + specified.hash.type, + info.path + ); + if (specified.hash != actualHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + specified.hash.to_string(Base32, true), + actualHash.hash.to_string(Base32, true)); } } @@ -1355,10 +1329,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -1434,8 +1406,8 @@ StorePath LocalStore::addTextToStore( { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, TextInfo { - { .hash = hash }, - references, + .hash = hash, + .references = references, }); addTempRoot(dstPath); @@ -1465,7 +1437,10 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); info.references = references; - info.ca = TextHash { .hash = hash }; + info.ca = { + .method = TextIngestionMethod {}, + .hash = hash, + }; registerValidPath(info); } @@ -1862,33 +1837,39 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, const HashType & hashType, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path) { return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash ) { HashModuloSink caSink ( hashType, std::string(pathHash) ); - switch (method) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - auto hash = caSink.finish().first; - return FixedOutputHash{ + std::visit(overloaded { + [&](const TextIngestionMethod &) { + readFile(path, caSink); + }, + [&](const FileIngestionMethod & m2) { + switch (m2) { + case FileIngestionMethod::Recursive: + dumpPath(path, caSink); + break; + case FileIngestionMethod::Flat: + readFile(path, caSink); + break; + } + }, + }, method.raw); + return ContentAddress { .method = method, - .hash = hash, + .hash = caSink.finish().first, }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae548cca9..9a44722d4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,8 +5,7 @@ #include "pathlocks.hh" #include "store-api.hh" -#include "local-fs-store.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "sync.hh" #include "util.hh" @@ -68,7 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig std::string doc() override; }; -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore +class LocalStore : public virtual LocalStoreConfig + , public virtual IndirectRootStore + , public virtual GcStore { private: @@ -165,7 +166,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryStaticPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; @@ -209,6 +210,12 @@ private: public: + /** + * Implementation of IndirectRootStore::addIndirectRoot(). + * + * The weak reference merely is a symlink to `path' from + * /nix/var/nix/gcroots/auto/. + */ void addIndirectRoot(const Path & path) override; private: @@ -352,13 +359,13 @@ private: void signRealisation(Realisation &); // XXX: Make a generic `Store` method - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path); - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 626a22480..253609ed2 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,10 +52,8 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, .references = std::move(refs), }, Hash::dummy, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 50336c779..14160dc8b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -310,43 +310,34 @@ std::map drvOutputReferences( OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { - auto & evalStore = evalStore_ ? *evalStore_ : store; + auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_); - OutputPathMap outputs; - auto drv = evalStore.readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(store, drv); - auto drvOutputs = drv.outputsAndOptPaths(store); - auto outputNames = std::visit(overloaded { + auto outputsOpt = std::visit(overloaded { [&](const OutputsSpec::All &) { - StringSet names; - for (auto & [outputName, _] : drv.outputs) - names.insert(outputName); - return names; + // Keep all outputs + return std::move(outputsOpt_); }, [&](const OutputsSpec::Names & names) { - return static_cast>(names); + // Get just those mentioned by name + std::map> outputsOpt; + for (auto & output : names) { + auto * pOutputPathOpt = get(outputsOpt_, output); + if (!pOutputPathOpt) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store.printStorePath(bfd.drvPath), output); + outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); + } + return outputsOpt; }, }, bfd.outputs.raw()); - for (auto & output : outputNames) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - DrvOutput outputId { *outputHash, output }; - auto realisation = store.queryRealisation(outputId); - if (!realisation) - throw MissingRealisation(outputId); - outputs.insert_or_assign(output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); - outputs.insert_or_assign(output, *drvOutput->second); - } + + OutputPathMap outputs; + for (auto & [outputName, outputPathOpt] : outputsOpt) { + if (!outputPathOpt) + throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName); + auto & outputPath = *outputPathOpt; + outputs.insert_or_assign(outputName, outputPath); } return outputs; } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 981bbfb14..ccb57104f 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -29,14 +29,14 @@ std::optional ValidPathInfo::contentAddressWithRef return std::nullopt; return std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - .hash = th, + .hash = ca->hash, .references = references, }; }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -44,14 +44,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .hash = foh, + .method = m2, + .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; }, - }, ca->raw); + }, ca->method.raw); } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -110,13 +111,19 @@ ValidPathInfo::ValidPathInfo( std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = std::move((TextHash &&) ti); + this->ca = ContentAddress { + .method = TextIngestionMethod {}, + .hash = std::move(ti.hash), + }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = std::move((FixedOutputHash &&) foi); + this->ca = ContentAddress { + .method = std::move(foi.method), + .hash = std::move(foi.hash), + }; }, }, std::move(ca).raw); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 2a093c128..0548b30c1 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" #include #include "comparator.hh" #include "crypto.hh" @@ -143,9 +144,13 @@ class MissingRealisation : public Error { public: MissingRealisation(DrvOutput & outputId) - : Error( "cannot operate on an output of the " + : MissingRealisation(outputId.outputName, outputId.strHash()) + {} + MissingRealisation(std::string_view drv, std::string outputName) + : Error( "cannot operate on output '%s' of the " "unbuilt derivation '%s'", - outputId.to_string()) + outputName, + drv) {} }; diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index d32d91a60..ce4740a9c 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,5 +1,6 @@ #include "remote-store.hh" #include "worker-protocol.hh" +#include "pool.hh" namespace nix { @@ -94,4 +95,34 @@ struct RemoteStore::Connection std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; +/** + * A wrapper around Pool::Handle that marks + * the connection as bad (causing it to be closed) if a non-daemon + * exception is thrown before the handle is closed. Such an exception + * causes a deviation from the expected protocol and therefore a + * desynchronization between the client and daemon. + */ +struct RemoteStore::ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle(); + + RemoteStore::Connection & operator * () { return *handle; } + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + + void withFramedSink(std::function fun); +}; + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1e2104e1f..21258daec 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -159,49 +159,25 @@ void RemoteStore::setOptions(Connection & conn) } -/* A wrapper around Pool::Handle that marks - the connection as bad (causing it to be closed) if a non-daemon - exception is thrown before the handle is closed. Such an exception - causes a deviation from the expected protocol and therefore a - desynchronization between the client and daemon. */ -struct ConnectionHandle +RemoteStore::ConnectionHandle::~ConnectionHandle() { - Pool::Handle handle; - bool daemonException = false; - - ConnectionHandle(Pool::Handle && handle) - : handle(std::move(handle)) - { } - - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } - - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); } +} - RemoteStore::Connection * operator -> () { return &*handle; } - RemoteStore::Connection & operator * () { return *handle; } - - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) - { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; - std::rethrow_exception(ex); - } +void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush) +{ + auto ex = handle->processStderr(sink, source, flush); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); } - - void withFramedSink(std::function fun); -}; +} -ConnectionHandle RemoteStore::getConnection() +RemoteStore::ConnectionHandle RemoteStore::getConnection() { return ConnectionHandle(connections->get()); } @@ -378,27 +354,36 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_) { if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { - auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return WorkerProto::Serialise>>::read(*this, *conn); + if (!evalStore_) { + auto conn(getConnection()); + conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return WorkerProto::Serialise>>::read(*this, *conn); + } else { + auto & evalStore = *evalStore_; + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + // union with the first branch overriding the statically-known ones + // when non-`std::nullopt`. + for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) { + if (optPath) + outputs.insert_or_assign(std::move(outputName), std::move(optPath)); + else + outputs.insert({std::move(outputName), std::nullopt}); + } + return outputs; + } } else { + auto & evalStore = evalStore_ ? *evalStore_ : *this; // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an // under-approximation as it only returns the paths that can be inferred // from the derivation itself (and not the ones that are known because // the have been built), but as old stores don't handle floating-CA // derivations this shouldn't matter - auto derivation = readDerivation(path); - auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); - std::map> ret; - for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { - ret.emplace(outputName, outputAndPath.second); - } - return ret; + return evalStore.queryStaticPartialDerivationOutputMap(path); } } @@ -837,15 +822,6 @@ void RemoteStore::addTempRoot(const StorePath & path) } -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << WorkerProto::Op::AddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); @@ -1090,7 +1066,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void ConnectionHandle::withFramedSink(std::function fun) +void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index cb7a71acf..a1ae82a0f 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -17,7 +17,6 @@ class Pid; struct FdSink; struct FdSource; template class Pool; -struct ConnectionHandle; struct RemoteStoreConfig : virtual StoreConfig { @@ -63,7 +62,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -127,8 +126,6 @@ public: void addTempRoot(const StorePath & path) override; - void addIndirectRoot(const Path & path) override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -182,6 +179,8 @@ protected: void setOptions() override; + struct ConnectionHandle; + ConnectionHandle getConnection(); friend struct ConnectionHandle; @@ -199,5 +198,4 @@ private: std::shared_ptr evalStore); }; - } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0200076c0..9c6c42ef4 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,5 +1,6 @@ #include "ssh-store-config.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" #include "remote-fs-accessor.hh" @@ -61,7 +62,7 @@ public: std::optional getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } -private: +protected: struct Connection : RemoteStore::Connection { @@ -93,9 +94,12 @@ private: ref SSHStore::openConnection() { auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --stdio", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + + std::string command = remoteProgram + " --stdio"; + if (remoteStore.get() != "") + command += " --store " + shellEscape(remoteStore.get()); + + conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); return conn; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 862a39490..43bec7604 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -185,15 +185,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.hash.method) - + info.hash.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } @@ -201,13 +201,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.hash.type == htSHA256); + assert(info.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash.hash, + info.hash, name); } @@ -233,10 +233,8 @@ std::pair Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); @@ -249,8 +247,8 @@ StorePath Store::computeStorePathForText( const StorePathSet & references) const { return makeTextPath(name, TextInfo { - { .hash = hashString(htSHA256, s) }, - references, + .hash = hashString(htSHA256, s), + .references = references, }); } @@ -442,10 +440,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = {}, }, narHash, @@ -497,22 +493,50 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +std::map> Store::queryStaticPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); - for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + for (auto & [outputName, output] : drv.outputsAndOptPaths(*this)) { outputs.emplace(outputName, output.second); } return outputs; } +std::map> Store::queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : *this; + + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return outputs; + + auto drv = evalStore.readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, hash] : drvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) { + outputs.insert_or_assign(outputName, realisation->outPath); + } else { + // queryStaticPartialDerivationOutputMap is not guaranteed + // to return std::nullopt for outputs which are not + // statically known. + outputs.insert({outputName, std::nullopt}); + } + } + + return outputs; +} + OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); + throw MissingRealisation(printStorePath(path), outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eef..3758c730f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -99,6 +99,8 @@ typedef std::map> StorePathCAMap; struct StoreConfig : public Config { + typedef std::map Params; + using Config::Config; StoreConfig() = delete; @@ -153,10 +155,6 @@ struct StoreConfig : public Config class Store : public std::enable_shared_from_this, public virtual StoreConfig { -public: - - typedef std::map Params; - protected: struct PathInfoCacheValue { @@ -425,7 +423,20 @@ public: * derivation. All outputs are mentioned so ones mising the mapping * are mapped to `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + virtual std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr); + + /** + * Like `queryPartialDerivationOutputMap` but only considers + * statically known output paths (i.e. those that can be gotten from + * the derivation itself. + * + * Just a helper function for implementing + * `queryPartialDerivationOutputMap`. + */ + virtual std::map> queryStaticPartialDerivationOutputMap( + const StorePath & path); /** * Query the mapping outputName=>outputPath for the given derivation. diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 6328ad370..0e28c1f08 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -81,7 +81,7 @@ TEST_JSON(DerivationTest, caFixedFlat, "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -95,7 +95,7 @@ TEST_JSON(DerivationTest, caFixedNAR, "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Recursive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -109,7 +109,7 @@ TEST_JSON(DynDerivationTest, caFixedText, "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = TextHash { + .ca = { .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 69dae2da5..99589f8b2 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "worker-protocol.hh" #include #include @@ -77,6 +78,15 @@ ref UDSRemoteStore::openConnection() } +void UDSRemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 2bd6517fa..cdb28a001 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -3,13 +3,13 @@ #include "remote-store.hh" #include "remote-store-connection.hh" -#include "local-fs-store.hh" +#include "indirect-root-store.hh" namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Store::Params & params) + UDSRemoteStoreConfig(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) @@ -21,7 +21,9 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::string doc() override; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore +class UDSRemoteStore : public virtual UDSRemoteStoreConfig + , public virtual IndirectRootStore + , public virtual RemoteStore { public: @@ -39,6 +41,16 @@ public: void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } + /** + * Implementation of `IndirectRootStore::addIndirectRoot()` which + * delegates to the remote store. + * + * The idea is that the client makes the direct symlink, so it is + * owned managed by the client's user account, and the server makes + * the indirect symlink. + */ + void addIndirectRoot(const Path & path) override; + private: struct Connection : RemoteStore::Connection diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index caa0248f1..94956df66 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,10 +220,8 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = Hash::parseAny(hash, hashAlgo), - }, + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 16e48a39b..39e5cc99d 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,10 +45,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - .hash = { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - }, + .method = std::move(ingestionMethod), + .hash = std::move(hash), .references = {}, }, narHash, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3b2e225f6..b67d381ca 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -71,10 +71,8 @@ std::tuple prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .hash = { - .method = ingestionMethod, - .hash = *expectedHash, - }, + .method = ingestionMethod, + .hash = *expectedHash, .references = {}, }); if (store->isValidPath(*storePath)) @@ -127,7 +125,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = info.ca->getHash(); + hash = info.ca->hash; } return {storePath.value(), hash.value()}; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b833b5192..476ddcd60 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -222,10 +222,8 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narHash, .references = { .others = std::move(references), // profiles never refer to themselves diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp new file mode 100644 index 000000000..e01f8e6d0 --- /dev/null +++ b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -0,0 +1,8 @@ +error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: + + 2| set = { "${"" + "b"}" = 1; }; + 3| set = { "${"b" + ""}" = 2; }; + | ^ + 4| } diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.nix b/tests/lang/eval-fail-dup-dynamic-attrs.nix new file mode 100644 index 000000000..7ea17f6c8 --- /dev/null +++ b/tests/lang/eval-fail-dup-dynamic-attrs.nix @@ -0,0 +1,4 @@ +{ + set = { "${"" + "b"}" = 1; }; + set = { "${"b" + ""}" = 2; }; +} diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.exp b/tests/lang/eval-okay-merge-dynamic-attrs.exp new file mode 100644 index 000000000..157d677ce --- /dev/null +++ b/tests/lang/eval-okay-merge-dynamic-attrs.exp @@ -0,0 +1 @@ +{ set1 = { a = 1; b = 2; }; set2 = { a = 1; b = 2; }; set3 = { a = 1; b = 2; }; set4 = { a = 1; b = 2; }; } diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.nix b/tests/lang/eval-okay-merge-dynamic-attrs.nix new file mode 100644 index 000000000..f459a554f --- /dev/null +++ b/tests/lang/eval-okay-merge-dynamic-attrs.nix @@ -0,0 +1,13 @@ +{ + set1 = { a = 1; }; + set1 = { "${"b" + ""}" = 2; }; + + set2 = { "${"b" + ""}" = 2; }; + set2 = { a = 1; }; + + set3.a = 1; + set3."${"b" + ""}" = 2; + + set4."${"b" + ""}" = 2; + set4.a = 1; +}