From 3c109095de0eca89cef4e53f988d3a0a1ba37c4e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Feb 2025 15:24:50 +0100 Subject: [PATCH 01/10] Use 'nix flake prefetch' to get the store path of a flake Future-proofing for when 'nix flake metadata' won't return a store path anymore. --- tests/functional/flakes/flake-in-submodule.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index f98c19aa8..fc732a7e2 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -74,7 +74,7 @@ EOF git -C "$rootRepo" add flake.nix git -C "$rootRepo" commit -m "Add flake.nix" -storePath=$(nix flake metadata --json "$rootRepo?submodules=1" | jq -r .path) +storePath=$(nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath) [[ -e "$storePath/submodule" ]] # The root repo may use the submodule repo as an input From 1ab97a70f5975403644d031cff2ea5bdb559b6fa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Feb 2025 16:14:34 +0100 Subject: [PATCH 02/10] fetchOrSubstituteTree(): Return an accessor This prepares lazy access to flake.nix etc. --- src/libfetchers/fetchers.cc | 6 ++-- src/libflake/flake/flake.cc | 58 +++++++++++++++++++++++++--------- src/libflake/flake/flakeref.cc | 6 ++++ src/libflake/flake/flakeref.hh | 3 ++ 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index aadeecba2..e908a6be2 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -200,10 +200,6 @@ std::pair Input::fetchToStore(ref store) const auto narHash = store->queryPathInfo(storePath)->narHash; result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - // FIXME: we would like to mark inputs as final in - // getAccessorUnchecked(), but then we can't add - // narHash. Or maybe narHash should be excluded from the - // concept of "final" inputs? result.attrs.insert_or_assign("__final", Explicit(true)); assert(result.isFinal()); @@ -284,6 +280,8 @@ std::pair, Input> Input::getAccessor(ref store) const try { auto [accessor, result] = getAccessorUnchecked(store); + result.attrs.insert_or_assign("__final", Explicit(true)); + checkLocks(*this, result); return {accessor, std::move(result)}; diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 9dc73d053..35595a535 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -12,6 +12,7 @@ #include "flake/settings.hh" #include "value-to-json.hh" #include "local-fs-store.hh" +#include "fetch-to-store.hh" #include @@ -24,7 +25,7 @@ namespace flake { struct FetchedFlake { FlakeRef lockedRef; - StorePath storePath; + ref accessor; }; typedef std::map FlakeCache; @@ -40,7 +41,7 @@ static std::optional lookupInFlakeCache( return i->second; } -static std::tuple fetchOrSubstituteTree( +static std::tuple, FlakeRef, FlakeRef> fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, bool useRegistries, @@ -51,8 +52,8 @@ static std::tuple fetchOrSubstituteTree( if (!fetched) { if (originalRef.input.isDirect()) { - auto [storePath, lockedRef] = originalRef.fetchTree(state.store); - fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); + auto [accessor, lockedRef] = originalRef.lazyFetch(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor}); } else { if (useRegistries) { resolvedRef = originalRef.resolve( @@ -64,8 +65,8 @@ static std::tuple fetchOrSubstituteTree( }); fetched = lookupInFlakeCache(flakeCache, originalRef); if (!fetched) { - auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store); - fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); + auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor}); } flakeCache.insert_or_assign(resolvedRef, *fetched); } @@ -76,14 +77,28 @@ static std::tuple fetchOrSubstituteTree( flakeCache.insert_or_assign(originalRef, *fetched); } - debug("got tree '%s' from '%s'", - state.store->printStorePath(fetched->storePath), fetched->lockedRef); + debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedRef); - state.allowPath(fetched->storePath); + return {fetched->accessor, resolvedRef, fetched->lockedRef}; +} +static StorePath copyInputToStore( + EvalState & state, + fetchers::Input & input, + ref accessor) +{ + auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName()); + + state.allowPath(storePath); + + auto narHash = state.store->queryPathInfo(storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + + #if 0 assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store)); + #endif - return {fetched->storePath, resolvedRef, fetched->lockedRef}; + return storePath; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -136,7 +151,9 @@ static FlakeInput parseFlakeInput( url = attr.value->string_view(); else if (attr.value->type() == nPath) { auto path = attr.value->path(); - if (path.accessor != flakeDir.accessor) + if (path.accessor != flakeDir.accessor + // FIXME: hack necessary since the parser currently stores all paths as inside rootFS. + && flakeDir.accessor == state.rootFS) throw Error("input attribute path '%s' at %s must be in the same source tree as %s", path, state.positions[attr.pos], flakeDir); url = "path:" + flakeDir.path.makeRelative(path.path); @@ -301,10 +318,12 @@ static Flake readFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - NixStringContext emptyContext = {}; + // FIXME: hack necessary since the parser currently stores all paths as inside rootFS. + SourcePath path(rootDir.accessor, setting.value->path().path); + auto storePath = fetchToStore(*state.store, path, FetchMode::Copy); flake.config.settings.emplace( state.symbols[setting.name], - state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true).toOwned()); + state.store->toRealPath(storePath)); } else if (setting.value->type() == nInt) flake.config.settings.emplace( @@ -349,9 +368,14 @@ static Flake getFlake( FlakeCache & flakeCache, const InputAttrPath & lockRootAttrPath) { - auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( + // Fetch a lazy tree first. + auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, useRegistries, flakeCache); + // Copy the tree to the store. + auto storePath = copyInputToStore(state, lockedRef.input, accessor); + + // Re-parse flake.nix from the store. return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootAttrPath); } @@ -707,8 +731,12 @@ LockedFlake lockFlake( if (auto resolvedPath = resolveRelativePath()) { return {*resolvedPath, *input.ref}; } else { - auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, *input.ref, useRegistries, flakeCache); + + // FIXME: allow input to be lazy. + auto storePath = copyInputToStore(state, lockedRef.input, accessor); + return {state.rootPath(state.store->toRealPath(storePath)), lockedRef}; } }(); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index fbe61c294..cc9403ceb 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -289,6 +289,12 @@ std::pair FlakeRef::fetchTree(ref store) const return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; } +std::pair, FlakeRef> FlakeRef::lazyFetch(ref store) const +{ + auto [accessor, lockedInput] = input.getAccessor(store); + return {accessor, FlakeRef(std::move(lockedInput), subdir)}; +} + std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( const fetchers::Settings & fetchSettings, const std::string & url, diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index c9cf7952d..f47730da5 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -71,7 +71,10 @@ struct FlakeRef const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs); + // FIXME: remove std::pair fetchTree(ref store) const; + + std::pair, FlakeRef> lazyFetch(ref store) const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); From c74ad6b32bd073bb93ea03618c472a3d25b8f12a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Feb 2025 16:26:22 +0100 Subject: [PATCH 03/10] Remove FlakeRef::fetchTree() --- src/libcmd/common-eval-args.cc | 8 ++++++-- src/libflake/flake/flakeref.cc | 6 ------ src/libflake/flake/flakeref.hh | 3 --- src/nix/flake.cc | 4 +++- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index de967e3fe..adf816494 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -34,7 +34,9 @@ EvalSettings evalSettings { // FIXME `parseFlakeRef` should take a `std::string_view`. auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; + auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); + auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); + state.allowPath(storePath); return state.rootPath(state.store->toRealPath(storePath)); }, }, @@ -183,7 +185,9 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; + auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); + auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); + state.allowPath(storePath); return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index cc9403ceb..4fc720eb5 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -283,12 +283,6 @@ FlakeRef FlakeRef::fromAttrs( fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } -std::pair FlakeRef::fetchTree(ref store) const -{ - auto [storePath, lockedInput] = input.fetchToStore(store); - return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; -} - std::pair, FlakeRef> FlakeRef::lazyFetch(ref store) const { auto [accessor, lockedInput] = input.getAccessor(store); diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index f47730da5..d3c15018e 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -71,9 +71,6 @@ struct FlakeRef const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs); - // FIXME: remove - std::pair fetchTree(ref store) const; - std::pair, FlakeRef> lazyFetch(ref store) const; }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index adf391b97..6f220b495 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -17,6 +17,7 @@ #include "eval-cache.hh" #include "markdown.hh" #include "users.hh" +#include "fetch-to-store.hh" #include #include @@ -1449,7 +1450,8 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON { auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); - auto [storePath, lockedRef] = resolvedRef.fetchTree(store); + auto [accessor, lockedRef] = resolvedRef.lazyFetch(store); + auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName()); auto hash = store->queryPathInfo(storePath)->narHash; if (json) { From 01598487b79ba167be4040e0c8a362c068d8155d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Feb 2025 16:41:10 +0100 Subject: [PATCH 04/10] Add FIXME --- src/libfetchers/fetchers.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e908a6be2..abf021554 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -186,6 +186,7 @@ bool Input::contains(const Input & other) const return false; } +// FIXME: remove std::pair Input::fetchToStore(ref store) const { if (!scheme) From 25fcc8d1aba201cb8d84d29a1f7a40b8fb1a0ce5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Feb 2025 20:51:14 +0100 Subject: [PATCH 05/10] Add `inputs.self.submodules` flake attribute This allows a flake to specify that it needs Git submodules to be enabled (or disabled, if we ever change the default) on the top-level flake. This requires the input to be refetched, but since the first fetch is lazy, this shouldn't be expensive. Currently the only attribute allowed by `inputs.self` is `submodules`, but more can be added in the future (e.g. a `lazy` attribute to opt in to lazy tree behaviour). Fixes #5312, #9842. --- src/libflake/flake/flake.cc | 147 ++++++++++++------ src/libflake/flake/flake.hh | 15 +- tests/functional/flakes/flake-in-submodule.sh | 14 ++ 3 files changed, 128 insertions(+), 48 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 35595a535..eee22d516 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -116,12 +116,47 @@ static void expectType(EvalState & state, ValueType type, showType(type), showType(value.type()), state.positions[pos]); } -static std::map parseFlakeInputs( +static std::pair, fetchers::Attrs> parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, const InputAttrPath & lockRootAttrPath, - const SourcePath & flakeDir); + const SourcePath & flakeDir, + bool allowSelf); + +static void parseFlakeInputAttr( + EvalState & state, + const Attr & attr, + fetchers::Attrs & attrs) +{ + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" + switch (attr.value->type()) { + case nString: + attrs.emplace(state.symbols[attr.name], attr.value->c_str()); + break; + case nBool: + attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); + break; + case nInt: { + auto intValue = attr.value->integer().value; + if (intValue < 0) + state.error("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow(); + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); + break; + } + default: + if (attr.name == state.symbols.create("publicKeys")) { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + NixStringContext emptyContext = {}; + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, attr.pos, emptyContext).dump()); + } else + state.error("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)).debugThrow(); + } + #pragma GCC diagnostic pop +} static FlakeInput parseFlakeInput( EvalState & state, @@ -166,44 +201,14 @@ static FlakeInput parseFlakeInput( expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputAttrPath(attr.value->c_str())); follows.insert(follows.begin(), lockRootAttrPath.begin(), lockRootAttrPath.end()); input.follows = follows; - } else { - // Allow selecting a subset of enum values - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wswitch-enum" - switch (attr.value->type()) { - case nString: - attrs.emplace(state.symbols[attr.name], attr.value->c_str()); - break; - case nBool: - attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); - break; - case nInt: { - auto intValue = attr.value->integer().value; - - if (intValue < 0) { - state.error("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow(); - } - - attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); - break; - } - default: - if (attr.name == state.symbols.create("publicKeys")) { - experimentalFeatureSettings.require(Xp::VerifiedFetches); - NixStringContext emptyContext = {}; - attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); - } else - state.error("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)).debugThrow(); - } - #pragma GCC diagnostic pop - } + } else + parseFlakeInputAttr(state, attr, attrs); } catch (Error & e) { e.addTrace( state.positions[attr.pos], @@ -233,28 +238,39 @@ static FlakeInput parseFlakeInput( return input; } -static std::map parseFlakeInputs( +static std::pair, fetchers::Attrs> parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, const InputAttrPath & lockRootAttrPath, - const SourcePath & flakeDir) + const SourcePath & flakeDir, + bool allowSelf) { std::map inputs; + fetchers::Attrs selfAttrs; expectType(state, nAttrs, *value, pos); for (auto & inputAttr : *value->attrs()) { - inputs.emplace(state.symbols[inputAttr.name], - parseFlakeInput(state, - state.symbols[inputAttr.name], - inputAttr.value, - inputAttr.pos, - lockRootAttrPath, - flakeDir)); + auto inputName = state.symbols[inputAttr.name]; + if (inputName == "self") { + if (!allowSelf) + throw Error("'self' input attribute not allowed at %s", state.positions[inputAttr.pos]); + expectType(state, nAttrs, *inputAttr.value, inputAttr.pos); + for (auto & attr : *inputAttr.value->attrs()) + parseFlakeInputAttr(state, attr, selfAttrs); + } else { + inputs.emplace(inputName, + parseFlakeInput(state, + inputName, + inputAttr.value, + inputAttr.pos, + lockRootAttrPath, + flakeDir)); + } } - return inputs; + return {inputs, selfAttrs}; } static Flake readFlake( @@ -286,8 +302,11 @@ static Flake readFlake( auto sInputs = state.symbols.create("inputs"); - if (auto inputs = vInfo.attrs()->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootAttrPath, flakeDir); + if (auto inputs = vInfo.attrs()->get(sInputs)) { + auto [flakeInputs, selfAttrs] = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootAttrPath, flakeDir, true); + flake.inputs = std::move(flakeInputs); + flake.selfAttrs = std::move(selfAttrs); + } auto sOutputs = state.symbols.create("outputs"); @@ -361,6 +380,23 @@ static Flake readFlake( return flake; } +static FlakeRef applySelfAttrs( + const FlakeRef & ref, + const Flake & flake) +{ + auto newRef(ref); + + std::set allowedAttrs{"submodules"}; + + for (auto & attr : flake.selfAttrs) { + if (!allowedAttrs.contains(attr.first)) + throw Error("flake 'self' attribute '%s' is not supported", attr.first); + newRef.input.attrs.insert_or_assign(attr.first, attr.second); + } + + return newRef; +} + static Flake getFlake( EvalState & state, const FlakeRef & originalRef, @@ -372,6 +408,22 @@ static Flake getFlake( auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, useRegistries, flakeCache); + // Parse/eval flake.nix to get at the input.self attributes. + auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {accessor}, lockRootAttrPath); + + // Re-fetch the tree if necessary. + auto newLockedRef = applySelfAttrs(lockedRef, flake); + + if (lockedRef != newLockedRef) { + debug("refetching input '%s' due to self attribute", newLockedRef); + // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. + newLockedRef.input.attrs.erase("narHash"); + auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree( + state, newLockedRef, false, flakeCache); + accessor = accessor2; + lockedRef = lockedRef2; + } + // Copy the tree to the store. auto storePath = copyInputToStore(state, lockedRef.input, accessor); @@ -492,6 +544,7 @@ LockedFlake lockFlake( /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ for (auto & [id, input] : flakeInputs) { + //if (id == "self") continue; for (auto & [idOverride, inputOverride] : input.overrides) { auto inputAttrPath(inputAttrPathPrefix); inputAttrPath.push_back(id); diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index 835d0ee06..8d9b9a698 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -79,24 +79,37 @@ struct Flake * The original flake specification (by the user) */ FlakeRef originalRef; + /** * registry references and caching resolved to the specific underlying flake */ FlakeRef resolvedRef; + /** * the specific local store result of invoking the fetcher */ FlakeRef lockedRef; + /** * The path of `flake.nix`. */ SourcePath path; + /** - * pretend that 'lockedRef' is dirty + * Pretend that `lockedRef` is dirty. */ bool forceDirty = false; + std::optional description; + FlakeInputs inputs; + + /** + * Attributes to be retroactively applied to the `self` input + * (such as `submodules = true`). + */ + fetchers::Attrs selfAttrs; + /** * 'nixConfig' attribute */ diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index fc732a7e2..b46eea2e0 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -77,6 +77,20 @@ git -C "$rootRepo" commit -m "Add flake.nix" storePath=$(nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath) [[ -e "$storePath/submodule" ]] +# Test the use of inputs.self. +cat > "$rootRepo"/flake.nix < Date: Tue, 4 Feb 2025 22:14:38 +0100 Subject: [PATCH 06/10] Add a test for a flake referencing a flake that has inputs.self --- tests/functional/flakes/flake-in-submodule.sh | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index b46eea2e0..fe5acf26d 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -27,6 +27,7 @@ git config --global protocol.file.allow always rootRepo=$TEST_ROOT/rootRepo subRepo=$TEST_ROOT/submodule +otherRepo=$TEST_ROOT/otherRepo createGitRepo "$subRepo" @@ -91,6 +92,33 @@ git -C "$rootRepo" commit -a -m "Bla" storePath=$(nix eval --raw "$rootRepo#foo") [[ -e "$storePath/submodule" ]] + +# Test another repo referring to a repo that uses inputs.self. +createGitRepo "$otherRepo" +cat > "$otherRepo"/flake.nix < Date: Thu, 6 Feb 2025 16:31:42 +0100 Subject: [PATCH 07/10] Parser: Respect the accessor of the source file for relative paths --- src/libexpr/parser.y | 15 +++++++++++---- src/libflake/flake/flake.cc | 8 ++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 944c7b1af..bde721401 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -359,11 +359,18 @@ string_parts_interpolated path_start : PATH { - Path path(absPath(std::string_view{$1.p, $1.l}, state->basePath.path.abs())); + std::string_view literal({$1.p, $1.l}); + Path path(absPath(literal, state->basePath.path.abs())); /* add back in the trailing '/' to the first segment */ - if ($1.p[$1.l-1] == '/' && $1.l > 1) - path += "/"; - $$ = new ExprPath(ref(state->rootFS), std::move(path)); + if (literal.size() > 1 && literal.back() == '/') + path += '/'; + $$ = + /* Absolute paths are always interpreted relative to the + root filesystem accessor, rather than the accessor of the + current Nix expression. */ + literal.front() == '/' + ? new ExprPath(state->rootFS, std::move(path)) + : new ExprPath(state->basePath.accessor, std::move(path)); } | HPATH { if (state->settings.pureEval) { diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index eee22d516..fcb04ac7f 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -186,9 +186,7 @@ static FlakeInput parseFlakeInput( url = attr.value->string_view(); else if (attr.value->type() == nPath) { auto path = attr.value->path(); - if (path.accessor != flakeDir.accessor - // FIXME: hack necessary since the parser currently stores all paths as inside rootFS. - && flakeDir.accessor == state.rootFS) + if (path.accessor != flakeDir.accessor) throw Error("input attribute path '%s' at %s must be in the same source tree as %s", path, state.positions[attr.pos], flakeDir); url = "path:" + flakeDir.path.makeRelative(path.path); @@ -337,9 +335,7 @@ static Flake readFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - // FIXME: hack necessary since the parser currently stores all paths as inside rootFS. - SourcePath path(rootDir.accessor, setting.value->path().path); - auto storePath = fetchToStore(*state.store, path, FetchMode::Copy); + auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy); flake.config.settings.emplace( state.symbols[setting.name], state.store->toRealPath(storePath)); From 4b75edacd78a8f053a5d92c4d401aaf185ab7b6c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 10 Feb 2025 17:13:17 +0100 Subject: [PATCH 08/10] Restore NAR hash assertion --- src/libflake/flake/flake.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index fcb04ac7f..f44984993 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -85,6 +85,7 @@ static std::tuple, FlakeRef, FlakeRef> fetchOrSubstituteTree static StorePath copyInputToStore( EvalState & state, fetchers::Input & input, + const fetchers::Input & originalInput, ref accessor) { auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName()); @@ -94,9 +95,7 @@ static StorePath copyInputToStore( auto narHash = state.store->queryPathInfo(storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - #if 0 - assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store)); - #endif + assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store)); return storePath; } @@ -421,7 +420,7 @@ static Flake getFlake( } // Copy the tree to the store. - auto storePath = copyInputToStore(state, lockedRef.input, accessor); + auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, accessor); // Re-parse flake.nix from the store. return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootAttrPath); @@ -784,7 +783,7 @@ LockedFlake lockFlake( state, *input.ref, useRegistries, flakeCache); // FIXME: allow input to be lazy. - auto storePath = copyInputToStore(state, lockedRef.input, accessor); + auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor); return {state.rootPath(state.store->toRealPath(storePath)), lockedRef}; } From 528b286cf7eb25644ad51d35b058b17160abd146 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 10 Feb 2025 17:14:55 +0100 Subject: [PATCH 09/10] Remove stray line --- src/libflake/flake/flake.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index f44984993..717848ee1 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -539,7 +539,6 @@ LockedFlake lockFlake( /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ for (auto & [id, input] : flakeInputs) { - //if (id == "self") continue; for (auto & [idOverride, inputOverride] : input.overrides) { auto inputAttrPath(inputAttrPathPrefix); inputAttrPath.push_back(id); From 2819d8b66a3f17d5ee26aa621cf9f2e9de6015ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 10 Feb 2025 17:19:34 +0100 Subject: [PATCH 10/10] Add release note --- doc/manual/rl-next/self-submodules-attr.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/manual/rl-next/self-submodules-attr.md diff --git a/doc/manual/rl-next/self-submodules-attr.md b/doc/manual/rl-next/self-submodules-attr.md new file mode 100644 index 000000000..0eb7e1602 --- /dev/null +++ b/doc/manual/rl-next/self-submodules-attr.md @@ -0,0 +1,12 @@ +--- +synopsis: "`inputs.self.submodules` flake attribute" +prs: [12421] +--- + +Flakes in Git repositories can now declare that they need Git submodules to be enabled: +``` +{ + inputs.self.submodules = true; +} +``` +Thus, it's no longer needed for the caller of the flake to pass `submodules = true`.