wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-24 18:01:16 +02:00
Code Activity

Merge pull request #12421 from DeterminateSystems/self-input-attrs

Browse source 
Add `inputs.self.submodules` flake attribute
This commit is contained in:
Eelco Dolstra 2025-02-10 17:35:46 +01:00 • committed by GitHub
commit 92bf150b1c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 222 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

12
doc/manual/rl-next/self-submodules-attr.md Normal file
View file

@ -0,0 +1,12 @@
---
synopsis: "`inputs.self.submodules` flake attribute"
prs: [12421]
---
Flakes in Git repositories can now declare that they need Git submodules to be enabled:
```
{
inputs.self.submodules = true;
}
```
Thus, it's no longer needed for the caller of the flake to pass `submodules = true`.

8
src/libcmd/common-eval-args.cc
View file

@ -34,7 +34,9 @@ EvalSettings evalSettings {
// FIXME `parseFlakeRef` should take a `std::string_view`.
auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
debug("fetching flake search path element '%s''", rest);
auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
state.allowPath(storePath);
return state.rootPath(state.store->toRealPath(storePath));
},
},
@ -183,7 +185,9 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
else if (hasPrefix(s, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
state.allowPath(storePath);
return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
}

7
src/libfetchers/fetchers.cc
View file

@ -186,6 +186,7 @@ bool Input::contains(const Input & other) const
return false;
}
// FIXME: remove
std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
{
if (!scheme)
@ -200,10 +201,6 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
auto narHash = store->queryPathInfo(storePath)->narHash;
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
// FIXME: we would like to mark inputs as final in
// getAccessorUnchecked(), but then we can't add
// narHash. Or maybe narHash should be excluded from the
// concept of "final" inputs?
result.attrs.insert_or_assign("__final", Explicit<bool>(true));
assert(result.isFinal());
@ -284,6 +281,8 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
try {
auto [accessor, result] = getAccessorUnchecked(store);
result.attrs.insert_or_assign("__final", Explicit<bool>(true));
checkLocks(*this, result);
return {accessor, std::move(result)};

199
src/libflake/flake/flake.cc
View file

@ -12,6 +12,7 @@
#include "flake/settings.hh"
#include "value-to-json.hh"
#include "local-fs-store.hh"
#include "fetch-to-store.hh"
#include <nlohmann/json.hpp>
@ -24,7 +25,7 @@ namespace flake {
struct FetchedFlake
{
FlakeRef lockedRef;
StorePath storePath;
ref<SourceAccessor> accessor;
};
typedef std::map<FlakeRef, FetchedFlake> FlakeCache;
@ -40,7 +41,7 @@ static std::optional<FetchedFlake> lookupInFlakeCache(
return i->second;
}
static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
static std::tuple<ref<SourceAccessor>, FlakeRef, FlakeRef> fetchOrSubstituteTree(
EvalState & state,
const FlakeRef & originalRef,
bool useRegistries,
@ -51,8 +52,8 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
if (!fetched) {
if (originalRef.input.isDirect()) {
auto [storePath, lockedRef] = originalRef.fetchTree(state.store);
fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath});
auto [accessor, lockedRef] = originalRef.lazyFetch(state.store);
fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
} else {
if (useRegistries) {
resolvedRef = originalRef.resolve(
@ -64,8 +65,8 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
});
fetched = lookupInFlakeCache(flakeCache, originalRef);
if (!fetched) {
auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store);
fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath});
auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store);
fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
}
flakeCache.insert_or_assign(resolvedRef, *fetched);
}
@ -76,14 +77,27 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
flakeCache.insert_or_assign(originalRef, *fetched);
}
debug("got tree '%s' from '%s'",
state.store->printStorePath(fetched->storePath), fetched->lockedRef);
debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedRef);
state.allowPath(fetched->storePath);
return {fetched->accessor, resolvedRef, fetched->lockedRef};
}
assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store));
static StorePath copyInputToStore(
EvalState & state,
fetchers::Input & input,
const fetchers::Input & originalInput,
ref<SourceAccessor> accessor)
{
auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName());
return {fetched->storePath, resolvedRef, fetched->lockedRef};
state.allowPath(storePath);
auto narHash = state.store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
return storePath;
}
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
@ -101,12 +115,47 @@ static void expectType(EvalState & state, ValueType type,
showType(type), showType(value.type()), state.positions[pos]);
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInputs(
EvalState & state,
Value * value,
const PosIdx pos,
const InputAttrPath & lockRootAttrPath,
const SourcePath & flakeDir);
const SourcePath & flakeDir,
bool allowSelf);
static void parseFlakeInputAttr(
EvalState & state,
const Attr & attr,
fetchers::Attrs & attrs)
{
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (attr.value->type()) {
case nString:
attrs.emplace(state.symbols[attr.name], attr.value->c_str());
break;
case nBool:
attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean() });
break;
case nInt: {
auto intValue = attr.value->integer().value;
if (intValue < 0)
state.error<EvalError>("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow();
attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
break;
}
default:
if (attr.name == state.symbols.create("publicKeys")) {
experimentalFeatureSettings.require(Xp::VerifiedFetches);
NixStringContext emptyContext = {};
attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, attr.pos, emptyContext).dump());
} else
state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
state.symbols[attr.name], showType(*attr.value)).debugThrow();
}
#pragma GCC diagnostic pop
}
static FlakeInput parseFlakeInput(
EvalState & state,
@ -149,44 +198,14 @@ static FlakeInput parseFlakeInput(
expectType(state, nBool, *attr.value, attr.pos);
input.isFlake = attr.value->boolean();
} else if (attr.name == sInputs) {
input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir);
input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first;
} else if (attr.name == sFollows) {
expectType(state, nString, *attr.value, attr.pos);
auto follows(parseInputAttrPath(attr.value->c_str()));
follows.insert(follows.begin(), lockRootAttrPath.begin(), lockRootAttrPath.end());
input.follows = follows;
} else {
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (attr.value->type()) {
case nString:
attrs.emplace(state.symbols[attr.name], attr.value->c_str());
break;
case nBool:
attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean() });
break;
case nInt: {
auto intValue = attr.value->integer().value;
if (intValue < 0) {
state.error<EvalError>("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow();
}
attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
break;
}
default:
if (attr.name == state.symbols.create("publicKeys")) {
experimentalFeatureSettings.require(Xp::VerifiedFetches);
NixStringContext emptyContext = {};
attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump());
} else
state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
state.symbols[attr.name], showType(*attr.value)).debugThrow();
}
#pragma GCC diagnostic pop
}
} else
parseFlakeInputAttr(state, attr, attrs);
} catch (Error & e) {
e.addTrace(
state.positions[attr.pos],
@ -216,28 +235,39 @@ static FlakeInput parseFlakeInput(
return input;
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInputs(
EvalState & state,
Value * value,
const PosIdx pos,
const InputAttrPath & lockRootAttrPath,
const SourcePath & flakeDir)
const SourcePath & flakeDir,
bool allowSelf)
{
std::map<FlakeId, FlakeInput> inputs;
fetchers::Attrs selfAttrs;
expectType(state, nAttrs, *value, pos);
for (auto & inputAttr : *value->attrs()) {
inputs.emplace(state.symbols[inputAttr.name],
parseFlakeInput(state,
state.symbols[inputAttr.name],
inputAttr.value,
inputAttr.pos,
lockRootAttrPath,
flakeDir));
auto inputName = state.symbols[inputAttr.name];
if (inputName == "self") {
if (!allowSelf)
throw Error("'self' input attribute not allowed at %s", state.positions[inputAttr.pos]);
expectType(state, nAttrs, *inputAttr.value, inputAttr.pos);
for (auto & attr : *inputAttr.value->attrs())
parseFlakeInputAttr(state, attr, selfAttrs);
} else {
inputs.emplace(inputName,
parseFlakeInput(state,
inputName,
inputAttr.value,
inputAttr.pos,
lockRootAttrPath,
flakeDir));
}
}
return inputs;
return {inputs, selfAttrs};
}
static Flake readFlake(
@ -269,8 +299,11 @@ static Flake readFlake(
auto sInputs = state.symbols.create("inputs");
if (auto inputs = vInfo.attrs()->get(sInputs))
flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootAttrPath, flakeDir);
if (auto inputs = vInfo.attrs()->get(sInputs)) {
auto [flakeInputs, selfAttrs] = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootAttrPath, flakeDir, true);
flake.inputs = std::move(flakeInputs);
flake.selfAttrs = std::move(selfAttrs);
}
auto sOutputs = state.symbols.create("outputs");
@ -301,10 +334,10 @@ static Flake readFlake(
state.symbols[setting.name],
std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
else if (setting.value->type() == nPath) {
NixStringContext emptyContext = {};
auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy);
flake.config.settings.emplace(
state.symbols[setting.name],
state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true).toOwned());
state.store->toRealPath(storePath));
}
else if (setting.value->type() == nInt)
flake.config.settings.emplace(
@ -342,6 +375,23 @@ static Flake readFlake(
return flake;
}
static FlakeRef applySelfAttrs(
const FlakeRef & ref,
const Flake & flake)
{
auto newRef(ref);
std::set<std::string> allowedAttrs{"submodules"};
for (auto & attr : flake.selfAttrs) {
if (!allowedAttrs.contains(attr.first))
throw Error("flake 'self' attribute '%s' is not supported", attr.first);
newRef.input.attrs.insert_or_assign(attr.first, attr.second);
}
return newRef;
}
static Flake getFlake(
EvalState & state,
const FlakeRef & originalRef,
@ -349,9 +399,30 @@ static Flake getFlake(
FlakeCache & flakeCache,
const InputAttrPath & lockRootAttrPath)
{
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
// Fetch a lazy tree first.
auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, originalRef, useRegistries, flakeCache);
// Parse/eval flake.nix to get at the input.self attributes.
auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {accessor}, lockRootAttrPath);
// Re-fetch the tree if necessary.
auto newLockedRef = applySelfAttrs(lockedRef, flake);
if (lockedRef != newLockedRef) {
debug("refetching input '%s' due to self attribute", newLockedRef);
// FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
newLockedRef.input.attrs.erase("narHash");
auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree(
state, newLockedRef, false, flakeCache);
accessor = accessor2;
lockedRef = lockedRef2;
}
// Copy the tree to the store.
auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, accessor);
// Re-parse flake.nix from the store.
return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootAttrPath);
}
@ -707,8 +778,12 @@ LockedFlake lockFlake(
if (auto resolvedPath = resolveRelativePath()) {
return {*resolvedPath, *input.ref};
} else {
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, useRegistries, flakeCache);
// FIXME: allow input to be lazy.
auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor);
return {state.rootPath(state.store->toRealPath(storePath)), lockedRef};
}
}();

15
src/libflake/flake/flake.hh
View file

@ -79,24 +79,37 @@ struct Flake
* The original flake specification (by the user)
*/
FlakeRef originalRef;
/**
* registry references and caching resolved to the specific underlying flake
*/
FlakeRef resolvedRef;
/**
* the specific local store result of invoking the fetcher
*/
FlakeRef lockedRef;
/**
* The path of `flake.nix`.
*/
SourcePath path;
/**
* pretend that 'lockedRef' is dirty
* Pretend that `lockedRef` is dirty.
*/
bool forceDirty = false;
std::optional<std::string> description;
FlakeInputs inputs;
/**
* Attributes to be retroactively applied to the `self` input
* (such as `submodules = true`).
*/
fetchers::Attrs selfAttrs;
/**
* 'nixConfig' attribute
*/

6
src/libflake/flake/flakeref.cc
View file

@ -283,10 +283,10 @@ FlakeRef FlakeRef::fromAttrs(
fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
}
std::pair<StorePath, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
std::pair<ref<SourceAccessor>, FlakeRef> FlakeRef::lazyFetch(ref<Store> store) const
{
auto [storePath, lockedInput] = input.fetchToStore(store);
return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)};
auto [accessor, lockedInput] = input.getAccessor(store);
return {accessor, FlakeRef(std::move(lockedInput), subdir)};
}
std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(

2
src/libflake/flake/flakeref.hh
View file

@ -71,7 +71,7 @@ struct FlakeRef
const fetchers::Settings & fetchSettings,
const fetchers::Attrs & attrs);
std::pair<StorePath, FlakeRef> fetchTree(ref<Store> store) const;
std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(ref<Store> store) const;
};
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);

4
src/nix/flake.cc
View file

@ -17,6 +17,7 @@
#include "eval-cache.hh"
#include "markdown.hh"
#include "users.hh"
#include "fetch-to-store.hh"
#include <filesystem>
#include <nlohmann/json.hpp>
@ -1449,7 +1450,8 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
{
auto originalRef = getFlakeRef();
auto resolvedRef = originalRef.resolve(store);
auto [storePath, lockedRef] = resolvedRef.fetchTree(store);
auto [accessor, lockedRef] = resolvedRef.lazyFetch(store);
auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName());
auto hash = store->queryPathInfo(storePath)->narHash;
if (json) {

44
tests/functional/flakes/flake-in-submodule.sh
View file

@ -27,6 +27,7 @@ git config --global protocol.file.allow always
rootRepo=$TEST_ROOT/rootRepo
subRepo=$TEST_ROOT/submodule
otherRepo=$TEST_ROOT/otherRepo
createGitRepo "$subRepo"
@ -74,9 +75,50 @@ EOF
git -C "$rootRepo" add flake.nix
git -C "$rootRepo" commit -m "Add flake.nix"
storePath=$(nix flake metadata --json "$rootRepo?submodules=1" | jq -r .path)
storePath=$(nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath)
[[ -e "$storePath/submodule" ]]
# Test the use of inputs.self.
cat > "$rootRepo"/flake.nix <<EOF
{
inputs.self.submodules = true;
outputs = { self }: {
foo = self.outPath;
};
}
EOF
git -C "$rootRepo" commit -a -m "Bla"
storePath=$(nix eval --raw "$rootRepo#foo")
[[ -e "$storePath/submodule" ]]
# Test another repo referring to a repo that uses inputs.self.
createGitRepo "$otherRepo"
cat > "$otherRepo"/flake.nix <<EOF
{
inputs.root.url = "git+file://$rootRepo";
outputs = { self, root }: {
foo = root.foo;
};
}
EOF
git -C "$otherRepo" add flake.nix
# The first call should refetch the root repo...
expectStderr 0 nix eval --raw "$otherRepo#foo" -vvvvv | grepQuiet "refetching"
[[ $(jq .nodes.root_2.locked.submodules "$otherRepo/flake.lock") == true ]]
# ... but the second call should have 'submodules = true' in flake.lock, so it should not refetch.
rm -rf "$TEST_HOME/.cache"
clearStore
expectStderr 0 nix eval --raw "$otherRepo#foo" -vvvvv | grepQuietInverse "refetching"
storePath=$(nix eval --raw "$otherRepo#foo")
[[ -e "$storePath/submodule" ]]
# The root repo may use the submodule repo as an input
# through the relative path. This may change in the future;
# see: https://discourse.nixos.org/t/57783 and #9708.