From 0c275558e75b62beb53e8ada2c116643c7ecb0ba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jul 2023 21:30:33 +0200 Subject: [PATCH 1/5] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index d76bd2ba3..cf8690732 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.17.0 +2.18.0 From 1b756e300f98afe6a583f8bed3acd687d4a8abf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 25 Jul 2023 15:51:15 +0200 Subject: [PATCH 2/5] doc: clarify release notes about nested attribute merges --- doc/manual/src/release-notes/rl-2.17.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md index 125a93cfd..0b861aecc 100644 --- a/doc/manual/src/release-notes/rl-2.17.md +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -11,8 +11,12 @@ ```nix { - nested = { foo = 1; }; - nested = { ${"ba" + "r"} = 2; }; + nested = { + foo = 1; + }; + nested = { + ${"ba" + "r"} = 2; + }; } ``` @@ -22,8 +26,17 @@ { nested = { bar = 2; foo = 1; }; } ``` - Note that the feature of merging multiple attribute set declarations is of questionable value. + Note that the feature of merging multiple *full declarations* of attribute sets like `nested` in the example is of questionable value. It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + Instead, consider using the *nested attribute path* syntax: + + ```nix + { + nested.foo = 1; + nested.${"ba" + "r"} = 2; + } + ``` + * Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. From 2d1d81114d72ace89ce08cd3bc93f4eb27a2975d Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Tue, 25 Jul 2023 12:43:33 -0500 Subject: [PATCH 3/5] Add `parseFlakeRef` and `flakeRefToString` builtins (#8670) Over the last year or so I've run into several use cases where I need to parse and/or serialize URLs for use by `builtins.fetchTree` or `builtins.getFlake`, largely in order to produce _lockfile-like_ files for lang2nix frameworks or tools which use `nix` internally to drive builds. I've gone through the painstaking process of emulating `nix::FlakeRef::fromAttrs` and `nix::parseFlakeRef` several times with mixed success; but these are difficult to create and even harder to maintain if I hope to stay aligned with changes to the real parser/serializer. I understand why adding new `builtins` isn't something we want to do flagrantly. I'm recommending this addition simply because I keep encountering use cases where I need to parse/serialize these URIs in `nix` expressions, and I want a reliable solution. Co-authored-by: Eelco Dolstra Co-authored-by: John Ericson --- doc/manual/src/release-notes/rl-next.md | 7 ++ src/libexpr/flake/flake.cc | 95 ++++++++++++++++++++ tests/lang/eval-okay-flake-ref-to-string.exp | 1 + tests/lang/eval-okay-flake-ref-to-string.nix | 7 ++ tests/lang/eval-okay-parse-flake-ref.exp | 1 + tests/lang/eval-okay-parse-flake-ref.nix | 1 + 6 files changed, 112 insertions(+) create mode 100644 tests/lang/eval-okay-flake-ref-to-string.exp create mode 100644 tests/lang/eval-okay-flake-ref-to-string.nix create mode 100644 tests/lang/eval-okay-parse-flake-ref.exp create mode 100644 tests/lang/eval-okay-parse-flake-ref.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..065364453 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,8 @@ # Release X.Y (202?-??-??) + +- Two new builtin functions, + [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) + and + [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), + have been added. + These functions are useful for converting between flake references encoded as attribute sets and URLs. diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5aa44d6a1..9112becff 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -793,6 +793,101 @@ static RegisterPrimOp r2({ .experimentalFeature = Xp::Flakes, }); +static void prim_parseFlakeRef( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, + "while evaluating the argument passed to builtins.parseFlakeRef")); + auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto binds = state.buildBindings(attrs.size()); + for (const auto & [key, value] : attrs) { + auto s = state.symbols.create(key); + auto & vv = binds.alloc(s); + std::visit(overloaded { + [&vv](const std::string & value) { vv.mkString(value); }, + [&vv](const uint64_t & value) { vv.mkInt(value); }, + [&vv](const Explicit & value) { vv.mkBool(value.t); } + }, value); + } + v.mkAttrs(binds); +} + +static RegisterPrimOp r3({ + .name = "__parseFlakeRef", + .args = {"flake-ref"}, + .doc = R"( + Parse a flake reference, and return its exploded form. + + For example: + ```nix + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + evaluates to: + ```nix + { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } + ``` + )", + .fun = prim_parseFlakeRef, + .experimentalFeature = Xp::Flakes, +}); + + +static void prim_flakeRefToString( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + state.forceAttrs(*args[0], noPos, + "while evaluating the argument passed to builtins.flakeRefToString"); + fetchers::Attrs attrs; + for (const auto & attr : *args[0]->attrs) { + auto t = attr.value->type(); + if (t == nInt) { + attrs.emplace(state.symbols[attr.name], + (uint64_t) attr.value->integer); + } else if (t == nBool) { + attrs.emplace(state.symbols[attr.name], + Explicit { attr.value->boolean }); + } else if (t == nString) { + attrs.emplace(state.symbols[attr.name], + std::string(attr.value->str())); + } else { + state.error( + "flake reference attribute sets may only contain integers, Booleans, " + "and strings, but attribute '%s' is %s", + state.symbols[attr.name], + showType(*attr.value)).debugThrow(); + } + } + auto flakeRef = FlakeRef::fromAttrs(attrs); + v.mkString(flakeRef.to_string()); +} + +static RegisterPrimOp r4({ + .name = "__flakeRefToString", + .args = {"attrs"}, + .doc = R"( + Convert a flake reference from attribute set format to URL format. + + For example: + ```nix + builtins.flakeRefToString { + dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; + } + ``` + evaluates to + ```nix + "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + )", + .fun = prim_flakeRefToString, + .experimentalFeature = Xp::Flakes, +}); + } Fingerprint LockedFlake::getFingerprint() const diff --git a/tests/lang/eval-okay-flake-ref-to-string.exp b/tests/lang/eval-okay-flake-ref-to-string.exp new file mode 100644 index 000000000..110f8442d --- /dev/null +++ b/tests/lang/eval-okay-flake-ref-to-string.exp @@ -0,0 +1 @@ +"github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/lang/eval-okay-flake-ref-to-string.nix b/tests/lang/eval-okay-flake-ref-to-string.nix new file mode 100644 index 000000000..dbb4e5b2a --- /dev/null +++ b/tests/lang/eval-okay-flake-ref-to-string.nix @@ -0,0 +1,7 @@ +builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + ref = "23.05"; + dir = "lib"; +} diff --git a/tests/lang/eval-okay-parse-flake-ref.exp b/tests/lang/eval-okay-parse-flake-ref.exp new file mode 100644 index 000000000..fc17ba085 --- /dev/null +++ b/tests/lang/eval-okay-parse-flake-ref.exp @@ -0,0 +1 @@ +{ dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } diff --git a/tests/lang/eval-okay-parse-flake-ref.nix b/tests/lang/eval-okay-parse-flake-ref.nix new file mode 100644 index 000000000..db4ed2742 --- /dev/null +++ b/tests/lang/eval-okay-parse-flake-ref.nix @@ -0,0 +1 @@ + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" From 33d58a90c26f9a62bb20863b767c0a505e6f997a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Jun 2023 01:29:11 +0200 Subject: [PATCH 4/5] toJSON: Add attribute path to trace --- doc/manual/src/release-notes/rl-next.md | 2 + src/libexpr/value-to-json.cc | 22 ++++++++-- tests/lang/eval-fail-toJSON.err.exp | 57 +++++++++++++++++++++++++ tests/lang/eval-fail-toJSON.nix | 10 +++++ 4 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 tests/lang/eval-fail-toJSON.err.exp create mode 100644 tests/lang/eval-fail-toJSON.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 065364453..df30ce83d 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -6,3 +6,5 @@ [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), have been added. These functions are useful for converting between flake references encoded as attribute sets and URLs. + +- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4996a5bde..ac3986c87 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -43,6 +43,7 @@ json printValueAsJSON(EvalState & state, bool strict, break; case nNull: + // already initialized as null break; case nAttrs: { @@ -59,7 +60,13 @@ json printValueAsJSON(EvalState & state, bool strict, names.emplace(state.symbols[j.name]); for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); - out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + try { + out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + } catch (Error & e) { + e.addTrace(state.positions[a.pos], + hintfmt("while evaluating attribute '%1%'", j)); + throw; + } } } else return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); @@ -68,8 +75,17 @@ json printValueAsJSON(EvalState & state, bool strict, case nList: { out = json::array(); - for (auto elem : v.listItems()) - out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + int i = 0; + for (auto elem : v.listItems()) { + try { + out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + } catch (Error & e) { + e.addTrace({}, + hintfmt("while evaluating list element at index %1%", i)); + throw; + } + i++; + } break; } diff --git a/tests/lang/eval-fail-toJSON.err.exp b/tests/lang/eval-fail-toJSON.err.exp new file mode 100644 index 000000000..4e618c203 --- /dev/null +++ b/tests/lang/eval-fail-toJSON.err.exp @@ -0,0 +1,57 @@ +error: + … while calling the 'toJSON' builtin + + at /pwd/lang/eval-fail-toJSON.nix:1:1: + + 1| builtins.toJSON { + | ^ + 2| a.b = [ + + … while evaluating attribute 'a' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating attribute 'b' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating list element at index 3 + + … while evaluating attribute 'c' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while evaluating attribute 'd' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while calling the 'throw' builtin + + at /pwd/lang/eval-fail-toJSON.nix:7:13: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + error: hah no diff --git a/tests/lang/eval-fail-toJSON.nix b/tests/lang/eval-fail-toJSON.nix new file mode 100644 index 000000000..8112e1c1f --- /dev/null +++ b/tests/lang/eval-fail-toJSON.nix @@ -0,0 +1,10 @@ +builtins.toJSON { + a.b = [ + true + false + "it's a bird" + { + c.d = throw "hah no"; + } + ]; +} From c9a87ce7ca50b0a14c4c77fd2b6ca41a74ba6cb9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Fri, 28 Jul 2023 10:11:45 +0100 Subject: [PATCH 5/5] Refactor verifyPath to take StorePath instead of Path. This way we avoid having to convert from Path to StorePath and vice versa in the body of verifyPath. --- src/libstore/local-store.cc | 21 ++++++++------------- src/libstore/local-store.hh | 4 ++-- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f8e0a0ca2..f78bd44ca 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1506,10 +1506,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking path existence..."); StorePathSet validPaths; - PathSet done; + StorePathSet done; for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, store, done, validPaths, repair, errors); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1595,19 +1595,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) +void LocalStore::verifyPath(const StorePath & path, const StringSet & store, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(pathS).second) return; - - if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); - return; - } - - auto path = parseStorePath(pathS); + if (!done.insert(path).second) return; if (!store.count(std::string(path.to_string()))) { /* Check any referrers first. If we can invalidate them @@ -1616,11 +1609,13 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, store, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } + auto pathS = printStorePath(path); + if (canInvalidate) { printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 4125cacf4..c9b570eaa 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -314,8 +314,8 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const StorePath & path, const StringSet & store, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path);