From 6406619c441c35ba323212a234e8923f2a2087da Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Jul 2024 16:49:49 +0200 Subject: [PATCH] Flake schemas This applies upstream https://github.com/NixOS/nix/pull/8892. --- Makefile.config.in | 1 + configure.ac | 6 + doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/protocols/flake-schemas.md | 60 ++ flake.lock | 16 + flake.nix | 6 +- package.nix | 3 + packaging/dependencies.nix | 2 + packaging/hydra.nix | 2 + src/libcmd/installable-flake.cc | 14 - src/libcmd/installable-flake.hh | 2 - src/libcmd/installables.cc | 5 - src/libexpr/eval-cache.cc | 6 + src/libexpr/eval-cache.hh | 7 + src/libflake/flake/flake.cc | 34 +- src/libflake/flake/flake.hh | 18 + src/nix/call-flake-schemas.nix | 43 + src/nix/flake-check.md | 58 +- src/nix/flake-schemas.cc | 221 ++++++ src/nix/flake-schemas.hh | 45 ++ src/nix/flake.cc | 907 +++++----------------- src/nix/local.mk | 6 + tests/functional/flakes/check.sh | 11 - tests/functional/flakes/show.sh | 43 +- tests/functional/fmt.sh | 4 +- 25 files changed, 702 insertions(+), 819 deletions(-) create mode 100644 doc/manual/src/protocols/flake-schemas.md create mode 100644 src/nix/call-flake-schemas.nix create mode 100644 src/nix/flake-schemas.cc create mode 100644 src/nix/flake-schemas.hh diff --git a/Makefile.config.in b/Makefile.config.in index 3100d2073..2ed716b5e 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -37,6 +37,7 @@ checkbindir = @checkbindir@ checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ +default_flake_schemas = @default_flake_schemas@ docdir = @docdir@ embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ diff --git a/configure.ac b/configure.ac index 4f66a3efc..caeb88b67 100644 --- a/configure.ac +++ b/configure.ac @@ -435,6 +435,12 @@ if test "$embedded_sandbox_shell" = yes; then AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.]) fi + +AC_ARG_WITH(default-flake-schemas, AS_HELP_STRING([--with-default-flake-schemas=PATH],[path of the default flake schemas flake]), + default_flake_schemas=$withval, + [AC_MSG_FAILURE([--with-default-flake-schemas is missing])]) +AC_SUBST(default_flake_schemas) + ]) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a6a2101e9..56e0dbeec 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -113,6 +113,7 @@ - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) + - [Flake Schemas](protocols/flake-schemas.md) - [C API](c-api.md) - [Glossary](glossary.md) - [Contributing](contributing/index.md) diff --git a/doc/manual/src/protocols/flake-schemas.md b/doc/manual/src/protocols/flake-schemas.md new file mode 100644 index 000000000..f6cdd6165 --- /dev/null +++ b/doc/manual/src/protocols/flake-schemas.md @@ -0,0 +1,60 @@ +# Flake Schemas + +Flake schemas are a mechanism to allow tools like `nix flake show` and `nix flake check` to enumerate and check the contents of a flake +in a generic way, without requiring built-in knowledge of specific flake output types like `packages` or `nixosConfigurations`. + +A flake can define schemas for its outputs by defining a `schemas` output. `schemas` should be an attribute set with an attribute for +every output type that you want to be supported. If a flake does not have a `schemas` attribute, Nix uses a built-in set of schemas (namely https://github.com/DeterminateSystems/flake-schemas). + +A schema is an attribute set with the following attributes: + +* `version`: Should be set to 1. +* `doc`: A string containing documentation about the flake output type in Markdown format. +* `allowIFD` (defaults to `true`): Whether the evaluation of the output attributes of this flake can read from derivation outputs. +* `inventory`: A function that returns the contents of the flake output (described below). + +# Inventory + +The `inventory` function returns a *node* describing the contents of the flake output. A node is either a *leaf node* or a *non-leaf node*. This allows nested flake output attributes to be described (e.g. `x86_64-linux.hello` inside a `packages` output). + +Non-leaf nodes must have the following attribute: + +* `children`: An attribute set of nodes. If this attribute is missing, the attribute if a leaf node. + +Leaf nodes can have the following attributes: + +* `derivation`: The main derivation of this node, if any. It must evaluate for `nix flake check` and `nix flake show` to succeed. + +* `evalChecks`: An attribute set of Boolean values, used by `nix flake check`. Each attribute must evaluate to `true`. + +* `isFlakeCheck`: Whether `nix flake check` should build the `derivation` attribute of this node. + +* `shortDescription`: A one-sentence description of the node (such as the `meta.description` attribute in Nixpkgs). + +* `what`: A brief human-readable string describing the type of the node, e.g. `"package"` or `"development environment"`. This is used by tools like `nix flake show` to describe the contents of a flake. + +Both leaf and non-leaf nodes can have the following attributes: + +* `forSystems`: A list of Nix system types (e.g. `["x86_64-linux"]`) supported by this node. This is used by tools to skip nodes that cannot be built on the user's system. Setting this on a non-leaf node allows all the children to be skipped, regardless of the `forSystems` attributes of the children. If this attribute is not set, the node is never skipped. + +# Example + +Here is a schema that checks that every element of the `nixosConfigurations` flake output evaluates and builds correctly (meaning that it has a `config.system.build.toplevel` attribute that yields a buildable derivation). + +```nix +outputs = { + schemas.nixosConfigurations = { + version = 1; + doc = '' + The `nixosConfigurations` flake output defines NixOS system configurations. + ''; + inventory = output: { + children = builtins.mapAttrs (configName: machine: + { + what = "NixOS configuration"; + derivation = machine.config.system.build.toplevel; + }) output; + }; + }; +}; +``` diff --git a/flake.lock b/flake.lock index f64e3ea37..8ea495401 100644 --- a/flake.lock +++ b/flake.lock @@ -36,6 +36,21 @@ "type": "github" } }, + "flake-schemas": { + "locked": { + "lastModified": 1719857163, + "narHash": "sha256-wM+8JtoKBkahHiKn+EM1ikurMnitwRQrZ91hipJIJK8=", + "owner": "DeterminateSystems", + "repo": "flake-schemas", + "rev": "61a02d7183d4241962025e6c6307a22a0bb72a21", + "type": "github" + }, + "original": { + "owner": "DeterminateSystems", + "repo": "flake-schemas", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1667395993, @@ -145,6 +160,7 @@ "inputs": { "flake-compat": "flake-compat", "flake-parts": "flake-parts", + "flake-schemas": "flake-schemas", "libgit2": "libgit2", "nixpkgs": "nixpkgs", "nixpkgs-23-11": "nixpkgs-23-11", diff --git a/flake.nix b/flake.nix index d83c2ecad..256ff66cd 100644 --- a/flake.nix +++ b/flake.nix @@ -8,6 +8,7 @@ inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.flake-schemas.url = "github:DeterminateSystems/flake-schemas"; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; @@ -20,8 +21,7 @@ inputs.pre-commit-hooks.inputs.flake-compat.follows = ""; inputs.pre-commit-hooks.inputs.gitignore.follows = ""; - outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, flake-schemas, ... }: let inherit (nixpkgs) lib; @@ -157,6 +157,8 @@ }; in { + schemas = flake-schemas.schemas; + # A Nixpkgs overlay that overrides the 'nix' and # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); diff --git a/package.nix b/package.nix index c3e565399..99ffd5e40 100644 --- a/package.nix +++ b/package.nix @@ -38,6 +38,8 @@ , busybox-sandbox-shell ? null +, flake-schemas + # Configuration Options #: # This probably seems like too many degrees of freedom, but it @@ -260,6 +262,7 @@ in { (lib.enableFeature enableMarkdown "markdown") (lib.enableFeature installUnitTests "install-unit-tests") (lib.withFeatureAs true "readline-flavor" readlineFlavor) + "--with-default-flake-schemas=${flake-schemas}" ] ++ lib.optionals (!forDevShell) [ "--sysconfdir=/etc" ] ++ lib.optionals installUnitTests [ diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 34b344971..4f7a6daab 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -114,4 +114,6 @@ scope: { inherit resolvePath filesetToSource; mkMesonDerivation = f: stdenv.mkDerivation (lib.extends localSourceLayer f); + + inherit (inputs) flake-schemas; } diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 4dfaf9bbf..d56340231 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -28,6 +28,8 @@ let test-daemon = daemon; doBuild = false; + + inherit (inputs) flake-schemas; }; # Technically we could just return `pkgs.nixComponents`, but for Hydra it's diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index d42fa7aac..899919550 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..30240a35a 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -52,8 +52,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eb7048d39..d10df2e54 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -444,11 +444,6 @@ ref openEvalCache( : std::nullopt; auto rootLoader = [&state, lockedFlake]() { - /* For testing whether the evaluation cache is - complete. */ - if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") - throw Error("not everything is cached, but evaluation is not allowed"); - auto vFlake = state.allocValue(); flake::callFlake(state, *lockedFlake, *vFlake); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2630c34d5..d43577cfd 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -368,6 +368,12 @@ Value * EvalCache::getRootValue() { if (!value) { debug("getting root value"); + + /* For testing whether the evaluation cache is + complete. */ + if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") + throw Error("not everything is cached, but evaluation is not allowed"); + value = allocRootValue(rootLoader()); } return *value; diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index b1911e3a4..a6c8ad011 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -34,7 +34,11 @@ class EvalCache : public std::enable_shared_from_this friend struct CachedEvalError; std::shared_ptr db; + +public: EvalState & state; + +private: typedef std::function RootLoader; RootLoader rootLoader; RootValue value; @@ -89,7 +93,10 @@ class AttrCursor : public std::enable_shared_from_this friend class EvalCache; friend struct CachedEvalError; +public: ref root; + +private: typedef std::optional, Symbol>> Parent; Parent parent; RootValue _value; diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 21acb93ee..c69c4d66e 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -204,7 +204,7 @@ static std::map parseFlakeInputs( return inputs; } -static Flake readFlake( +Flake readFlake( EvalState & state, const FlakeRef & originalRef, const FlakeRef & resolvedRef, @@ -336,19 +336,15 @@ static LockFile readLockFile(const SourcePath & lockFilePath) : LockFile(); } -/* Compute an in-memory lock file for the specified top-level flake, - and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( EvalState & state, const FlakeRef & topRef, - const LockFlags & lockFlags) + const LockFlags & lockFlags, + Flake flake, + FlakeCache & flakeCache) { - FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(flakeSettings.useRegistries); - auto flake = getFlake(state, topRef, useRegistries, flakeCache); - if (lockFlags.applyNixConfig) { flake.config.apply(); state.store->setOptions(); @@ -738,6 +734,28 @@ LockedFlake lockFlake( } } +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & topRef, + const LockFlags & lockFlags) +{ + FlakeCache flakeCache; + + auto useRegistries = lockFlags.useRegistries.value_or(flakeSettings.useRegistries); + + return lockFlake(state, topRef, lockFlags, getFlake(state, topRef, useRegistries, flakeCache), flakeCache); +} + +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & topRef, + const LockFlags & lockFlags, + Flake flake) +{ + FlakeCache flakeCache; + return lockFlake(state, topRef, lockFlags, std::move(flake), flakeCache); +} + void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index 1ba085f0f..2ac12b590 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -193,11 +193,29 @@ struct LockFlags std::set inputUpdates; }; +Flake readFlake( + EvalState & state, + const FlakeRef & originalRef, + const FlakeRef & resolvedRef, + const FlakeRef & lockedRef, + const SourcePath & rootDir, + const InputPath & lockRootPath); + +/** + * Compute an in-memory lock file for the specified top-level flake, + * and optionally write it to file, if the flake is writable. + */ LockedFlake lockFlake( EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & topRef, + const LockFlags & lockFlags, + Flake flake); + void callFlake( EvalState & state, const LockedFlake & lockedFlake, diff --git a/src/nix/call-flake-schemas.nix b/src/nix/call-flake-schemas.nix new file mode 100644 index 000000000..cd6d4c3ae --- /dev/null +++ b/src/nix/call-flake-schemas.nix @@ -0,0 +1,43 @@ +/* The flake providing default schemas. */ +defaultSchemasFlake: + +/* The flake whose contents we want to extract. */ +flake: + +let + + # Helper functions. + + mapAttrsToList = f: attrs: map (name: f name attrs.${name}) (builtins.attrNames attrs); + +in + +rec { + outputNames = builtins.attrNames flake.outputs; + + allSchemas = (flake.outputs.schemas or defaultSchemasFlake.schemas) // schemaOverrides; + + schemaOverrides = {}; # FIXME + + schemas = + builtins.listToAttrs (builtins.concatLists (mapAttrsToList + (outputName: output: + if allSchemas ? ${outputName} then + [{ name = outputName; value = allSchemas.${outputName}; }] + else + [ ]) + flake.outputs)); + + inventory = + builtins.mapAttrs + (outputName: output: + if schemas ? ${outputName} && schemas.${outputName}.version == 1 + then + { output = schemas.${outputName}.inventory output; + inherit (schemas.${outputName}) doc; + } + else + { unknown = true; } + ) + flake.outputs; +} diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index c8307f8d8..71dd91640 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -18,56 +18,20 @@ R""( # Description This command verifies that the flake specified by flake reference -*flake-url* can be evaluated successfully (as detailed below), and -that the derivations specified by the flake's `checks` output can be -built successfully. +*flake-url* can be evaluated and built successfully according to its +`schemas` flake output. For every flake output that has a schema +definition, `nix flake check` uses the schema to extract the contents +of the output. Then, for every item in the contents: + +* It evaluates the elements of the `evalChecks` attribute set returned + by the schema for that item, printing an error or warning for every + check that fails to evaluate or that evaluates to `false`. + +* It builds `derivation` attribute returned by the schema for that + item, if the item has the `isFlakeCheck` attribute. If the `keep-going` option is set to `true`, Nix will keep evaluating as much as it can and report the errors as it encounters them. Otherwise it will stop at the first error. -# Evaluation checks - -The following flake output attributes must be derivations: - -* `checks.`*system*`.`*name* -* `defaultPackage.`*system* -* `devShell.`*system* -* `devShells.`*system*`.`*name* -* `nixosConfigurations.`*name*`.config.system.build.toplevel` -* `packages.`*system*`.`*name* - -The following flake output attributes must be [app -definitions](./nix3-run.md): - -* `apps.`*system*`.`*name* -* `defaultApp.`*system* - -The following flake output attributes must be [template -definitions](./nix3-flake-init.md): - -* `defaultTemplate` -* `templates.`*name* - -The following flake output attributes must be *Nixpkgs overlays*: - -* `overlay` -* `overlays.`*name* - -The following flake output attributes must be *NixOS modules*: - -* `nixosModule` -* `nixosModules.`*name* - -The following flake output attributes must be -[bundlers](./nix3-bundle.md): - -* `bundlers.`*name* -* `defaultBundler` - -In addition, the `hydraJobs` output is evaluated in the same way as -Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested -attribute set of derivations). Similarly, the -`legacyPackages`.*system* output is evaluated like `nix-env --query --available `. - )"" diff --git a/src/nix/flake-schemas.cc b/src/nix/flake-schemas.cc new file mode 100644 index 000000000..b93aaa4a4 --- /dev/null +++ b/src/nix/flake-schemas.cc @@ -0,0 +1,221 @@ +#include "flake-schemas.hh" +#include "eval-settings.hh" +#include "fetch-to-store.hh" +#include "memory-source-accessor.hh" + +namespace nix::flake_schemas { + +using namespace eval_cache; +using namespace flake; + +static LockedFlake getBuiltinDefaultSchemasFlake(EvalState & state) +{ + auto accessor = make_ref(); + + accessor->setPathDisplay("«builtin-flake-schemas»"); + + accessor->addFile( + CanonPath("flake.nix"), +#include "builtin-flake-schemas.nix.gen.hh" + ); + + // FIXME: remove this when we have lazy trees. + auto storePath = fetchToStore(*state.store, {accessor}, FetchMode::Copy); + state.allowPath(storePath); + + // Construct a dummy flakeref. + auto flakeRef = parseFlakeRef( + fmt("tarball+https://builtin-flake-schemas?narHash=%s", + state.store->queryPathInfo(storePath)->narHash.to_string(HashFormat::SRI, true))); + + auto flake = readFlake(state, flakeRef, flakeRef, flakeRef, state.rootPath(state.store->toRealPath(storePath)), {}); + + return lockFlake(state, flakeRef, {}, flake); +} + +std::tuple, ref> +call(EvalState & state, std::shared_ptr lockedFlake, std::optional defaultSchemasFlake) +{ + auto fingerprint = lockedFlake->getFingerprint(state.store); + + std::string callFlakeSchemasNix = +#include "call-flake-schemas.nix.gen.hh" + ; + + auto lockedDefaultSchemasFlake = + defaultSchemasFlake ? flake::lockFlake(state, *defaultSchemasFlake, {}) : getBuiltinDefaultSchemasFlake(state); + auto lockedDefaultSchemasFlakeFingerprint = lockedDefaultSchemasFlake.getFingerprint(state.store); + + std::optional fingerprint2; + if (fingerprint && lockedDefaultSchemasFlakeFingerprint) + fingerprint2 = hashString( + HashAlgorithm::SHA256, + fmt("app:%s:%s:%s", + hashString(HashAlgorithm::SHA256, callFlakeSchemasNix).to_string(HashFormat::Base16, false), + fingerprint->to_string(HashFormat::Base16, false), + lockedDefaultSchemasFlakeFingerprint->to_string(HashFormat::Base16, false))); + + // FIXME: merge with openEvalCache(). + auto cache = make_ref( + evalSettings.useEvalCache && evalSettings.pureEval ? fingerprint2 : std::nullopt, + state, + [&state, lockedFlake, callFlakeSchemasNix, lockedDefaultSchemasFlake]() { + auto vCallFlakeSchemas = state.allocValue(); + state.eval( + state.parseExprFromString(callFlakeSchemasNix, state.rootPath(CanonPath::root)), *vCallFlakeSchemas); + + auto vFlake = state.allocValue(); + flake::callFlake(state, *lockedFlake, *vFlake); + + auto vDefaultSchemasFlake = state.allocValue(); + if (vFlake->type() == nAttrs && vFlake->attrs()->get(state.symbols.create("schemas"))) + vDefaultSchemasFlake->mkNull(); + else + flake::callFlake(state, lockedDefaultSchemasFlake, *vDefaultSchemasFlake); + + auto vRes = state.allocValue(); + Value * args[] = {vDefaultSchemasFlake, vFlake}; + state.callFunction(*vCallFlakeSchemas, 2, args, *vRes, noPos); + + return vRes; + }); + + return {cache, cache->getRoot()->getAttr("inventory")}; +} + +/* Derive the flake output attribute path from the cursor used to + traverse the inventory. We do this so we don't have to maintain a + separate attrpath for that. */ +std::vector toAttrPath(ref cursor) +{ + auto attrPath = cursor->getAttrPath(); + std::vector res; + auto i = attrPath.begin(); + assert(i != attrPath.end()); + ++i; // skip "inventory" + assert(i != attrPath.end()); + res.push_back(*i++); // copy output name + if (i != attrPath.end()) + ++i; // skip "outputs" + while (i != attrPath.end()) { + ++i; // skip "children" + if (i != attrPath.end()) + res.push_back(*i++); + } + return res; +} + +std::string toAttrPathStr(ref cursor) +{ + return concatStringsSep(".", cursor->root->state.symbols.resolve(toAttrPath(cursor))); +} + +void forEachOutput( + ref inventory, + std::function output, const std::string & doc, bool isLast)> f) +{ + // FIXME: handle non-IFD outputs first. + // evalSettings.enableImportFromDerivation.setDefault(false); + + auto outputNames = inventory->getAttrs(); + for (const auto & [i, outputName] : enumerate(outputNames)) { + auto output = inventory->getAttr(outputName); + try { + auto isUnknown = (bool) output->maybeGetAttr("unknown"); + Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", toAttrPathStr(output))); + f(outputName, + isUnknown ? std::shared_ptr() : output->getAttr("output"), + isUnknown ? "" : output->getAttr("doc")->getString(), + i + 1 == outputNames.size()); + } catch (Error & e) { + e.addTrace(nullptr, "while evaluating the flake output '%s':", toAttrPathStr(output)); + throw; + } + } +} + +void visit( + std::optional system, + ref node, + std::function leaf)> visitLeaf, + std::function)> visitNonLeaf, + std::function node, const std::vector & systems)> visitFiltered) +{ + Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", toAttrPathStr(node))); + + /* Apply the system type filter. */ + if (system) { + if (auto forSystems = node->maybeGetAttr("forSystems")) { + auto systems = forSystems->getListOfStrings(); + if (std::find(systems.begin(), systems.end(), system) == systems.end()) { + visitFiltered(node, systems); + return; + } + } + } + + if (auto children = node->maybeGetAttr("children")) { + visitNonLeaf([&](ForEachChild f) { + auto attrNames = children->getAttrs(); + for (const auto & [i, attrName] : enumerate(attrNames)) { + try { + f(attrName, children->getAttr(attrName), i + 1 == attrNames.size()); + } catch (Error & e) { + // FIXME: make it a flake schema attribute whether to ignore evaluation errors. + if (node->root->state.symbols[toAttrPath(node)[0]] != "legacyPackages") { + e.addTrace(nullptr, "while evaluating the flake output attribute '%s':", toAttrPathStr(node)); + throw; + } + } + } + }); + } + + else + visitLeaf(ref(node)); +} + +std::optional what(ref leaf) +{ + if (auto what = leaf->maybeGetAttr("what")) + return what->getString(); + else + return std::nullopt; +} + +std::optional shortDescription(ref leaf) +{ + if (auto what = leaf->maybeGetAttr("shortDescription")) { + auto s = trim(what->getString()); + if (s != "") + return s; + } + return std::nullopt; +} + +std::shared_ptr derivation(ref leaf) +{ + return leaf->maybeGetAttr("derivation"); +} + +MixFlakeSchemas::MixFlakeSchemas() +{ + addFlag( + {.longName = "default-flake-schemas", + .description = "The URL of the flake providing default flake schema definitions.", + .labels = {"flake-ref"}, + .handler = {&defaultFlakeSchemas}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }}}); +} + +std::optional MixFlakeSchemas::getDefaultFlakeSchemas() +{ + if (!defaultFlakeSchemas) + return std::nullopt; + else + return parseFlakeRef(*defaultFlakeSchemas, absPath(".")); +} + +} diff --git a/src/nix/flake-schemas.hh b/src/nix/flake-schemas.hh new file mode 100644 index 000000000..9d1ba75a0 --- /dev/null +++ b/src/nix/flake-schemas.hh @@ -0,0 +1,45 @@ +#include "eval-cache.hh" +#include "flake/flake.hh" +#include "command.hh" + +namespace nix::flake_schemas { + +using namespace eval_cache; + +std::tuple, ref> +call(EvalState & state, std::shared_ptr lockedFlake, std::optional defaultSchemasFlake); + +std::vector toAttrPath(ref cursor); + +std::string toAttrPathStr(ref cursor); + +void forEachOutput( + ref inventory, + std::function output, const std::string & doc, bool isLast)> f); + +typedef std::function attr, bool isLast)> ForEachChild; + +void visit( + std::optional system, + ref node, + std::function leaf)> visitLeaf, + std::function)> visitNonLeaf, + std::function node, const std::vector & systems)> visitFiltered); + +std::optional what(ref leaf); + +std::optional shortDescription(ref leaf); + +std::shared_ptr derivation(ref leaf); + +/* Some helper functions for processing flake schema output. */ +struct MixFlakeSchemas : virtual Args, virtual StoreCommand +{ + std::optional defaultFlakeSchemas; + + MixFlakeSchemas(); + + std::optional getDefaultFlakeSchemas(); +}; + +} diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a86e36206..691632e2e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -17,6 +17,7 @@ #include "eval-cache.hh" #include "markdown.hh" #include "users.hh" +#include "flake-schemas.hh" #include #include @@ -164,31 +165,6 @@ struct CmdFlakeLock : FlakeCommand } }; -static void enumerateOutputs(EvalState & state, Value & vFlake, - std::function callback) -{ - auto pos = vFlake.determinePos(noPos); - state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); - - auto aOutputs = vFlake.attrs()->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceAttrs(*aOutputs->value, pos, "while evaluating the outputs of a flake"); - - auto sHydraJobs = state.symbols.create("hydraJobs"); - - /* Hack: ensure that hydraJobs is evaluated before anything - else. This way we can disable IFD for hydraJobs and then enable - it for other outputs. */ - if (auto attr = aOutputs->value->attrs()->get(sHydraJobs)) - callback(state.symbols[attr->name], *attr->value, attr->pos); - - for (auto & attr : *aOutputs->value->attrs()) { - if (attr.name != sHydraJobs) - callback(state.symbols[attr.name], *attr.value, attr.pos); - } -} - struct CmdFlakeMetadata : FlakeCommand, MixJSON { std::string description() override @@ -319,7 +295,7 @@ struct CmdFlakeInfo : CmdFlakeMetadata } }; -struct CmdFlakeCheck : FlakeCommand +struct CmdFlakeCheck : FlakeCommand, flake_schemas::MixFlakeSchemas { bool build = true; bool checkAllSystems = false; @@ -360,16 +336,26 @@ struct CmdFlakeCheck : FlakeCommand auto state = getEvalState(); lockFlags.applyNixConfig = true; - auto flake = lockFlake(); + auto flake = std::make_shared(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); + auto [cache, inventory] = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); + + std::vector drvPaths; + + std::set uncheckedOutputs; + std::set omittedSystems; + + std::function node)> visit; + bool hasErrors = false; + auto reportError = [&](const Error & e) { try { throw e; } catch (Error & e) { if (settings.keepGoing) { - ignoreException(); + logError({.msg = e.info().msg}); hasErrors = true; } else @@ -377,428 +363,70 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::set omittedSystems; - - // FIXME: rewrite to use EvalCache. - - auto resolve = [&] (PosIdx p) { - return state->positions[p]; - }; - - auto argHasName = [&] (Symbol arg, std::string_view expected) { - std::string_view name = state->symbols[arg]; - return - name == expected - || name == "_" - || (hasPrefix(name, "_") && name.substr(1) == expected); - }; - - auto checkSystemName = [&](const std::string & system, const PosIdx pos) { - // FIXME: what's the format of "system"? - if (system.find('-') == std::string::npos) - reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); - }; - - auto checkSystemType = [&](const std::string & system, const PosIdx pos) { - if (!checkAllSystems && system != localSystem) { - omittedSystems.insert(system); - return false; - } else { - return true; - } - }; - - auto checkDerivation = [&](const std::string & attrPath, Value & v, const PosIdx pos) -> std::optional { - try { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking derivation %s", attrPath)); - auto packageInfo = getDerivation(*state, v, false); - if (!packageInfo) - throw Error("flake attribute '%s' is not a derivation", attrPath); - else { - // FIXME: check meta attributes - auto storePath = packageInfo->queryDrvPath(); - if (storePath) { - logger->log(lvlInfo, - fmt("derivation evaluated to %s", - store->printStorePath(storePath.value()))); - } - return storePath; - } - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking the derivation '%s'", attrPath)); - reportError(e); - } - return std::nullopt; - }; - - std::vector drvPaths; - - auto checkApp = [&](const std::string & attrPath, Value & v, const PosIdx pos) { - try { - #if 0 - // FIXME - auto app = App(*state, v); - for (auto & i : app.context) { - auto [drvPathS, outputName] = NixStringContextElem::parse(i); - store->parseStorePath(drvPathS); - } - #endif - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking the app definition '%s'", attrPath)); - reportError(e); - } - }; - - auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { - try { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking overlay '%s'", attrPath)); - state->forceValue(v, pos); - if (!v.isLambda()) { - throw Error("overlay is not a function, but %s instead", showType(v)); - } - if (v.payload.lambda.fun->hasFormals() - || !argHasName(v.payload.lambda.fun->arg, "final")) - throw Error("overlay does not take an argument named 'final'"); - // FIXME: if we have a 'nixpkgs' input, use it to - // evaluate the overlay. - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking the overlay '%s'", attrPath)); - reportError(e); - } - }; - - auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { - try { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking NixOS module '%s'", attrPath)); - state->forceValue(v, pos); - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking the NixOS module '%s'", attrPath)); - reportError(e); - } - }; - - std::function checkHydraJobs; - - checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { - try { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking Hydra job '%s'", attrPath)); - state->forceAttrs(v, pos, ""); - - if (state->isDerivation(v)) - throw Error("jobset should not be a derivation at top-level"); - - for (auto & attr : *v.attrs()) { - state->forceAttrs(*attr.value, attr.pos, ""); - auto attrPath2 = concatStrings(attrPath, ".", state->symbols[attr.name]); - if (state->isDerivation(*attr.value)) { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking Hydra job '%s'", attrPath2)); - checkDerivation(attrPath2, *attr.value, attr.pos); - } else - checkHydraJobs(attrPath2, *attr.value, attr.pos); - } - - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking the Hydra jobset '%s'", attrPath)); - reportError(e); - } - }; - - auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const PosIdx pos) { - try { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking NixOS configuration '%s'", attrPath)); - Bindings & bindings(*state->allocBindings(0)); - auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; - state->forceValue(*vToplevel, pos); - if (!state->isDerivation(*vToplevel)) - throw Error("attribute 'config.system.build.toplevel' is not a derivation"); - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking the NixOS configuration '%s'", attrPath)); - reportError(e); - } - }; - - auto checkTemplate = [&](const std::string & attrPath, Value & v, const PosIdx pos) { - try { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking template '%s'", attrPath)); - - state->forceAttrs(v, pos, ""); - - if (auto attr = v.attrs()->get(state->symbols.create("path"))) { - if (attr->name == state->symbols.create("path")) { - NixStringContext context; - auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); - if (!path.pathExists()) - throw Error("template '%s' refers to a non-existent path '%s'", attrPath, path); - // TODO: recursively check the flake in 'path'. - } - } else - throw Error("template '%s' lacks attribute 'path'", attrPath); - - if (auto attr = v.attrs()->get(state->symbols.create("description"))) - state->forceStringNoCtx(*attr->value, attr->pos, ""); - else - throw Error("template '%s' lacks attribute 'description'", attrPath); - - for (auto & attr : *v.attrs()) { - std::string_view name(state->symbols[attr.name]); - if (name != "path" && name != "description" && name != "welcomeText") - throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); - } - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking the template '%s'", attrPath)); - reportError(e); - } - }; - - auto checkBundler = [&](const std::string & attrPath, Value & v, const PosIdx pos) { - try { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking bundler '%s'", attrPath)); - state->forceValue(v, pos); - if (!v.isLambda()) - throw Error("bundler must be a function"); - // TODO: check types of inputs/outputs? - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking the template '%s'", attrPath)); - reportError(e); - } - }; - + visit = [&](ref node) { - Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); + flake_schemas::visit( + checkAllSystems ? std::optional() : localSystem, + node, - auto vFlake = state->allocValue(); - flake::callFlake(*state, flake, *vFlake); - - enumerateOutputs(*state, - *vFlake, - [&](const std::string & name, Value & vOutput, const PosIdx pos) { - Activity act(*logger, lvlInfo, actUnknown, - fmt("checking flake output '%s'", name)); - - try { - evalSettings.enableImportFromDerivation.setDefault(name != "hydraJobs"); - - state->forceValue(vOutput, pos); - - std::string_view replacement = - name == "defaultPackage" ? "packages..default" : - name == "defaultApp" ? "apps..default" : - name == "defaultTemplate" ? "templates.default" : - name == "defaultBundler" ? "bundlers..default" : - name == "overlay" ? "overlays.default" : - name == "devShell" ? "devShells..default" : - name == "nixosModule" ? "nixosModules.default" : - ""; - if (replacement != "") - warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); - - if (name == "checks") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs()) { - auto drvPath = checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - if (drvPath && attr_name == settings.thisSystem.get()) { - drvPaths.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(*drvPath), - .outputs = OutputsSpec::All { }, - }); - } - } - } - } + [&](ref leaf) + { + if (auto evalChecks = leaf->maybeGetAttr("evalChecks")) { + auto checkNames = evalChecks->getAttrs(); + for (auto & checkName : checkNames) { + // FIXME: update activity + auto cursor = evalChecks->getAttr(checkName); + auto b = cursor->getBool(); + if (!b) + reportError(Error("Evaluation check '%s' failed.", flake_schemas::toAttrPathStr(cursor))); } - - else if (name == "formatter") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - checkApp( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); - }; - } - } - - else if (name == "packages" || name == "devShells") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs()) - checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - }; - } - } - - else if (name == "apps") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs()) - checkApp( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - }; - } - } - - else if (name == "defaultPackage" || name == "devShell") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - checkDerivation( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); - }; - } - } - - else if (name == "defaultApp") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos) ) { - checkApp( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); - }; - } - } - - else if (name == "legacyPackages") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - checkSystemName(state->symbols[attr.name], attr.pos); - checkSystemType(state->symbols[attr.name], attr.pos); - // FIXME: do getDerivations? - } - } - - else if (name == "overlay") - checkOverlay(name, vOutput, pos); - - else if (name == "overlays") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) - checkOverlay(fmt("%s.%s", name, state->symbols[attr.name]), - *attr.value, attr.pos); - } - - else if (name == "nixosModule") - checkModule(name, vOutput, pos); - - else if (name == "nixosModules") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) - checkModule(fmt("%s.%s", name, state->symbols[attr.name]), - *attr.value, attr.pos); - } - - else if (name == "nixosConfigurations") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) - checkNixOSConfiguration(fmt("%s.%s", name, state->symbols[attr.name]), - *attr.value, attr.pos); - } - - else if (name == "hydraJobs") - checkHydraJobs(name, vOutput, pos); - - else if (name == "defaultTemplate") - checkTemplate(name, vOutput, pos); - - else if (name == "templates") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) - checkTemplate(fmt("%s.%s", name, state->symbols[attr.name]), - *attr.value, attr.pos); - } - - else if (name == "defaultBundler") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - checkBundler( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); - }; - } - } - - else if (name == "bundlers") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs()) { - checkBundler( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - } - }; - } - } - - else if ( - name == "lib" - || name == "darwinConfigurations" - || name == "darwinModules" - || name == "flakeModule" - || name == "flakeModules" - || name == "herculesCI" - || name == "homeConfigurations" - || name == "homeModule" - || name == "homeModules" - || name == "nixopsConfigurations" - ) - // Known but unchecked community attribute - ; - - else - warn("unknown flake output '%s'", name); - - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking flake output '%s'", name)); - reportError(e); } + + if (auto drv = flake_schemas::derivation(leaf)) { + if (auto isFlakeCheck = leaf->maybeGetAttr("isFlakeCheck")) { + if (isFlakeCheck->getBool()) { + auto drvPath = drv->forceDerivation(); + drvPaths.push_back(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::All { }, + }); + } + } + } + }, + + [&](std::function forEachChild) + { + forEachChild([&](Symbol attrName, ref node, bool isLast) + { + visit(node); + }); + }, + + [&](ref node, const std::vector & systems) { + for (auto & s : systems) + omittedSystems.insert(s); }); - } + }; + + flake_schemas::forEachOutput(inventory, [&](Symbol outputName, std::shared_ptr output, const std::string & doc, bool isLast) + { + if (output) { + visit(ref(output)); + } else + uncheckedOutputs.insert(state->symbols[outputName]); + }); + + if (!uncheckedOutputs.empty()) + warn("The following flake outputs are unchecked: %s.", + concatStringsSep(", ", uncheckedOutputs)); // FIXME: quote if (build && !drvPaths.empty()) { Activity act(*logger, lvlInfo, actUnknown, fmt("running %d flake checks", drvPaths.size())); store->buildPaths(drvPaths); } + if (hasErrors) throw Error("some errors were encountered during the evaluation"); @@ -808,7 +436,7 @@ struct CmdFlakeCheck : FlakeCommand "Use '--all-systems' to check all.", concatStringsSep(", ", omittedSystems) ); - }; + } }; }; @@ -1092,7 +720,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun } }; -struct CmdFlakeShow : FlakeCommand, MixJSON +struct CmdFlakeShow : FlakeCommand, MixJSON, flake_schemas::MixFlakeSchemas { bool showLegacy = false; bool showAllSystems = false; @@ -1125,267 +753,158 @@ struct CmdFlakeShow : FlakeCommand, MixJSON void run(nix::ref store) override { - evalSettings.enableImportFromDerivation.setDefault(false); - auto state = getEvalState(); auto flake = std::make_shared(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); - std::function &attrPath, - const Symbol &attr)> hasContent; + auto [cache, inventory] = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); - // For frameworks it's important that structures are as lazy as possible - // to prevent infinite recursions, performance issues and errors that - // aren't related to the thing to evaluate. As a consequence, they have - // to emit more attributes than strictly (sic) necessary. - // However, these attributes with empty values are not useful to the user - // so we omit them. - hasContent = [&]( - eval_cache::AttrCursor & visitor, - const std::vector &attrPath, - const Symbol &attr) -> bool - { - auto attrPath2(attrPath); - attrPath2.push_back(attr); - auto attrPathS = state->symbols.resolve(attrPath2); - const auto & attrName = state->symbols[attr]; + if (json) { + std::function node, nlohmann::json & obj)> visit; - auto visitor2 = visitor.getAttr(attrName); + visit = [&](ref node, nlohmann::json & obj) + { + flake_schemas::visit( + showAllSystems ? std::optional() : localSystem, + node, - try { - if ((attrPathS[0] == "apps" - || attrPathS[0] == "checks" - || attrPathS[0] == "devShells" - || attrPathS[0] == "legacyPackages" - || attrPathS[0] == "packages") - && (attrPathS.size() == 1 || attrPathS.size() == 2)) { - for (const auto &subAttr : visitor2->getAttrs()) { - if (hasContent(*visitor2, attrPath2, subAttr)) { - return true; - } - } - return false; + [&](ref leaf) + { + obj.emplace("leaf", true); + + if (auto what = flake_schemas::what(leaf)) + obj.emplace("what", what); + + if (auto shortDescription = flake_schemas::shortDescription(leaf)) + obj.emplace("shortDescription", shortDescription); + + if (auto drv = flake_schemas::derivation(leaf)) + obj.emplace("derivationName", drv->getAttr(state->sName)->getString()); + + // FIXME: add more stuff + }, + + [&](std::function forEachChild) + { + auto children = nlohmann::json::object(); + forEachChild([&](Symbol attrName, ref node, bool isLast) + { + auto j = nlohmann::json::object(); + try { + visit(node, j); + } catch (EvalError & e) { + // FIXME: make it a flake schema attribute whether to ignore evaluation errors. + if (node->root->state.symbols[flake_schemas::toAttrPath(node)[0]] == "legacyPackages") + j.emplace("failed", true); + else + throw; + } + children.emplace(state->symbols[attrName], std::move(j)); + }); + obj.emplace("children", std::move(children)); + }, + + [&](ref node, const std::vector & systems) + { + obj.emplace("filtered", true); + }); + }; + + auto res = nlohmann::json::object(); + + flake_schemas::forEachOutput(inventory, [&](Symbol outputName, std::shared_ptr output, const std::string & doc, bool isLast) + { + auto j = nlohmann::json::object(); + + if (!showLegacy && state->symbols[outputName] == "legacyPackages") { + j.emplace("skipped", true); + } else if (output) { + j.emplace("doc", doc); + auto j2 = nlohmann::json::object(); + visit(ref(output), j2); + j.emplace("output", std::move(j2)); + } else + j.emplace("unknown", true); + + res.emplace(state->symbols[outputName], j); + }); + + logger->cout("%s", res.dump()); + } + + else { + logger->cout(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef); + + std::function node, + const std::string & headerPrefix, + const std::string & prevPrefix)> visit; + + visit = [&]( + ref node, + const std::string & headerPrefix, + const std::string & prevPrefix) + { + flake_schemas::visit( + showAllSystems ? std::optional() : localSystem, + node, + + [&](ref leaf) + { + auto s = headerPrefix; + + if (auto what = flake_schemas::what(leaf)) + s += fmt(": %s", *what); + + if (auto drv = flake_schemas::derivation(leaf)) + s += fmt(ANSI_ITALIC " [%s]" ANSI_NORMAL, drv->getAttr(state->sName)->getString()); + + logger->cout(s); + }, + + [&](std::function forEachChild) + { + logger->cout(headerPrefix); + forEachChild([&](Symbol attrName, ref node, bool isLast) + { + visit(node, + fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, prevPrefix, + isLast ? treeLast : treeConn, state->symbols[attrName]), + prevPrefix + (isLast ? treeNull : treeLine)); + }); + }, + + [&](ref node, const std::vector & systems) + { + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + }); + }; + + flake_schemas::forEachOutput(inventory, [&](Symbol outputName, std::shared_ptr output, const std::string & doc, bool isLast) + { + auto headerPrefix = fmt( + ANSI_GREEN "%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, + isLast ? treeLast : treeConn, state->symbols[outputName]); + + if (!showLegacy && state->symbols[outputName] == "legacyPackages") { + logger->cout(headerPrefix); + logger->cout( + ANSI_GREEN "%s" "%s" ANSI_NORMAL ANSI_ITALIC "%s" ANSI_NORMAL, + isLast ? treeNull : treeLine, + treeLast, + "(skipped; use '--legacy' to show)"); + } else if (output) { + visit(ref(output), headerPrefix, isLast ? treeNull : treeLine); + } else { + logger->cout(headerPrefix); + logger->cout( + ANSI_GREEN "%s" "%s" ANSI_NORMAL ANSI_ITALIC "%s" ANSI_NORMAL, + isLast ? treeNull : treeLine, + treeLast, + "(unknown flake output)"); } - - if ((attrPathS.size() == 1) - && (attrPathS[0] == "formatter" - || attrPathS[0] == "nixosConfigurations" - || attrPathS[0] == "nixosModules" - || attrPathS[0] == "overlays" - )) { - for (const auto &subAttr : visitor2->getAttrs()) { - if (hasContent(*visitor2, attrPath2, subAttr)) { - return true; - } - } - return false; - } - - // If we don't recognize it, it's probably content - return true; - } catch (EvalError & e) { - // Some attrs may contain errors, e.g. legacyPackages of - // nixpkgs. We still want to recurse into it, instead of - // skipping it at all. - return true; - } - }; - - std::function & attrPath, - const std::string & headerPrefix, - const std::string & nextPrefix)> visit; - - visit = [&]( - eval_cache::AttrCursor & visitor, - const std::vector & attrPath, - const std::string & headerPrefix, - const std::string & nextPrefix) - -> nlohmann::json - { - auto j = nlohmann::json::object(); - - auto attrPathS = state->symbols.resolve(attrPath); - - Activity act(*logger, lvlInfo, actUnknown, - fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); - - try { - auto recurse = [&]() - { - if (!json) - logger->cout("%s", headerPrefix); - std::vector attrs; - for (const auto &attr : visitor.getAttrs()) { - if (hasContent(visitor, attrPath, attr)) - attrs.push_back(attr); - } - - for (const auto & [i, attr] : enumerate(attrs)) { - const auto & attrName = state->symbols[attr]; - bool last = i + 1 == attrs.size(); - auto visitor2 = visitor.getAttr(attrName); - auto attrPath2(attrPath); - attrPath2.push_back(attr); - auto j2 = visit(*visitor2, attrPath2, - fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attrName), - nextPrefix + (last ? treeNull : treeLine)); - if (json) j.emplace(attrName, std::move(j2)); - } - }; - - auto showDerivation = [&]() - { - auto name = visitor.getAttr(state->sName)->getString(); - if (json) { - std::optional description; - if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { - if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) - description = aDescription->getString(); - } - j.emplace("type", "derivation"); - j.emplace("name", name); - if (description) - j.emplace("description", *description); - } else { - logger->cout("%s: %s '%s'", - headerPrefix, - attrPath.size() == 2 && attrPathS[0] == "devShell" ? "development environment" : - attrPath.size() >= 2 && attrPathS[0] == "devShells" ? "development environment" : - attrPath.size() == 3 && attrPathS[0] == "checks" ? "derivation" : - attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : - "package", - name); - } - }; - - if (attrPath.size() == 0 - || (attrPath.size() == 1 && ( - attrPathS[0] == "defaultPackage" - || attrPathS[0] == "devShell" - || attrPathS[0] == "formatter" - || attrPathS[0] == "nixosConfigurations" - || attrPathS[0] == "nixosModules" - || attrPathS[0] == "defaultApp" - || attrPathS[0] == "templates" - || attrPathS[0] == "overlays")) - || ((attrPath.size() == 1 || attrPath.size() == 2) - && (attrPathS[0] == "checks" - || attrPathS[0] == "packages" - || attrPathS[0] == "devShells" - || attrPathS[0] == "apps")) - ) - { - recurse(); - } - - else if ( - (attrPath.size() == 2 && (attrPathS[0] == "defaultPackage" || attrPathS[0] == "devShell" || attrPathS[0] == "formatter")) - || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells")) - ) - { - if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { - if (!json) - logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); - else { - logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); - } - } else { - if (visitor.isDerivation()) - showDerivation(); - else - throw Error("expected a derivation"); - } - } - - else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { - if (visitor.isDerivation()) - showDerivation(); - else - recurse(); - } - - else if (attrPath.size() > 0 && attrPathS[0] == "legacyPackages") { - if (attrPath.size() == 1) - recurse(); - else if (!showLegacy){ - if (!json) - logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); - else { - logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); - } - } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { - if (!json) - logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); - else { - logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); - } - } else { - if (visitor.isDerivation()) - showDerivation(); - else if (attrPath.size() <= 2) - // FIXME: handle recurseIntoAttrs - recurse(); - } - } - - else if ( - (attrPath.size() == 2 && attrPathS[0] == "defaultApp") || - (attrPath.size() == 3 && attrPathS[0] == "apps")) - { - auto aType = visitor.maybeGetAttr("type"); - if (!aType || aType->getString() != "app") - state->error("not an app definition").debugThrow(); - if (json) { - j.emplace("type", "app"); - } else { - logger->cout("%s: app", headerPrefix); - } - } - - else if ( - (attrPath.size() == 1 && attrPathS[0] == "defaultTemplate") || - (attrPath.size() == 2 && attrPathS[0] == "templates")) - { - auto description = visitor.getAttr("description")->getString(); - if (json) { - j.emplace("type", "template"); - j.emplace("description", description); - } else { - logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); - } - } - - else { - auto [type, description] = - (attrPath.size() == 1 && attrPathS[0] == "overlay") - || (attrPath.size() == 2 && attrPathS[0] == "overlays") ? std::make_pair("nixpkgs-overlay", "Nixpkgs overlay") : - attrPath.size() == 2 && attrPathS[0] == "nixosConfigurations" ? std::make_pair("nixos-configuration", "NixOS configuration") : - (attrPath.size() == 1 && attrPathS[0] == "nixosModule") - || (attrPath.size() == 2 && attrPathS[0] == "nixosModules") ? std::make_pair("nixos-module", "NixOS module") : - std::make_pair("unknown", "unknown"); - if (json) { - j.emplace("type", type); - } else { - logger->cout("%s: " ANSI_WARNING "%s" ANSI_NORMAL, headerPrefix, description); - } - } - } catch (EvalError & e) { - if (!(attrPath.size() > 0 && attrPathS[0] == "legacyPackages")) - throw; - } - - return j; - }; - - auto cache = openEvalCache(*state, flake); - - auto j = visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); - if (json) - logger->cout("%s", j.dump()); + }); + } } }; diff --git a/src/nix/local.mk b/src/nix/local.mk index 28b30b586..43a22a2af 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -55,3 +55,9 @@ $(d)/main.cc: \ $(d)/profile.cc: $(d)/profile.md $(d)/profile.md: $(d)/profiles.md.gen.hh + +src/nix/flake.cc: src/nix/call-flake-schemas.nix.gen.hh src/nix/builtin-flake-schemas.nix.gen.hh + +src/nix/builtin-flake-schemas.nix: $(default_flake_schemas)/flake.nix + $(trace-gen) cp $^ $@ + @chmod +w $@ diff --git a/tests/functional/flakes/check.sh b/tests/functional/flakes/check.sh index 3b83dcafe..48a0d333a 100755 --- a/tests/functional/flakes/check.sh +++ b/tests/functional/flakes/check.sh @@ -16,17 +16,6 @@ EOF nix flake check $flakeDir -cat > $flakeDir/flake.nix < $flakeDir/flake.nix < show-output.json nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in -assert show_output.packages.someOtherSystem.default == {}; -assert show_output.packages.${builtins.currentSystem}.default.name == "simple"; -assert show_output.legacyPackages.${builtins.currentSystem} == {}; +assert show_output.packages.output.children.someOtherSystem.filtered; +assert show_output.packages.output.children.${builtins.currentSystem}.children.default.derivationName == "simple"; +assert show_output.legacyPackages.skipped; true ' @@ -26,8 +26,8 @@ nix flake show --json --all-systems > show-output.json nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in -assert show_output.packages.someOtherSystem.default.name == "simple"; -assert show_output.legacyPackages.${builtins.currentSystem} == {}; +assert show_output.packages.output.children.someOtherSystem.children.default.derivationName == "simple"; +assert show_output.legacyPackages.skipped; true ' @@ -36,34 +36,7 @@ nix flake show --json --legacy > show-output.json nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in -assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple"; -true -' - -# Test that attributes are only reported when they have actual content -cat >flake.nix < show-output.json -nix eval --impure --expr ' -let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); -in -assert show_output == { }; +assert show_output.legacyPackages.output.children.${builtins.currentSystem}.children.hello.derivationName == "simple"; true ' @@ -83,7 +56,7 @@ nix flake show --json --legacy --all-systems > show-output.json nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in -assert show_output.legacyPackages.${builtins.currentSystem}.AAAAAASomeThingsFailToEvaluate == { }; -assert show_output.legacyPackages.${builtins.currentSystem}.simple.name == "simple"; +assert show_output.legacyPackages.output.children.${builtins.currentSystem}.children.AAAAAASomeThingsFailToEvaluate.failed; +assert show_output.legacyPackages.output.children.${builtins.currentSystem}.children.simple.derivationName == "simple"; true ' diff --git a/tests/functional/fmt.sh b/tests/functional/fmt.sh index b29fe64d6..b0a0b2e5f 100755 --- a/tests/functional/fmt.sh +++ b/tests/functional/fmt.sh @@ -32,4 +32,6 @@ cat << EOF > flake.nix EOF nix fmt ./file ./folder | grep 'Formatting: ./file ./folder' nix flake check -nix flake show | grep -P "package 'formatter'" + +clearStore +nix flake show | grep -P "package.*\[formatter\]"