From 408dcfc0d3898fb855c1038d24c09c82fe58c379 Mon Sep 17 00:00:00 2001 From: Sandro Date: Mon, 5 Sep 2022 15:42:10 +0200 Subject: [PATCH 001/402] Improve experimental-features error wording --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index fa79cca6b..d8b0d30fa 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -52,7 +52,7 @@ std::set parseFeatures(const std::set & rawFea } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; pass '--extra-experimental-features %1%' as an argument to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} From 481e4082bf96f5613693785e97a9a5cec7c09940 Mon Sep 17 00:00:00 2001 From: Sandro Date: Wed, 9 Nov 2022 12:03:53 +0100 Subject: [PATCH 002/402] Update src/libutil/experimental-features.cc Co-authored-by: Valentin Gagarin --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index d8b0d30fa..052988b34 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -52,7 +52,7 @@ std::set parseFeatures(const std::set & rawFea } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; pass '--extra-experimental-features %1%' as an argument to enable it", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} From c74248c56e6e5558603be6c6eda43eb439772f20 Mon Sep 17 00:00:00 2001 From: Michael Lohmann Date: Thu, 12 Jan 2023 15:45:19 +0100 Subject: [PATCH 003/402] install-multi-user: ignore profile_target backups that have no change If there was a prior nix installation that created this backup file and then you tried to install it again, it would stop to tell you there is this file. But if the file and its backup are identical in content, there is no harm in continuing and in a later step overwriting the existing backup file with the identical one. This is just a convenience feature. --- scripts/install-multi-user.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index f149ea0d7..b2f2a9e45 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -445,6 +445,14 @@ EOF # a row for different files. if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then # this backup process first released in Nix 2.1 + + if diff -q "$profile_target$PROFILE_BACKUP_SUFFIX" "$profile_target" > /dev/null; then + # a backup file for the rc-file exist, but they are identical, + # so we can safely ignore it and overwrite it with the same + # content later + continue + fi + failure < Date: Wed, 19 Apr 2023 16:32:47 -0700 Subject: [PATCH 004/402] Mention `$DRV_PATH` in post-build-hook docs --- doc/manual/src/advanced-topics/post-build-hook.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index 1479cc3a4..17b67111c 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -69,6 +69,8 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. +> If you want to upload the .drv file too, the `$DRV_PATH` variable +> is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: From bf693319f63601c0e72107012328c298cd78e88b Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Tue, 14 Mar 2023 11:54:04 -0300 Subject: [PATCH 005/402] feat: add always-allow-substitutes This adds a new configuration option to Nix, `always-allow-substitutes`, whose effect is simple: it causes the `allowSubstitutes` attribute in derivations to be ignored, and for substituters to always be used. This is extremely valuable for users of Nix in CI, where usually `nix-build-uncached` is used. There, derivations which disallow substitutes cause headaches as the inputs for building already-cached derivations need to be fetched to spuriously rebuild some simple text file. This option should be a good middle-ground, since it doesn't imply rebuilding the world, such as the approach I took in https://github.com/NixOS/nixpkgs/pull/221048 --- doc/manual/src/language/advanced-attributes.md | 3 +++ src/libstore/globals.hh | 8 ++++++++ src/libstore/parsed-derivations.cc | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 307971434..211d195b2 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -261,6 +261,9 @@ Derivations can declare some infrequently used optional attributes. useful for very trivial derivations (such as `writeText` in Nixpkgs) that are cheaper to build than to substitute from a binary cache. + You may disable the effects of this attibute by enabling the + `always-allow-substitutes` configuration option in Nix. + > **Note** > > You need to have a builder configured which satisfies the diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 31dfe5b4e..8b0e3fc7d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -255,6 +255,14 @@ public: For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) )"}; + Setting alwaysAllowSubstitutes{ + this, false, "always-allow-substitutes", + R"( + If set to `true`, Nix will ignore the `allowSubstitutes` attribute in + derivations and always attempt to use available substituters. + For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md). + )"}; + Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", R"( diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index cc4a94fab..1d900c272 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -122,7 +122,7 @@ bool ParsedDerivation::willBuildLocally(Store & localStore) const bool ParsedDerivation::substitutesAllowed() const { - return getBoolAttr("allowSubstitutes", true); + return settings.alwaysAllowSubstitutes ? true : getBoolAttr("allowSubstitutes", true); } bool ParsedDerivation::useUidRange() const From f11445952fcf0f5fd3f827997591d43a41f2074a Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 21:36:56 +0200 Subject: [PATCH 006/402] Document builtins.fetchTree Co-authored-by: Valentin Gagarin Supersedes #6740 --- src/libexpr/primops/fetchTree.cc | 43 ++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 579a45f92..3b048c446 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -198,10 +198,49 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } -// FIXME: document static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", - .arity = 1, + .args = {"input"}, + .doc = R"( + Fetch a source tree or a plain file using one of the supported backends. + *input* can be an attribute set representation of [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) or a URL. + The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is allowed. + + Here are some examples of how to use `fetchTree`: + + - Fetch a GitHub repository: + + ```nix + builtins.fetchTree { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + } + ``` + + This evaluates to attribute set: + + ``` + { + lastModified = 1686503798; + lastModifiedDate = "20230611171638"; + narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + shortRev = "ae2e6b3"; + } + ``` + - Fetch a single file from a URL: + + ```nix + builtins.fetchTree "https://example.com/" + ``` + + > **Note** + > + > This function requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + )", .fun = prim_fetchTree }); From feb01b22ed2954232455b5c8346cc5ed09c21f81 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 11:28:12 +0200 Subject: [PATCH 007/402] add links to store API documentation --- src/libstore/store-api.hh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eef..43aa00637 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -136,19 +136,22 @@ struct StoreConfig : public Config Setting priority{this, 0, "priority", R"( - Priority of this store when used as a substituter. A lower value means a higher priority. + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. )"}; Setting wantMassQuery{this, false, "want-mass-query", R"( - Whether this store (when used as a substituter) can be - queried efficiently for path validity. + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", - "Optional features that the system this store builds on implements (like \"kvm\")."}; + R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + Example: `"kvm"` + )" }; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig From 493ddf617f0b2c406dc35026b6d481a69cf1d934 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 11:28:27 +0200 Subject: [PATCH 008/402] reformat `system-features` setting documentation --- src/libstore/globals.hh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d4b8fb1f9..879588375 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -689,19 +689,21 @@ public: getDefaultSystemFeatures(), "system-features", R"( - A set of system “features” supported by this machine, e.g. `kvm`. - Derivations can express a dependency on such features through the - derivation attribute `requiredSystemFeatures`. For example, the - attribute + A set of system “features” supported by this machine. + Derivations can express a dependency on such features through the derivation attribute `requiredSystemFeatures`. + + For example, the attribute requiredSystemFeatures = [ "kvm" ]; - ensures that the derivation can only be built on a machine with the - `kvm` feature. + ensures that the derivation can only be built on a machine with the `kvm` feature. - This setting by default includes `kvm` if `/dev/kvm` is accessible, - and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` - that are used in Nixpkgs to route builds to specific machines. + This setting by default includes + - `kvm` if `/dev/kvm` is accessible + - historical pseudo-features for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines + - `nixos-test` + - `benchmark` + - `big-parallel` )", {}, false}; Setting substituters{ From 5f37ebcf83c856b08a63778e07bfc5c84ea4a5ec Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 11:57:20 +0200 Subject: [PATCH 009/402] document all special system features and their behavior --- .../src/language/advanced-attributes.md | 12 ++++++ doc/manual/src/release-notes/rl-2.12.md | 14 +------ src/libstore/globals.hh | 39 +++++++++++++------ 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 307971434..2961f9dcc 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -345,3 +345,15 @@ Derivations can declare some infrequently used optional attributes. This is useful, for example, when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. + +- [`requiredSystemFeatures`]{#adv-attr-requiredSystemFeatures}\ + + If a derivation has the `requiredSystemFeatures` attribute, then Nix will only build it on a machine that has the corresponding features set in its [`system-features` configuration](@docroot@/command-ref/conf-file.md#conf-system-features). + + For example, setting + + ```nix + requiredSystemFeatures = [ "kvm" ]; + ``` + + ensures that the derivation can only be built on a machine with the `kvm` feature. diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index e2045d7bf..57d092e01 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,20 +2,8 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. - - This is primarily useful for running containers such as `systemd-nspawn` - inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. - [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. - - A build can enable this by setting the derivation attribute: - - ``` - requiredSystemFeatures = [ "uid-range" ]; - ``` - - The `uid-range` [system feature] requires the [`auto-allocate-uids`] - setting to be enabled. + This can be used by requiring `uid-range` [system feature] in derivations. [system feature]: ../command-ref/conf-file.md#conf-system-features diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 879588375..9f852dc7b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -337,7 +337,7 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - )"}; + )", {}, true, Xp::AutoAllocateUids}; Setting startId{this, #if __linux__ @@ -690,20 +690,37 @@ public: "system-features", R"( A set of system “features” supported by this machine. - Derivations can express a dependency on such features through the derivation attribute `requiredSystemFeatures`. - For example, the attribute + This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. - requiredSystemFeatures = [ "kvm" ]; + Derivations can require system features in the derivation attribute [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures). - ensures that the derivation can only be built on a machine with the `kvm` feature. + System features are generally user-defined, but the following have special treatment: - This setting by default includes - - `kvm` if `/dev/kvm` is accessible - - historical pseudo-features for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines - - `nixos-test` - - `benchmark` - - `big-parallel` + - `kvm` + + Set by default if `/dev/kvm` is accessible. + + - `nixos-test`, `benchmark`, `big-parallel` + + These historical pseudo-features are always enabled for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines. + + - `ca-derivations` + + Set by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + + - `recursive-nix` + + Set by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + + - `uid-range` + + On Linux, Nix can run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + Set by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. )", {}, false}; Setting substituters{ From 2fa90e5824635998b445b6862bb49469e8fc75de Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 13:59:18 +0200 Subject: [PATCH 010/402] add more details on CA derivations --- doc/manual/src/language/advanced-attributes.md | 2 ++ src/libstore/globals.hh | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2961f9dcc..5292a2ffc 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -229,6 +229,8 @@ Derivations can declare some infrequently used optional attributes. [`outputHashAlgo`](#adv-attr-outputHashAlgo) like for *fixed-output derivations* (see above). + It also implicitly requires that the machine to build the derivation must have the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). + - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather than environment variables. For example, if you have diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9f852dc7b..c08a1a943 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -693,25 +693,27 @@ public: This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. - Derivations can require system features in the derivation attribute [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures). + A derivation can require system features in the [`requiredSystemFeatures` attribute](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures), and the machine to build the derivation must have them. - System features are generally user-defined, but the following have special treatment: + System features are user-defined, but Nix sets the following defaults: - `kvm` - Set by default if `/dev/kvm` is accessible. + Included by default if `/dev/kvm` is accessible. - `nixos-test`, `benchmark`, `big-parallel` - These historical pseudo-features are always enabled for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines. + These historical pseudo-features are always enabled for backwards compatibility, as they are used in Nixpkgs to route Hydra builds to specific machines. - `ca-derivations` - Set by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + + This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - `recursive-nix` - Set by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. - `uid-range` @@ -720,7 +722,7 @@ public: [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. - Set by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. + Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. )", {}, false}; Setting substituters{ From 52248b1c2786f7aa98109d47ee12f527bd202b12 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 19 Aug 2023 17:03:31 -0400 Subject: [PATCH 011/402] feat: notation to refer to no attribute search prefix An attrPath prefix of "." indicates no need to try default attrPath prefixes. For example 1nixpkgs#legacyPackages.x86_64-linux.ERROR` searches through ``` trying flake output attribute 'packages.x86_64-linux.legacyPackages.x86_64-linux.ERROR' using cached attrset attribute '' trying flake output attribute 'legacyPackages.x86_64-linux.legacyPackages.x86_64-linux.ERROR' using cached attrset attribute 'legacyPackages.x86_64-linux' trying flake output attribute 'legacyPackages.x86_64-linux.ERROR' using cached attrset attribute 'legacyPackages.x86_64-linux' ``` And there is no way to specify that one does not want the automatic search behavior. Now one can specify `nixpkgs#.legacyPackages.x86_64-linux.ERROR` to only refer to the rooted attribute path without any default injection of attribute search path or system. --- src/libcmd/installable-flake.cc | 5 +++++ src/libcmd/installables.cc | 12 ++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4074da06d..2f428cb7e 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -28,6 +28,11 @@ namespace nix { std::vector InstallableFlake::getActualAttrPaths() { std::vector res; + if (attrPaths.size() == 1 && attrPaths.front().starts_with(".")){ + attrPaths.front().erase(0,1); + res.push_back(attrPaths.front()); + return res; + } for (auto & prefix : prefixes) res.push_back(prefix + *attrPaths.begin()); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eb1903084..01c01441c 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -301,6 +301,11 @@ void completeFlakeRefWithFragment( completionType = ctAttrs; auto fragment = prefix.substr(hash + 1); + std::string prefixRoot = ""; + if (fragment.starts_with(".")){ + fragment = fragment.substr(1); + prefixRoot = "."; + } auto flakeRefS = std::string(prefix.substr(0, hash)); auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); @@ -309,6 +314,9 @@ void completeFlakeRefWithFragment( auto root = evalCache->getRoot(); + if (prefixRoot == "."){ + attrPathPrefixes.clear(); + } /* Complete 'fragment' relative to all the attrpath prefixes as well as the root of the flake. */ @@ -333,7 +341,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + completions->add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -344,7 +352,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->add(flakeRefS + "#"); + completions->add(flakeRefS + "#" + prefixRoot); } } } From c609be4072b03dc3eabcb2ac9916593847fd3b19 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 19 Aug 2023 17:19:52 -0400 Subject: [PATCH 012/402] doc: explain the . attrPath prefix notation --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix/nix.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7ddb5ca00..56222a21b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -22,3 +22,5 @@ - Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. diff --git a/src/nix/nix.md b/src/nix/nix.md index e0f459d6b..6e7e8a649 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -132,6 +132,8 @@ subcommands, these are `packages.`*system*, attributes `packages.x86_64-linux.hello`, `legacyPackages.x86_64-linux.hello` and `hello`. +If *attrpath* begins with `.` then no prefixes or defaults are attempted. This allows the form *flakeref*[`#.`*attrpath*], such as `github:NixOS/nixpkgs#.lib.fakeSha256` to avoid a search of `packages.*system*.lib.fakeSha256` + ### Store path Example: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` From 696eb79b150011629d86addd10f79c943b2f16b2 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sun, 27 Aug 2023 04:42:52 -0400 Subject: [PATCH 013/402] test: test behavior of .-prefixed attrPaths --- tests/flakes/absolute-attr-paths.sh | 17 +++++++++++++++++ tests/local.mk | 1 + 2 files changed, 18 insertions(+) create mode 100644 tests/flakes/absolute-attr-paths.sh diff --git a/tests/flakes/absolute-attr-paths.sh b/tests/flakes/absolute-attr-paths.sh new file mode 100644 index 000000000..491adceb7 --- /dev/null +++ b/tests/flakes/absolute-attr-paths.sh @@ -0,0 +1,17 @@ +source ./common.sh + +flake1Dir=$TEST_ROOT/flake1 + +mkdir -p $flake1Dir +cat > $flake1Dir/flake.nix < Date: Mon, 28 Aug 2023 20:51:44 +0200 Subject: [PATCH 014/402] do not change existing release notes --- doc/manual/src/release-notes/rl-2.12.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index 57d092e01..e1e3efe1a 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,8 +2,19 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` + inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. - This can be used by requiring `uid-range` [system feature] in derivations. + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + A build can enable this by setting the derivation attribute: + + ``` + requiredSystemFeatures = [ "uid-range" ]; + ``` + + The `uid-range` [system feature] requires the [`auto-allocate-uids`] + setting to be enabled. [system feature]: ../command-ref/conf-file.md#conf-system-features From 3ae1489847e3bde531833571d28a6e0915b4b766 Mon Sep 17 00:00:00 2001 From: Walter Franzini <5097668+wfranzini@users.noreply.github.com> Date: Tue, 31 Jan 2023 23:33:05 +0100 Subject: [PATCH 015/402] nix flakes metadata: Show lastModified timestamp for each input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this commit, the output of `nix flake metadata` in nix repo looked like this: ... Last modified: 2023-07-09 16:00:16 Inputs: ├───flake-compat: github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9 ├───lowdown-src: github:kristapsdz/lowdown/d2c2b44ff6c27b936ec27358a2653caaef8f73b8 ├───nixpkgs: github:NixOS/nixpkgs/04a75b2eecc0acf6239acf9dd04485ff8d14f425 └───nixpkgs-regression: github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2 This commit changes that to: ... Last modified: 2023-07-09 16:00:16 Inputs: ├───flake-compat: github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9 (2023-01-17 11:47:33) ├───lowdown-src: github:kristapsdz/lowdown/d2c2b44ff6c27b936ec27358a2653caaef8f73b8 (2021-10-06 10:00:07) ├───nixpkgs: github:NixOS/nixpkgs/04a75b2eecc0acf6239acf9dd04485ff8d14f425 (2022-12-08 01:04:00) └───nixpkgs-regression: github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2 (2022-01-24 19:20:45) --- src/nix/flake.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1b..b62c45d7e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -233,9 +233,13 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON bool last = i + 1 == node.inputs.size(); if (auto lockedNode = std::get_if<0>(&input.second)) { - logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + std::string lastModifiedStr = ""; + if (auto lastModified = (*lockedNode)->lockedRef.input.getLastModified()) + lastModifiedStr = fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%F %T")); + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s%s", prefix + (last ? treeLast : treeConn), input.first, - (*lockedNode)->lockedRef); + (*lockedNode)->lockedRef, + lastModifiedStr); bool firstVisit = visited.insert(*lockedNode).second; From 4c50f5d130fc3adc3b048923e267d936df5b6250 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 3 Sep 2023 13:44:32 +0200 Subject: [PATCH 016/402] traces: Do not print unknown location Solves 1/3 of the infinite recursion at unknown location meme. See #8879 for ensuring we always have a trace (for stack overflows) We might want to re-add this for finding missing location info *while hacking on that problem only*. --- src/libutil/error.cc | 12 ++++-------- src/libutil/error.hh | 7 +++++++ tests/lang/eval-fail-fromTOML-timestamps.err.exp | 2 -- tests/lang/eval-fail-hashfile-missing.err.exp | 4 ---- tests/lang/eval-fail-set-override.err.exp | 2 -- tests/lang/eval-fail-substring.err.exp | 2 -- tests/lang/eval-fail-to-path.err.exp | 2 -- 7 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 4a1b346ef..3917fffc3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -203,8 +203,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s std::ostringstream oss; - auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n"; - /* * Traces * ------ @@ -320,7 +318,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << "\n" << "… " << trace.hint.str() << "\n"; - if (trace.pos) { + if (trace.pos && *trace.pos) { count++; oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; @@ -329,8 +327,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << "\n"; printCodeLines(oss, "", *trace.pos, *loc); oss << "\n"; - } else - oss << noSource; + } } } oss << "\n" << prefix; @@ -338,15 +335,14 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << einfo.msg << "\n"; - if (einfo.errPos) { + if (einfo.errPos && *einfo.errPos) { oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; if (auto loc = einfo.errPos->getCodeLines()) { oss << "\n"; printCodeLines(oss, "", *einfo.errPos, *loc); oss << "\n"; - } else - oss << noSource; + } } auto suggestions = einfo.suggestions.trim(); diff --git a/src/libutil/error.hh b/src/libutil/error.hh index be5c5e252..c04dcbd77 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -70,6 +70,13 @@ struct AbstractPos uint32_t line = 0; uint32_t column = 0; + /** + * An AbstractPos may be a "null object", representing an unknown position. + * + * Return true if this position is known. + */ + inline operator bool() const { return line != 0; }; + /** * Return the contents of the source file. */ diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/lang/eval-fail-fromTOML-timestamps.err.exp index f6bd19f5a..5b60d253d 100644 --- a/tests/lang/eval-fail-fromTOML-timestamps.err.exp +++ b/tests/lang/eval-fail-fromTOML-timestamps.err.exp @@ -8,5 +8,3 @@ error: 2| key = "value" error: while parsing a TOML string: Dates and times are not supported - - at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/lang/eval-fail-hashfile-missing.err.exp index 8e77dec1e..6d38608c0 100644 --- a/tests/lang/eval-fail-hashfile-missing.err.exp +++ b/tests/lang/eval-fail-hashfile-missing.err.exp @@ -10,10 +10,6 @@ error: … while evaluating the first argument passed to builtins.toString - at «none»:0: (source not available) - … while calling the 'hashFile' builtin - at «none»:0: (source not available) - error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/lang/eval-fail-set-override.err.exp index beb29d678..71481683d 100644 --- a/tests/lang/eval-fail-set-override.err.exp +++ b/tests/lang/eval-fail-set-override.err.exp @@ -1,6 +1,4 @@ error: … while evaluating the `__overrides` attribute - at «none»:0: (source not available) - error: value is an integer while a set was expected diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/lang/eval-fail-substring.err.exp index dc26a00bd..5c58be29a 100644 --- a/tests/lang/eval-fail-substring.err.exp +++ b/tests/lang/eval-fail-substring.err.exp @@ -8,5 +8,3 @@ error: 2| error: negative start position in 'substring' - - at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/lang/eval-fail-to-path.err.exp index 43ed2bdfc..4ffa2cf6d 100644 --- a/tests/lang/eval-fail-to-path.err.exp +++ b/tests/lang/eval-fail-to-path.err.exp @@ -9,6 +9,4 @@ error: … while evaluating the first argument passed to builtins.toPath - at «none»:0: (source not available) - error: string 'foo/bar' doesn't represent an absolute path From f1aeeea32b113ac5ae9518221015a945625fa112 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 3 Sep 2023 14:00:55 +0200 Subject: [PATCH 017/402] traces: DRY printPosMaybe --- src/libutil/error.cc | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 3917fffc3..5b604c328 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -155,6 +155,25 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR return res; } +/** + * Print a position, if it is known. + * + * @return true if a position was printed. + */ +static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { + bool hasPos = pos && *pos; + if (hasPos) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; + + if (auto loc = pos->getCodeLines()) { + oss << "\n"; + printCodeLines(oss, "", *pos, *loc); + oss << "\n"; + } + } + return hasPos; +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { std::string prefix; @@ -318,32 +337,15 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << "\n" << "… " << trace.hint.str() << "\n"; - if (trace.pos && *trace.pos) { + if (printPosMaybe(oss, ellipsisIndent, trace.pos)) count++; - - oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; - - if (auto loc = trace.pos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *trace.pos, *loc); - oss << "\n"; - } - } } oss << "\n" << prefix; } oss << einfo.msg << "\n"; - if (einfo.errPos && *einfo.errPos) { - oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; - - if (auto loc = einfo.errPos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *einfo.errPos, *loc); - oss << "\n"; - } - } + printPosMaybe(oss, "", einfo.errPos); auto suggestions = einfo.suggestions.trim(); if (!suggestions.suggestions.empty()) { From 477bc617bba5617d1d1da3ce039b5bd296a3e56e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 3 Sep 2023 14:08:03 +0200 Subject: [PATCH 018/402] traces: Add _NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS --- src/libutil/error.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 5b604c328..dd9612471 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -155,6 +155,15 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR return res; } +/** + * A development aid for finding missing positions, to improve error messages. Example use: + * + * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * git diff -U20 tests + * + */ +static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value(); + /** * Print a position, if it is known. * @@ -170,6 +179,8 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std printCodeLines(oss, "", *pos, *loc); oss << "\n"; } + } else if (printUnknownLocations) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_RED << "UNKNOWN LOCATION" << ANSI_NORMAL << "\n"; } return hasPos; } From 7ff43435f9b23c9e0d445427cc37b147c84b056f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Sep 2023 18:15:32 -0400 Subject: [PATCH 019/402] Unit test some worker protocol serializers Continue with the characterization testing idioms begun in c70484454f52a3bba994ac0c155b4a6f35a5013c, but this time for unit tests. Co-authored-by: Andreas Rammhold --- Makefile | 1 + doc/manual/src/contributing/testing.md | 73 ++++++++- flake.nix | 1 + mk/programs.mk | 2 +- src/libstore/tests/worker-protocol.cc | 139 ++++++++++++++++++ src/libstore/worker-protocol-impl.hh | 16 ++ src/libstore/worker-protocol.hh | 4 + .../worker-protocol/content-address.bin | Bin 0 -> 208 bytes .../libstore/worker-protocol/derived-path.bin | Bin 0 -> 120 bytes .../libstore/worker-protocol/store-path.bin | Bin 0 -> 120 bytes .../libstore/worker-protocol/string.bin | Bin 0 -> 88 bytes 11 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 src/libstore/tests/worker-protocol.cc create mode 100644 unit-test-data/libstore/worker-protocol/content-address.bin create mode 100644 unit-test-data/libstore/worker-protocol/derived-path.bin create mode 100644 unit-test-data/libstore/worker-protocol/store-path.bin create mode 100644 unit-test-data/libstore/worker-protocol/string.bin diff --git a/Makefile b/Makefile index 31b54b93d..2eabeb077 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ makefiles = \ -include Makefile.config ifeq ($(tests), yes) +UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ src/libutil/tests/local.mk \ src/libstore/tests/local.mk \ diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index cd94d5cfb..329b34575 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -2,14 +2,70 @@ ## Unit-tests -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. +The unit tests are defined using the [googletest] and [rapidcheck] frameworks. + +[googletest]: https://google.github.io/googletest/ +[rapidcheck]: https://github.com/emil-e/rapidcheck + +### Source and header layout + +> An example of some files, demonstrating much of what is described below +> +> ``` +> src +> ├── libexpr +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ │ +> │ … +> └── tests +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ │ +> │ … +> │ +> ├── unit-test-data +> │ ├── libstore +> │ │ ├── worker-protocol/content-address.bin +> │ │ … +> │ … +> … +> ``` + +The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). + +The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. +The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. + +> **Note** +> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. +> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. +> That is why we have the test data nested within a single `unit-test-data` directory. + +### Running tests You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. +### Characterization testing + +See [below](#characterization-testing-1) for a broader discussion of characterization testing. + +Like with the functional characterization, `_NIX_TEST_ACCEPT=1` is also used. +For example: +```shell-session +$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +... +[ SKIPPED ] WorkerProtoTest.string_read +[ SKIPPED ] WorkerProtoTest.string_write +[ SKIPPED ] WorkerProtoTest.storePath_read +[ SKIPPED ] WorkerProtoTest.storePath_write +... +``` +will regenerate the "golden master" expected result for the `libnixstore` characterization tests. +The characterization tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. + ## Functional tests The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. @@ -124,9 +180,12 @@ This technique is to include the exact output/behavior of a former version of Ni For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. It is frequently useful to regenerate the expected output. -To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. -(At least, this is the convention we've used for `tests/lang.sh`. -If we add more characterization testing we should always strive to be consistent.) +To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. +For example: +```bash +_NIX_TEST_ACCEPT=1 make tests/lang.sh.test +``` +This convention is shared with the [characterization unit tests](#characterization-testing-1) too. An interesting situation to document is the case when these tests are "overfitted". The language tests are, again, an example of this. diff --git a/flake.nix b/flake.nix index cf7c1fa12..ec6ab48bd 100644 --- a/flake.nix +++ b/flake.nix @@ -75,6 +75,7 @@ ./precompiled-headers.h ./src ./tests + ./unit-test-data ./COPYING ./scripts/local.mk (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) diff --git a/mk/programs.mk b/mk/programs.mk index 1ee1d3fa5..a88d9d949 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$($(1)_PATH) + $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) endef diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc new file mode 100644 index 000000000..4a6ccf7c0 --- /dev/null +++ b/src/libstore/tests/worker-protocol.cc @@ -0,0 +1,139 @@ +#include + +#include +#include + +#include "worker-protocol.hh" +#include "worker-protocol-impl.hh" +#include "derived-path.hh" +#include "tests/libstore.hh" + +namespace nix { + +class WorkerProtoTest : public LibStoreTest +{ +public: + Path unitTestData = getEnv("_NIX_TEST_UNIT_DATA").value() + "/libstore/worker-protocol"; + + bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; + } + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem + ".bin"; + } + + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + WorkerProto::Serialise::read( + *store, + WorkerProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + WorkerProto::write( + *store, + WorkerProto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(WorkerProtoTest, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(WorkerProtoTest, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } + +CHARACTERIZATION_TEST( + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + derivedPath, + "derived-path", + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +} diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index d3d2792ff..4f797f95a 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -75,4 +75,20 @@ void WorkerProto::Serialise>::write(const Store & store, WorkerPr } } +template +std::tuple WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +{ + return std::tuple { + WorkerProto::Serialise::read(store, conn)..., + }; +} + +template +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (WorkerProto::Serialise::write(store, conn, args), ...); + }, res); +} + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ff762c924..70a5bddb9 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -28,6 +28,8 @@ class Store; struct Source; // items being serialised +class StorePath; +struct ContentAddress; struct DerivedPath; struct DrvOutput; struct Realisation; @@ -220,6 +222,8 @@ template MAKE_WORKER_PROTO(std::vector); template MAKE_WORKER_PROTO(std::set); +template +MAKE_WORKER_PROTO(std::tuple); template #define X_ std::map diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/unit-test-data/libstore/worker-protocol/content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..8f14bcdb3e50cc72d87106ce159a6fd7b9f75713 GIT binary patch literal 208 zcmY+;K@P$o5QSmy;|7WYrYjRqQfMm$+LWPzDW_MfG4bucKks(>Y#V56lkFOiEzi24 zUMdC5VUCa86OJ&gL0BF^B{HBQEY%f|I<6v7#q+l_PBirI5N~-md%37WE|g33QTE2k zc~-WF&R}7Oxc@o)T?ojpD!+*5=xpO;%IoCKSV5FMh={>xePK|8<${>E)*huNHZ(pX literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..bb1a81ac6d096cddf510a457e7506e6a81ee66a3 GIT binary patch literal 120 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*IT`wr5{vXwipmr#bSfDDBbg*z literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/unit-test-data/libstore/worker-protocol/store-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..3fc05f2981d3398ab30ec34125c903cdcefa8729 GIT binary patch literal 120 tcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F?srQlM;)-IsqP{BM|@q literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/unit-test-data/libstore/worker-protocol/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..aa7b5a604745b735473ec34283177e69320776e4 GIT binary patch literal 88 zcmZQzfB+^aoskJ)@Id+H8JQ)i3Pp)YNtq=eAx^0H( Date: Wed, 6 Sep 2023 09:14:56 +0200 Subject: [PATCH 020/402] add todo on store docs --- doc/manual/generate-manpage.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index bc4d2c5bc..0f91a9742 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -70,6 +70,7 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; + # TODO: move this confusing special case out of here when implementing #8496 maybeDocumentation = optionalString (details ? doc) (replaceStrings ["@stores@"] [storeDocs] details.doc); From b0fe7f560ddb31fea55546a79a89c44a63cd56ce Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:15:03 +0200 Subject: [PATCH 021/402] add missing link --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 0f91a9742..705fc0b69 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -148,7 +148,7 @@ let To use this store, you need to make sure the corresponding experimental feature, [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), is enabled. - For example, include the following in [`nix.conf`](#): + For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): ``` extra-experimental-features = ${experimentalFeature} From 09eb7f1ef653822616e7371c906ef1ae34b3ba09 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:30:32 +0200 Subject: [PATCH 022/402] do not show configuration override flags for each command this removes a lot of noise from the web search, which precludes finding the actual documentation. some configuration settings have enough documentation to warrant individual pages, so the alternative of including full setting documentation in each command page doesn't make much sense here. this change technically means that the command line flags to override settings are "invisible", and not exported as JSON. this may or may not be desirable. a more explicit approach would be adding a `hidden` field to the flag's JSON output, but would also require adjusting post-processing of that JSON for manual rendering. --- doc/manual/generate-manpage.nix | 4 ++++ src/libutil/args.cc | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 705fc0b69..87891fa7a 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -79,6 +79,10 @@ let # Options ${showOptions details.flags toplevel.flags} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; showOptions = options: commonOptions: diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 00281ce6f..8db293762 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -236,6 +236,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); + if (hiddenCategories.count(flag->category)) continue; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); From aa46f536e80ca20e0fc45664af867b47597a87f3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:40:51 +0200 Subject: [PATCH 023/402] add note on overriding settings for stable commands --- doc/manual/src/command-ref/opt-common.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 0647228c2..114b292f9 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,3 +203,7 @@ Most Nix commands accept the following command-line options: Fix corrupted or missing store paths by redownloading or rebuilding them. Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build. Also note the warning under `nix-store --repair-path`. + +> **Note** +> +> See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. From 82ddb130984c7bdc45cdffc14e81bed720089200 Mon Sep 17 00:00:00 2001 From: Graham Bennett Date: Fri, 29 Apr 2022 15:15:25 -0400 Subject: [PATCH 024/402] Unlock output paths when a derivation is already built Without this change, nix build processes will not drop the locks for derivation goals which have already been built by another process when the current process gets round to building them. This means the locks are held until the process terminates. If there are other nix build processes in a similar state, they will also try to acquire the same locks when they try to build the same derivation, and so will wait until the lock holder terminates (which might be a very long time if it has a lot to build). In some pathological cases, those processes might be holding their own locks on other derivations due to the same issue, and this can lead to deadlock. Resolves #6468 --- src/libstore/build/derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6472ecd99..befbfd10e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1456,6 +1456,7 @@ void DerivationGoal::done( SingleDrvOutputs builtOutputs, std::optional ex) { + outputLocks.unlock(); buildResult.status = status; if (ex) buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); From 3720e811fa2883bca1a2698543146070e4d8d32c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Sep 2023 13:21:55 +0200 Subject: [PATCH 025/402] libexpr: Add nrExprs to NIX_SHOW_STATS --- src/libexpr/eval.cc | 1 + src/libexpr/nixexpr.hh | 4 ++++ src/libexpr/parser.y | 1 + 3 files changed, 6 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6d445fd96..afa864730 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2506,6 +2506,7 @@ void EvalState::printStats() {"elements", nrValuesInEnvs}, {"bytes", bEnvs}, }; + topObj["nrExprs"] = Expr::nrExprs; topObj["list"] = { {"elements", nrListElems}, {"bytes", bLists}, diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5ca3d1fa6..b80c07f46 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -155,6 +155,10 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { + static unsigned long nrExprs; + inline Expr() { + nrExprs++; + } virtual ~Expr() { }; virtual void show(const SymbolTable & symbols, std::ostream & str) const; virtual void bindVars(EvalState & es, const std::shared_ptr & env); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde..a3d3c7041 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -653,6 +653,7 @@ formal namespace nix { +unsigned long Expr::nrExprs = 0; Expr * EvalState::parse( char * text, From bf8deb4991286a44e1b6cc2142721a594276dd12 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Sep 2023 13:45:45 +0200 Subject: [PATCH 026/402] Expr: remove redundant int and float fields --- src/libexpr/nixexpr.cc | 4 ++-- src/libexpr/nixexpr.hh | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 4566a1388..22be8e68c 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -76,12 +76,12 @@ void Expr::show(const SymbolTable & symbols, std::ostream & str) const void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const { - str << n; + str << v.integer; } void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const { - str << nf; + str << v.fpoint; } void ExprString::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index b80c07f46..17b2d1136 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -175,18 +175,16 @@ struct Expr struct ExprInt : Expr { - NixInt n; Value v; - ExprInt(NixInt n) : n(n) { v.mkInt(n); }; + ExprInt(NixInt n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; struct ExprFloat : Expr { - NixFloat nf; Value v; - ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; + ExprFloat(NixFloat nf) { v.mkFloat(nf); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; From 408055a9ddf855e32a59e07e564ad893b2cb2cdf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 18 Sep 2023 20:08:48 +0200 Subject: [PATCH 027/402] add clarifying example to `nix-env` output selection there is a very confusing warning in the Nixpkgs manual that mischaracterises `nix-env` behavior, and this example shows what's really happening. note that it doesn't use `pkgs.runCommand` or other `pkgs.stdenv` facilities, as deep down those set `meta.outputsToInstall` to very particular defaults that do not generally apply to Nix. --- doc/manual/src/command-ref/nix-env/install.md | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index 5dc04c385..95648e9e0 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -19,12 +19,15 @@ current generation of the active profile, to which a set of store paths described by *args* is added. The arguments *args* map to store paths in a number of possible ways: - - By default, *args* is a set of derivation names denoting derivations + + - By default, *args* is a set of [derivation] names denoting derivations in the active Nix expression. These are realised, and the resulting output paths are installed. Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. + [derivation]: @docroot@/language/derivations.md + If there are multiple derivations matching a name in *args* that have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the derivation with the highest *priority* is used. A derivation can @@ -66,8 +69,59 @@ a number of possible ways: - If *args* are store paths that are not store derivations, then these are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. - - By default all outputs are installed for each derivation. That can - be reduced by setting `meta.outputsToInstall`. + - By default all outputs are installed for each derivation. + This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. + + + + Example: + + The file `example.nix` defines a [derivation] with two outputs `foo` and `bar`, each containing a file. + + ```nix + # example.nix + let + pkgs = import {}; + command = '' + ${pkgs.coreutils}/bin/mkdir -p $foo $bar + echo foo > $foo/foo-file + echo bar > $bar/bar-file + ''; + in + derivation { + name = "example"; + builder = "${pkgs.bash}/bin/bash"; + args = [ "-c" command ]; + outputs = [ "foo" "bar" ]; + system = builtins.currentSystem; + } + ``` + + Installing from this Nix expression will make files from both outputs appear in the current profile. + + ```console + $ nix-env --install --file example.nix + installing 'example' + $ ls ~/.nix-profile + foo-file + bar-file + manifest.nix + ``` + + Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. + + ```nix + # example-outputs.nix + import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } + ``` + + ```console + $ nix-env --install --file example-outputs.nix + installing 'example' + $ ls ~/.nix-profile + bar-file + manifest.nix + ``` # Flags From e44d2a6bbef113b3d8f162f75bd5c94e0101075f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 22:57:59 -0400 Subject: [PATCH 028/402] Add FreeBSD and NetBSD cross to Nix's flake --- flake.nix | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index dfb2dc1f8..871f0d7ba 100644 --- a/flake.nix +++ b/flake.nix @@ -25,8 +25,11 @@ linuxSystems = linux32BitSystems ++ linux64BitSystems; darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ]; systems = linuxSystems ++ darwinSystems; - - crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + + crossSystems = [ + "armv6l-linux" "armv7l-linux" + "x86_64-freebsd13" "x86_64-netbsd" + ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -94,7 +97,14 @@ nixpkgsFor = forAllSystems (system: let make-pkgs = crossSystem: stdenv: import nixpkgs { - inherit system crossSystem; + localSystem = { + inherit system; + }; + crossSystem = if crossSystem == null then null else { + system = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") { + useLLVM = true; + }; overlays = [ (overlayFor (p: p.${stdenv})) ]; From 28850ee90095e337bf47fbe03a0d937cb98784e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:27:53 -0400 Subject: [PATCH 029/402] Make dev shells work for cross Need to get tools from right package set. Could build clang tools but I don't want to wait :D. --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 871f0d7ba..24d9520ea 100644 --- a/flake.nix +++ b/flake.nix @@ -750,7 +750,11 @@ outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps - ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]); + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + ++ lib.optional + (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) + pkgs.buildPackages.clang-tools + ; buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; From 564392b57bc44c407303e4eaf6138d61a8ea50b0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:29:17 -0400 Subject: [PATCH 030/402] Make libsodium an unconditional dependency The configure script will not tolerate it being missing. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 24d9520ea..50a226347 100644 --- a/flake.nix +++ b/flake.nix @@ -190,9 +190,9 @@ libarchive boost lowdown-nix + libsodium ] ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; checkDeps = [ From 0db251e4ad07338375fd59134fb467e9ebc4176a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:43:48 -0400 Subject: [PATCH 031/402] Do not build docs in cross devShell Coppied from the main build; we really should deduplicate this more. --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 50a226347..36937057d 100644 --- a/flake.nix +++ b/flake.nix @@ -743,6 +743,9 @@ devShells = let makeShell = pkgs: stdenv: + let + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + in with commonDeps { inherit pkgs; }; stdenv.mkDerivation { name = "nix"; @@ -760,7 +763,8 @@ ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags; + ++ testConfigureFlags ++ internalApiDocsConfigureFlags + ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; From 7f76d7f038fbb5cb7982cf9aa951b9730566e275 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:44:23 -0400 Subject: [PATCH 032/402] Rename an identifier of ours called `stdout` This is a reserved identifier on NetBSD --- it is replaced by a macro on that platform --- and so we cannot use it. --- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 56f47a4ac..9c2ad039a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -379,9 +379,9 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); + std_out = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); + throw SysError("dupping standard output"); } @@ -390,7 +390,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - dup2(stdout, STDOUT_FILENO); + dup2(std_out, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 7a9e83c6c..9415be78a 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -85,8 +85,9 @@ struct LegacyArgs : public MixCommonArgs void showManPage(const std::string & name); /** - * The constructor of this class starts a pager if stdout is a - * terminal and $PAGER is set. Stdout is redirected to the pager. + * The constructor of this class starts a pager if standard output is a + * terminal and $PAGER is set. Standard output is redirected to the + * pager. */ class RunPager { @@ -96,7 +97,7 @@ public: private: Pid pid; - int stdout; + int std_out; }; extern volatile ::sig_atomic_t blockInt; From c18911602eb4260d59acf8c17f1c3b4c7fcf7cee Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 31 Aug 2023 00:07:49 -0400 Subject: [PATCH 033/402] Fix `boehmgc-coroutine-sp-fallback.diff` for FreeBSD Our FreeBSD headers have `pthread_getattr_np`, but we get a link-time error that is missing. The good news is that there is another similar function which does exist, and the upstream project elsewhere does just the [fallback code] we need. As the fallback code indicates, the two functions are not identical however as the other one needs explicit initialization. NetBSD supports both in fact, and its [manpage] is therefore a good resource on what the differences are. [fallback code]: https://github.com/ivmai/bdwgc/blob/07a6d0ee8889bca5eaeadc13cabadc363725d216/os_dep.c#L1266-L1272 [manpage]: https://man.netbsd.org/pthread_attr_get_np.3 --- boehmgc-coroutine-sp-fallback.diff | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 5066d8278..2afbe9671 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -59,12 +59,18 @@ index b5d71e62..aed7b0bf 100644 GC_bool found_me = FALSE; size_t nthreads = 0; int i; -@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void) +@@ -851,6 +853,37 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ + } else { -+ if (pthread_getattr_np(p->id, &pattr)) { ++#ifdef HAVE_PTHREAD_ATTR_GET_NP ++ if (!pthread_attr_init(&pattr) ++ || !pthread_attr_get_np(p->id, &pattr)) ++#else /* HAVE_PTHREAD_GETATTR_NP */ ++ if (pthread_getattr_np(p->id, &pattr)) ++#endif ++ { + ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); + } + if (pthread_attr_getstacksize(&pattr, &stack_limit)) { From b7acef1ceb756c72462a896d90c1b17ddc32a6df Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Sep 2023 15:21:21 +0200 Subject: [PATCH 034/402] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index cf8690732..ef0f38abe 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.18.0 +2.19.0 From 883092e3f78d4efb1066a2e24e343b307035a04c Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Wed, 20 Sep 2023 09:09:01 -0700 Subject: [PATCH 035/402] Re-enable systemd-nspawn test It was disabled in c6953d1ff62fb6dc4fbd89c03e7949c552c19382 because a recent Nixpkgs bump brought in a new systemd which changed how systemd-nspawn worked. As far as I can tell, the issue was caused by this upstream systemd commit: https://github.com/systemd/systemd/commit/b71a0192c040f585397cfc6fc2ca025bf839733d Bind-mounting the host's `/sys` and `/proc` into the container's `/run/host/{sys,proc}` fixes the issue and allows the test to succeed. --- tests/nixos/containers/containers.nix | 4 ++-- tests/nixos/containers/systemd-nspawn.nix | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index e721be48f..c8ee78a4a 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -56,8 +56,8 @@ host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") # Run systemd-nspawn in a Nix build. - #host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") - #host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; } diff --git a/tests/nixos/containers/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix index f54f32f2a..1dad4ebd7 100644 --- a/tests/nixos/containers/systemd-nspawn.nix +++ b/tests/nixos/containers/systemd-nspawn.nix @@ -73,6 +73,8 @@ runCommand "test" --resolv-conf=off \ --bind-ro=/nix/store \ --bind=$out \ + --bind=/proc:/run/host/proc \ + --bind=/sys:/run/host/sys \ --private-network \ $toplevel/init '' From e0e5943db28271cd6f1dcf56f3a0b12d0bc37b6c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:51:39 +0200 Subject: [PATCH 036/402] remove IRC from links in README the community has moved away from IRC a long time ago --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 85b0902b1..46a4218d9 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ to set up a development environment and build Nix from source. - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [NixOS Discourse](https://discourse.nixos.org/) - [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) -- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) ## License From 747b2baf21f5bc3924945b02d10cd37b17e2e1bb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:51:54 +0200 Subject: [PATCH 037/402] fix rendering error for consecutive spaces --- doc/manual/src/language/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 29950a52d..a26e43a05 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -83,7 +83,8 @@ This is an incomplete overview of language features, by example. - A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n string"`. + + A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`. From 954890a42f143896d5892066f7b5a0f2b120fbfe Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:52:05 +0200 Subject: [PATCH 038/402] add information on release cycle and backports --- doc/manual/src/release-notes/release-notes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md index b05d5ee0a..dbda0babd 100644 --- a/doc/manual/src/release-notes/release-notes.md +++ b/doc/manual/src/release-notes/release-notes.md @@ -1 +1,7 @@ # Nix Release Notes + +Nix has a release cycle of roughly 6 weeks. +Notable changes and additions are announced in the release notes for each version. + +Bugfixes can be backported on request to the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). + From 02649d247b7336697f059186ea8e9f3076dcda0a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:52:28 +0200 Subject: [PATCH 039/402] move test coverage section to testing page --- doc/manual/src/contributing/hacking.md | 14 -------------- doc/manual/src/contributing/testing.md | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 4b0a3a3e5..22f629c6f 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -271,17 +271,3 @@ or inside a `nix develop` shell by running: # make internal-api-html # xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html ``` - -## Coverage analysis - -A coverage analysis report is [available -online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You -can build it yourself: - -``` -# nix build .#hydraJobs.coverage -# xdg-open ./result/coverage/index.html -``` - -Metrics about the change in line/function coverage over time are also -[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts). diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index cd94d5cfb..881118505 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -1,5 +1,19 @@ # Running tests +## Coverage analysis + +A [coverage analysis report] is available online +You can build it yourself: + +[coverage analysis report]: https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage + +``` +# nix build .#hydraJobs.coverage +# xdg-open ./result/coverage/index.html +``` + +[Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. + ## Unit-tests The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined From 4685260a77266382b5c2c2b68ff1e6e0ddc8a908 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:52:58 +0200 Subject: [PATCH 040/402] fix links to configuration settings --- src/libutil/experimental-features.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 782331283..211374bf6 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -171,7 +171,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "auto-allocate-uids", .description = R"( Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. + `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details. )", }, { @@ -179,7 +179,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "cgroups", .description = R"( Allows Nix to execute builds inside cgroups. See - the [`use-cgroups`](#conf-use-cgroups) setting for details. + the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details. )", }, { From 8e25450ff4bbf2b2c6951ebc78fd1bdec7097660 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:53:11 +0200 Subject: [PATCH 041/402] refer to nix.dev for installation instructions there are currently multiple places with installation instructions that all have to be updated when a change to any of them is accepted. this reduces the number of places by one, and directs beginners to the maintained and curated resource for Nix learning materials. --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 85b0902b1..1440239bc 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,11 @@ Nix is a powerful package manager for Linux and other Unix systems that makes pa management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) for more details. -## Installation +## Installation and first steps -On Linux and macOS the easiest way to install Nix is to run the following shell command -(as a user other than root): +Visit [nix.dev](https://nix.dev) for installation instructions and beginner tutorials. -```console -$ curl -L https://nixos.org/nix/install | sh -``` - -Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). +Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). ## Building And Developing From 984bd4cb0e2973ad756b6b1692919980f14f5df2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:53:36 +0200 Subject: [PATCH 042/402] README: link to CONTRIBUTING --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 85b0902b1..3b31b4d93 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ Information on additional installation methods is available on the [Nix download See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to to set up a development environment and build Nix from source. +## Contributing + +Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. + ## Additional Resources - [Nix manual](https://nixos.org/nix/manual) From cf6ba7256f849ec1fe209b18c311e61ecd7719d3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:57:00 +0200 Subject: [PATCH 043/402] delete unused files --- doc/manual/src/command-ref/opt-common-syn.md | 57 -------------------- doc/manual/src/command-ref/opt-inst-syn.md | 15 ------ 2 files changed, 72 deletions(-) delete mode 100644 doc/manual/src/command-ref/opt-common-syn.md delete mode 100644 doc/manual/src/command-ref/opt-inst-syn.md diff --git a/doc/manual/src/command-ref/opt-common-syn.md b/doc/manual/src/command-ref/opt-common-syn.md deleted file mode 100644 index b66d318c2..000000000 --- a/doc/manual/src/command-ref/opt-common-syn.md +++ /dev/null @@ -1,57 +0,0 @@ -\--help - -\--version - -\--verbose - -\-v - -\--quiet - -\--log-format - -format - -\--no-build-output - -\-Q - -\--max-jobs - -\-j - -number - -\--cores - -number - -\--max-silent-time - -number - -\--timeout - -number - -\--keep-going - -\-k - -\--keep-failed - -\-K - -\--fallback - -\--readonly-mode - -\-I - -path - -\--option - -name - -value diff --git a/doc/manual/src/command-ref/opt-inst-syn.md b/doc/manual/src/command-ref/opt-inst-syn.md deleted file mode 100644 index 1703c40e3..000000000 --- a/doc/manual/src/command-ref/opt-inst-syn.md +++ /dev/null @@ -1,15 +0,0 @@ -\--prebuilt-only - -\-b - -\--attr - -\-A - -\--from-expression - -\-E - -\--from-profile - -path From 1a412a8d782b11b3af08be5b7ee3b4000b72cb32 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 11:38:13 +0200 Subject: [PATCH 044/402] fix typo in docstring --- src/libutil/config.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index cc8532587..96c2cd75d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -36,8 +36,8 @@ namespace nix { * * std::map settings; * config.getSettings(settings); - * config["system"].description == "the current system" - * config["system"].value == "x86_64-linux" + * settings["system"].description == "the current system" + * settings["system"].value == "x86_64-linux" * * * The above retrieves all currently known settings from the `Config` object From 1b560ea502cf0308b829e4b3425307f80fc3528a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 23:23:14 +0200 Subject: [PATCH 045/402] more detail on backports Co-authored-by: John Ericson --- doc/manual/src/release-notes/release-notes.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md index dbda0babd..1fa2f5580 100644 --- a/doc/manual/src/release-notes/release-notes.md +++ b/doc/manual/src/release-notes/release-notes.md @@ -3,5 +3,9 @@ Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes can be backported on request to previous Nix releases. +We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). + +Backports never skip releases (if a feature is backported to `x.y`, it must also be available in `x.(y+1)`; +This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. From f264d9ff0862277523a207c7afaed4894a40dc11 Mon Sep 17 00:00:00 2001 From: Gerg-L Date: Thu, 21 Sep 2023 21:00:35 -0400 Subject: [PATCH 046/402] flake: complete update to 23.05 --- flake.lock | 12 ++++++------ flake.nix | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index 75b6ae6a7..56df9c3fb 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695124524, - "narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=", - "owner": "edolstra", + "lastModified": 1695283060, + "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "a3d30b525535e3158221abc1a957ce798ab159fe", + "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4", "type": "github" }, "original": { - "owner": "edolstra", - "ref": "fix-aws-sdk-cpp", + "owner": "NixOS", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index dfb2dc1f8..a7680a759 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,7 @@ { description = "The purely functional package manager"; - #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; - inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From d8cebae939932f2a5b458b5d92c82cd1a025dc0a Mon Sep 17 00:00:00 2001 From: Rasmus Rendal Date: Sat, 4 Jun 2022 12:07:08 +0200 Subject: [PATCH 047/402] Add a test for flake paths with spaces in them --- tests/flakes/common.sh | 10 +- tests/flakes/flakes.sh | 302 +++++++++++++++++++++-------------------- 2 files changed, 157 insertions(+), 155 deletions(-) diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index 427abcdde..8aed296e6 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -61,10 +61,10 @@ createGitRepo() { local repo="$1" local extraArgs="${2-}" - rm -rf $repo $repo.tmp - mkdir -p $repo + rm -rf "$repo" "$repo".tmp + mkdir -p "$repo" - git -C $repo init $extraArgs - git -C $repo config user.email "foobar@example.com" - git -C $repo config user.name "Foobar" + git -C "$repo" init $extraArgs + git -C "$repo" config user.email "foobar@example.com" + git -C "$repo" config user.name "Foobar" } diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index 128f759ea..70de28628 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -6,27 +6,29 @@ clearStore rm -rf $TEST_HOME/.cache $TEST_HOME/.config flake1Dir=$TEST_ROOT/flake1 -flake2Dir=$TEST_ROOT/flake2 -flake3Dir=$TEST_ROOT/flake3 +flake2Dir=$TEST_ROOT/flake\ 2 +percentEncodedFlake2Dir=$TEST_ROOT/flake%202 +flake3Dir=$TEST_ROOT/flake%20 +percentEncodedFlake3Dir=$TEST_ROOT/flake%2520 flake5Dir=$TEST_ROOT/flake5 flake7Dir=$TEST_ROOT/flake7 nonFlakeDir=$TEST_ROOT/nonFlake badFlakeDir=$TEST_ROOT/badFlake flakeGitBare=$TEST_ROOT/flakeGitBare -for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do +for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir"; do # Give one repo a non-main initial branch. extraArgs= - if [[ $repo == $flake2Dir ]]; then + if [[ "$repo" == "$flake2Dir" ]]; then extraArgs="--initial-branch=main" fi createGitRepo "$repo" "$extraArgs" done -createSimpleGitFlake $flake1Dir +createSimpleGitFlake "$flake1Dir" -cat > $flake2Dir/flake.nix < "$flake2Dir/flake.nix" < $flake2Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/default.nix < "$flake3Dir/default.nix" < $nonFlakeDir/README.md < "$nonFlakeDir/README.md" < $flake1Dir/foo -git -C $flake1Dir add $flake1Dir/foo +echo foo > "$flake1Dir/foo" +git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] -echo -n '# foo' >> $flake1Dir/flake.nix -flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) -git -C $flake1Dir commit -a -m 'Foo' -flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) +echo -n '# foo' >> "$flake1Dir/flake.nix" +flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) +git -C "$flake1Dir" commit -a -m 'Foo' +flake1NewCommit=$(git -C "$flake1Dir" rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] [[ $hash1 != $hash2 ]] # Test 'nix build' on a flake. -nix build -o $TEST_ROOT/result flake1#foo -[[ -e $TEST_ROOT/result/hello ]] +nix build -o "$TEST_ROOT/result" flake1#foo +[[ -e "$TEST_ROOT/result/hello" ]] # Test packages.default. -nix build -o $TEST_ROOT/result flake1 -[[ -e $TEST_ROOT/result/hello ]] +nix build -o "$TEST_ROOT/result" flake1 +[[ -e "$TEST_ROOT/result/hello" ]] -nix build -o $TEST_ROOT/result $flake1Dir -nix build -o $TEST_ROOT/result git+file://$flake1Dir +nix build -o "$TEST_ROOT/result" "$flake1Dir" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" # Check that store symlinks inside a flake are not interpreted as flakes. -nix build -o $flake1Dir/result git+file://$flake1Dir -nix path-info $flake1Dir/result +nix build -o "$flake1Dir/result" "git+file://$flake1Dir" +nix path-info "$flake1Dir/result" # 'getFlake' on an unlocked flakeref should fail in pure mode, but # succeed in impure mode. -(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure +(! nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure # 'getFlake' on a locked flakeref should succeed even in pure mode. -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" # Building a flake with an unlocked dependency should fail in pure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) -(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-use-registries) (! nix eval --expr "builtins.getFlake \"$flake2Dir\"") # But should succeed in impure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --impure) -nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file +(! nix build -o "$TEST_ROOT/result" flake2#bar --impure) +nix build -o "$TEST_ROOT/result" flake2#bar --impure --no-write-lock-file nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure # Building a local flake with an unlocked dependency should fail with --no-update-lock-file. -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' # But it should succeed without that flag. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' -nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file -[[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-write-lock-file +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file +[[ -e "$flake2Dir/flake.lock" ]] +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Building with a lockfile should not require a fetch of the registry. -nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh +nix build -o "$TEST_ROOT/result" --flake-registry file:///no-registry.json "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-registries "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "$flake2Dir#bar" --refresh # Updating the flake should not change the lockfile. -nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix flake lock "$flake2Dir" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Now we should be able to build the flake in pure mode. -nix build -o $TEST_ROOT/result flake2#bar +nix build -o "$TEST_ROOT/result" flake2#bar # Or without a registry. -nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh +nix build -o "$TEST_ROOT/result" --no-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh # Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result $flake3Dir#xyzzy -git -C $flake3Dir add flake.lock +nix build -o "$TEST_ROOT/result" "$flake3Dir#xyzzy" +git -C "$flake3Dir" add flake.lock # Add dependency to flake3. -rm $flake3Dir/flake.nix +rm "$flake3Dir/flake.nix" -cat > $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix <' -A x) = 123 ]] [[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] # Test alternate lockfile paths. -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock -cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2.lock +cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit -expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock -nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2-overridden.lock +nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit # reference-lock-file can only be used if allow-dirty is set. -expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock +expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock From 50e61f579cfc8cf64031845f3e73f82593ad2ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 31 May 2023 10:36:43 +0200 Subject: [PATCH 048/402] Allow special characters in flake paths Support using nix flakes in paths with spaces or abitrary unicode characters. This introduces the convention that the path part of the URL should be percent-encoded when dealing with `path:` urls and not when using filepaths (following the convention of firefox). Co-authored-by: Rendal --- src/libexpr/flake/flakeref.cc | 85 +++++++++++++++++++---------------- src/libutil/tests/url.cc | 9 ++++ src/libutil/url.cc | 2 +- 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index e1bce90bc..2155c5f12 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -77,14 +77,6 @@ std::pair parseFlakeRefWithFragment( { using namespace fetchers; - static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; - - static std::regex pathUrlRegex( - "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" - + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", - std::regex::ECMAScript); - static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + "(?:#(" + queryRegex + "))?", @@ -92,26 +84,23 @@ std::pair parseFlakeRefWithFragment( std::smatch match; - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - - if (std::regex_match(url, match, flakeRegex)) { - auto parsedURL = ParsedURL{ - .url = url, - .base = "flake:" + match.str(1), - .scheme = "flake", - .authority = "", - .path = match[1], - }; - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL, isFlake), ""), - percentDecode(match.str(6))); - } - - else if (std::regex_match(url, match, pathUrlRegex)) { - std::string path = match[1]; - std::string fragment = percentDecode(match.str(3)); + auto parsePathFlakeRef = [&]() { + std::string path = url; + std::string fragment = ""; + std::map query = {}; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') + fragmentStart = url.find("#"); + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (fragmentStart != std::string::npos && pathEnd != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + } if (baseDir) { /* Check if 'url' is a path (either absolute or relative @@ -163,7 +152,7 @@ std::pair parseFlakeRefWithFragment( .scheme = "git+file", .authority = "", .path = flakeRoot, - .query = decodeQuery(match[2]), + .query = query, }; if (subdir != "") { @@ -188,7 +177,6 @@ std::pair parseFlakeRefWithFragment( } else { if (!hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); - auto query = decodeQuery(match[2]); path = canonPath(path + "/" + getOr(query, "dir", "")); } @@ -197,19 +185,40 @@ std::pair parseFlakeRefWithFragment( attrs.insert_or_assign("path", path); return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); + }; + + /* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ + + if (std::regex_match(url, match, flakeRegex)) { + auto parsedURL = ParsedURL{ + .url = url, + .base = "flake:" + match.str(1), + .scheme = "flake", + .authority = "", + .path = match[1], + }; + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), ""), + percentDecode(match.str(6))); } else { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); + try { + auto parsedURL = parseURL(url); + std::string fragment; + std::swap(fragment, parsedURL.fragment); - auto input = Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; + auto input = Input::fromURL(parsedURL, isFlake); + input.parent = baseDir; - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); + } catch (BadURL &) { + return parsePathFlakeRef(); + } } } diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index a908631e6..a678dad20 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -335,4 +335,13 @@ namespace nix { ASSERT_EQ(d, s); } + TEST(percentEncode, yen) { + // https://en.wikipedia.org/wiki/Percent-encoding#Character_data + std::string s = reinterpret_cast(u8"円"); + std::string e = "%E5%86%86"; + + ASSERT_EQ(percentEncode(s), e); + ASSERT_EQ(percentDecode(e), s); + } + } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index a8f7d39fd..e700c8eaf 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -103,7 +103,7 @@ std::string percentEncode(std::string_view s, std::string_view keep) || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02X", (unsigned int) c); + res += fmt("%%%02X", c & 0xFF); return res; } From e8113747e1abf7fa6225f7fcd31ca9e71e03cfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 20 Apr 2023 06:51:47 +0200 Subject: [PATCH 049/402] Split the parseFlakeRefWithFragment function Was starting to be very complex and hard to follow. Now the different cases should be easier to understand. --- src/libexpr/flake/flakeref.cc | 279 +++++++++++++++++++--------------- 1 file changed, 155 insertions(+), 124 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 2155c5f12..f7ef19b3f 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -69,127 +69,130 @@ std::optional maybeParseFlakeRef( } } -std::pair parseFlakeRefWithFragment( +std::pair parsePathFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { - using namespace fetchers; + std::string path = url; + std::string fragment = ""; + std::map query; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') { + fragmentStart = url.find("#"); + } + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + } + + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir); + + if (isFlake) { + + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); + + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); + } + if (!found) + throw BadURL("could not find a flake.nix file"); + } + + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = query, + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + path = canonPath(path + "/" + getOr(query, "dir", "")); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); +}; + + +/* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ +std::optional> parseFlakeIdRef( + const std::string & url, + bool isFlake +) +{ + std::smatch match; static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + "(?:#(" + queryRegex + "))?", std::regex::ECMAScript); - std::smatch match; - - auto parsePathFlakeRef = [&]() { - std::string path = url; - std::string fragment = ""; - std::map query = {}; - auto pathEnd = url.find_first_of("#?"); - auto fragmentStart = pathEnd; - if (pathEnd != std::string::npos && url[pathEnd] == '?') - fragmentStart = url.find("#"); - if (pathEnd != std::string::npos) { - path = url.substr(0, pathEnd); - } - if (fragmentStart != std::string::npos) { - fragment = percentDecode(url.substr(fragmentStart+1)); - } - if (fragmentStart != std::string::npos && pathEnd != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); - } - - if (baseDir) { - /* Check if 'url' is a path (either absolute or relative - to 'baseDir'). If so, search upward to the root of the - repo (i.e. the directory containing .git). */ - - path = absPath(path, baseDir); - - if (isFlake) { - - if (!allowMissing && !pathExists(path + "/flake.nix")){ - notice("path '%s' does not contain a 'flake.nix', searching up",path); - - // Save device to detect filesystem boundary - dev_t device = lstat(path).st_dev; - bool found = false; - while (path != "/") { - if (pathExists(path + "/flake.nix")) { - found = true; - break; - } else if (pathExists(path + "/.git")) - throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); - else { - if (lstat(path).st_dev != device) - throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); - } - path = dirOf(path); - } - if (!found) - throw BadURL("could not find a flake.nix file"); - } - - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); - - auto flakeRoot = path; - std::string subdir; - - while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, - .scheme = "git+file", - .authority = "", - .path = flakeRoot, - .query = query, - }; - - if (subdir != "") { - if (parsedURL.query.count("dir")) - throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); - parsedURL.query.insert_or_assign("dir", subdir); - } - - if (pathExists(flakeRoot + "/.git/shallow")) - parsedURL.query.insert_or_assign("shallow", "1"); - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")), - fragment); - } - - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); - } - } - - } else { - if (!hasPrefix(path, "/")) - throw BadURL("flake reference '%s' is not an absolute path", url); - path = canonPath(path + "/" + getOr(query, "dir", "")); - } - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - - return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); - }; - - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ .url = url, @@ -200,25 +203,53 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), + FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } - else { - try { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); + return {}; +} - auto input = Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; +std::optional> parseURLFlakeRef( + const std::string & url, + const std::optional & baseDir, + bool isFlake +) +{ + ParsedURL parsedURL; + try { + parsedURL = parseURL(url); + } catch (BadURL &) { + return std::nullopt; + } - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); - } catch (BadURL &) { - return parsePathFlakeRef(); - } + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + auto input = fetchers::Input::fromURL(parsedURL, isFlake); + input.parent = baseDir; + + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); +} + +std::pair parseFlakeRefWithFragment( + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + using namespace fetchers; + + std::smatch match; + + if (auto res = parseFlakeIdRef(url, isFlake)) { + return *res; + } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + return *res; + } else { + return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); } } From 34115076969f20be935ae8eaf579928ac07981e3 Mon Sep 17 00:00:00 2001 From: Rasmus Rendal Date: Tue, 14 Jun 2022 17:38:48 +0200 Subject: [PATCH 050/402] Document the percent-encoding mechanism --- doc/manual/src/release-notes/rl-next.md | 4 ++++ src/nix/flake.md | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..2467e7ccc 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,5 @@ # Release X.Y (202?-??-??) + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/flake.md b/src/nix/flake.md index 92f477917..1c512b315 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -67,6 +67,13 @@ inputs.nixpkgs = { }; ``` +Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), +characters outside of the allowed range (i.e. are neither [reserved +character](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) or +[unreserved +characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) must be +percent-encoded. + ### Examples Here are some examples of flake references in their URL-like representation: @@ -103,10 +110,15 @@ The semantic of such a path is as follows: 2. The filesystem root (/), or 3. A folder on a different mount point. +Contrary to URL-like reference, path-like flake references can contain +arbitrary unicode characters (except `#` and `?`). + ### Examples * `.`: The flake to which the current directory belongs to. * `/home/alice/src/patchelf`: A flake in some other directory. +* `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that + has Unicode characters in its name. ## Flake reference attributes From 39ba81a4eb7404f9a12cf0768da51c0e0e26b588 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:52:57 -0400 Subject: [PATCH 051/402] Improve internal API docs for two file hashing functions Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/hash.hh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4..c3aa5cd81 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -145,13 +145,17 @@ std::string printHash16or32(const Hash & hash); Hash hashString(HashType ht, std::string_view s); /** - * Compute the hash of the given file. + * Compute the hash of the given file, hashing its contents directly. + * + * (Metadata, such as the executable permission bit, is ignored.) */ Hash hashFile(HashType ht, const Path & path); /** - * Compute the hash of the given path. The hash is defined as - * (essentially) hashString(ht, dumpPath(path)). + * Compute the hash of the given path, serializing as a Nix Archive and + * then hashing that. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, From 9d6114313b1b06a020bed230cfe7f05e6075b875 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:31:19 -0400 Subject: [PATCH 052/402] Move `ParseSink` to its own header We will soon add a new implemenation so the one for NARs in `archive.cc` isn't the only one. Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/archive.hh | 17 +---------------- src/libutil/fs-sink.hh | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 src/libutil/fs-sink.hh diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 2cf164a41..3530783c1 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "fs-sink.hh" namespace nix { @@ -72,22 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * \todo Fix this API, it sucks. - */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; -}; - /** * If the NAR archive contains a single file at top-level, then save * the contents of the file to `s`. Otherwise barf. diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 000000000..ed6484e61 --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,25 @@ +#pragma once +///@file + +#include "types.hh" +#include "serialise.hh" + +namespace nix { + +/** + * \todo Fix this API, it sucks. + */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void closeRegularFile() { }; + virtual void isExecutable() { }; + virtual void preallocateContents(uint64_t size) { }; + virtual void receiveContents(std::string_view data) { }; + + virtual void createSymlink(const Path & path, const std::string & target) { }; +}; + +} From 8a416e819c9727c2e9efd70f1d2a4b0bfda11663 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:49:01 -0400 Subject: [PATCH 053/402] Move `RestoreSink` to `fs-sink.cc` Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/archive.cc | 67 ---------------------------------- src/libutil/fs-sink.cc | 82 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 67 deletions(-) create mode 100644 src/libutil/fs-sink.cc diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 268a798d9..0f0cae7c0 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -27,8 +27,6 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, false, "preallocate-contents", - "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; @@ -302,71 +300,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - void restorePath(const Path & path, Source & source) { RestoreSink sink; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc new file mode 100644 index 000000000..fcffd3216 --- /dev/null +++ b/src/libutil/fs-sink.cc @@ -0,0 +1,82 @@ +#include + +#include "config.hh" +#include "fs-sink.hh" + +namespace nix { + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{this, false, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; +}; + +static RestoreSinkSettings restoreSinkSettings; + +static GlobalConfig::Register r1(&restoreSinkSettings); + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) override + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); + }; + + void createRegularFile(const Path & path) override + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); + } + + void closeRegularFile() override + { + /* Call close explicitly to make sure the error is checked */ + fd.close(); + } + + void isExecutable() override + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(uint64_t len) override + { + if (!archiveSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif + } + + void receiveContents(std::string_view data) override + { + writeFull(fd.get(), data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + +} From f2e201fbdb93925b9588f78a03d67fcd91c74b65 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:49:01 -0400 Subject: [PATCH 054/402] Expose `RestoreSink` in header (`fs-sink.hh`) Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/fs-sink.cc | 117 ++++++++++++++++++++--------------------- src/libutil/fs-sink.hh | 17 ++++++ 2 files changed, 73 insertions(+), 61 deletions(-) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index fcffd3216..a08a723a4 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -15,68 +15,63 @@ static RestoreSinkSettings restoreSinkSettings; static GlobalConfig::Register r1(&restoreSinkSettings); -struct RestoreSink : ParseSink + +void RestoreSink::createDirectory(const Path & path) { - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); }; +void RestoreSink::createRegularFile(const Path & path) +{ + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); +} + +void RestoreSink::closeRegularFile() +{ + /* Call close explicitly to make sure the error is checked */ + fd.close(); +} + +void RestoreSink::isExecutable() +{ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); +} + +void RestoreSink::preallocateContents(uint64_t len) +{ + if (!restoreSinkSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif +} + +void RestoreSink::receiveContents(std::string_view data) +{ + writeFull(fd.get(), data); +} + +void RestoreSink::createSymlink(const Path & path, const std::string & target) +{ + Path p = dstPath + path; + nix::createSymlink(target, p); +} + } diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index ed6484e61..6837e2fc4 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -22,4 +22,21 @@ struct ParseSink virtual void createSymlink(const Path & path, const std::string & target) { }; }; +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void closeRegularFile() override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; + void receiveContents(std::string_view data) override; + + void createSymlink(const Path & path, const std::string & target) override; +}; + } From 694810ba34a74226eba22daf5c6264b9d81e2926 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Sep 2023 23:54:04 -0400 Subject: [PATCH 055/402] doc: `showOptions`: Move union to caller `showOptions` itself doesn't care, so it shouldn't take two separate arguments. --- doc/manual/generate-manpage.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 87891fa7a..b85c488c9 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -78,16 +78,15 @@ let maybeOptions = optionalString (details.flags != {}) '' # Options - ${showOptions details.flags toplevel.flags} + ${showOptions (details.flags // toplevel.flags)} > **Note** > > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; - showOptions = options: commonOptions: + showOptions = allOptions: let - allOptions = options // commonOptions; showCategory = cat: '' ${optionalString (cat != "") "**${cat}:**"} From 9c640c122931aa6515e1f3c80cb4d415a1352cc7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Sep 2023 00:28:16 -0400 Subject: [PATCH 056/402] doc: `showOptions`: Simplify code with `builtins.groupBy` This makes grouping options by category much nicer. No behavior should be changed. --- doc/manual/generate-manpage.nix | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index b85c488c9..e18810594 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,8 +1,8 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs + attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings; + inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique showSettings; in commandDump: @@ -87,12 +87,11 @@ let showOptions = allOptions: let - showCategory = cat: '' + showCategory = cat: opts: '' ${optionalString (cat != "") "**${cat}:**"} - ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} + ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} ''; - listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); showOption = name: option: let shortName = optionalString @@ -106,8 +105,12 @@ let ${option.description} ''; - categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); - in concatStrings (map showCategory categories); + categories = mapAttrs + (_: listToAttrs) + (groupBy + (cmd: cmd.value.category) + (attrsToList allOptions)); + in concatStrings (attrValues (mapAttrs showCategory categories)); in squash result; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; From 1d9fd3a6f8ac53be4ba7d409defe291c0dbdf97a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Sep 2023 23:55:06 -0400 Subject: [PATCH 057/402] manual / manpages: Adjust option filter filtering, move from C++ to Nix Behavior change: Before we only showed uption if the command-specific options were non-empty. But that is somewhat odd since we also show common options. Now, we do everything based on the union of both sorts of options (with hidden-categories filtered, as before). Implementation change: The JSON dumping once again includes all options; the filtering of hidden categories is done in the Nix instead. This is better separation of "content" vs "presentation", and prepare the way for the HTML manual vs manpages / `--help` doing different things. --- doc/manual/generate-manpage.nix | 8 ++++++-- src/libutil/args.cc | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index e18810594..b41730bc2 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -75,10 +75,14 @@ let (details ? doc) (replaceStrings ["@stores@"] [storeDocs] details.doc); - maybeOptions = optionalString (details.flags != {}) '' + maybeOptions = let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in optionalString (allVisibleOptions != {}) '' # Options - ${showOptions (details.flags // toplevel.flags)} + ${showOptions allVisibleOptions} > **Note** > diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 8db293762..e410c7eec 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -236,7 +236,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); - if (hiddenCategories.count(flag->category)) continue; + j["hiddenCategory"] = hiddenCategories.count(flag->category) > 0; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); From 9f93972c4d73abdcde789112229209af2cbda520 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Sep 2023 00:35:03 -0400 Subject: [PATCH 058/402] manual / manpages: Make option category names a proper subheader Before they were an "ad-hoc" header with bold and a colon; now they are a proper subheader. For the man pages, this doesn't make much of a difference, but it will help more on for the HTML manual, where things can be restyled. Again, good separation of content vs presentation. --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index b41730bc2..21a4802e5 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -92,7 +92,7 @@ let showOptions = allOptions: let showCategory = cat: opts: '' - ${optionalString (cat != "") "**${cat}:**"} + ${optionalString (cat != "") "## ${cat}"} ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} ''; From e4b83fbfe2e08ddd2ebd409c26f2dc4c11ee1d4a Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Sun, 24 Sep 2023 14:54:41 +0800 Subject: [PATCH 059/402] libexpr: const rvalue reference -> value for nix::Expr nodes --- src/libexpr/nixexpr.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5ca3d1fa6..944977e50 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -238,7 +238,7 @@ struct ExprSelect : Expr PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; + ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -248,7 +248,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; + ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; From 89e5e68799132bc2f1d2551002dab84b63fde9ac Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 24 Sep 2023 20:28:34 +0200 Subject: [PATCH 060/402] doc/hacking: fix `make` target to build the docs (#9033) Was confused why `make html` didn't work while working on #9032, but then I realized that after this section was written, the target was renamed to `manual-html` in 6910f5dcb6784360589b860b8f80487a5c97fe08. --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 22f629c6f..a2446b53a 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -228,7 +228,7 @@ This happens late in the process, so `nix build` is not suitable for iterating. To build the manual incrementally, run: ```console -make html -j $NIX_BUILD_CORES +make manual-html -j $NIX_BUILD_CORES ``` In order to reflect changes to the [Makefile], clear all generated files before re-building: From b3433099d4ef3fbc2ef86f97cf19dcd543bd05e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Mon, 25 Sep 2023 09:56:49 +0200 Subject: [PATCH 061/402] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- src/nix/flake.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 1c512b315..939269c6a 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -68,11 +68,9 @@ inputs.nixpkgs = { ``` Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), -characters outside of the allowed range (i.e. are neither [reserved -character](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) or -[unreserved -characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) must be -percent-encoded. +characters outside of the allowed range (i.e. neither [reserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) +nor [unreserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) +must be percent-encoded. ### Examples @@ -110,8 +108,7 @@ The semantic of such a path is as follows: 2. The filesystem root (/), or 3. A folder on a different mount point. -Contrary to URL-like reference, path-like flake references can contain -arbitrary unicode characters (except `#` and `?`). +Contrary to URL-like references, path-like flake references can contain arbitrary unicode characters (except `#` and `?`). ### Examples From 4606a07bb62c650d2261f2d8d5e93e547acf0af6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 25 Sep 2023 08:20:39 -0400 Subject: [PATCH 062/402] generate-manpage.nix: Add comment explaining one bit --- doc/manual/generate-manpage.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 21a4802e5..41725aed6 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -110,6 +110,7 @@ let ${option.description} ''; categories = mapAttrs + # Convert each group from a list of key-value pairs back to an attrset (_: listToAttrs) (groupBy (cmd: cmd.value.category) From 70b5e6050cd135b75942f731ad892e4f473c6c21 Mon Sep 17 00:00:00 2001 From: waalge <47293755+waalge@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:39:11 +0000 Subject: [PATCH 063/402] fix docstring --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cd9a05bb2..0d503ae26 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2991,7 +2991,7 @@ static RegisterPrimOp primop_tail({ .name = "__tail", .args = {"list"}, .doc = R"( - Return the second to last elements of a list; abort evaluation if + Return the list without its first item; abort evaluation if the argument isn’t a list or is an empty list. > **Warning** From ad213103d85d22b52c608362cb2127fa10e49620 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 25 Sep 2023 17:45:13 +0100 Subject: [PATCH 064/402] src/libstore/profiles.cc: fix comparison of sign difference Detected by `gcc` as: CXX src/libstore/profiles.o src/libstore/profiles.cc: In function 'void nix::deleteGenerationsGreaterThan(const Path&, GenerationNumber, bool)': src/libstore/profiles.cc:186:50: warning: comparison of integer expressions of different signedness: 'int' and 'nix::GenerationNumber' {aka 'long unsigned int'} [-Wsign-compare] 186 | for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); | ~~~~~^~~~~ --- src/libstore/profiles.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4e9955948..239047dd6 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -183,7 +183,7 @@ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bo iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; }); // Skip over `max` generations, preserving them - for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); + for (GenerationNumber keep = 0; i != gens.rend() && keep < max; ++i, ++keep); // Delete the rest for (; i != gens.rend(); ++i) From bd24176ac57b9d418b4ec51448b188cb74b87945 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 25 Sep 2023 17:51:17 +0100 Subject: [PATCH 065/402] libexpr/nixexpr.hh: Remove redundant inline This is redundant since definitions in C++ record are implicitly inline-ed. Co-authored-by: Yingchi Long --- src/libexpr/nixexpr.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 17b2d1136..5c9242759 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -156,7 +156,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { static unsigned long nrExprs; - inline Expr() { + Expr() { nrExprs++; } virtual ~Expr() { }; From cba53b3a18c2758e3d969d9d5781294560afe1b7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:39:11 +0200 Subject: [PATCH 066/402] reformat for readability --- doc/manual/src/command-ref/nix-store/realise.md | 8 ++++++-- doc/manual/src/glossary.md | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index a421a4220..5428d57fa 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -15,8 +15,12 @@ Each of *paths* is processed as follows: 1. If it is not [valid], substitute the store derivation file itself. 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. - - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. + - With [content-addressed derivations] (experimental): + Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. + - For any store paths that cannot be substituted, produce the required store objects: + 1. Realise all outputs of the derivation's dependencies + 2. Run the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable + - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b7c340d36..5345f2deb 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,11 +33,15 @@ Ensure a [store path] is [valid][validity]. - This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] + - Delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs + - See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). [realise]: #gloss-realise From 541890463d83d1a818dfabc79053d1302bd828a2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 13:49:59 +0200 Subject: [PATCH 067/402] make separate section for builder execution --- doc/manual/src/language/derivations.md | 116 +++++++++++++------------ 1 file changed, 59 insertions(+), 57 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 79a09122a..df4673cf5 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -93,69 +93,71 @@ wrapper around `derivation` that adds a default value for `system` and always uses Bash as the builder, to which the supplied builder is passed as a command-line argument. See the Nixpkgs manual for details. -The builder is executed as follows: +## Builder execution - - A temporary directory is created under the directory specified by - `TMPDIR` (default `/tmp`) where the build will take place. The - current directory is changed to this directory. +The [`builder`](#attr-builder) is executed as follows: - - The environment is cleared and set to the derivation attributes, as - specified above. +- A temporary directory is created under the directory specified by + `TMPDIR` (default `/tmp`) where the build will take place. The + current directory is changed to this directory. - - In addition, the following variables are set: - - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. - - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. - - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. - - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. - - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). - - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +- The environment is cleared and set to the derivation attributes, as + specified above. - - If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple Nix instances from performing the same - build at the same time. +- In addition, the following variables are set: - - A log of the combined standard output and error is written to - `/nix/var/log/nix`. + - `NIX_BUILD_TOP` contains the path of the temporary directory for + this build. - - The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. + - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the + temporary directory. This is to prevent the builder from + accidentally writing temporary files anywhere else. Doing so + might cause interference by other processes. - - The temporary directory is removed (unless the `-K` option was - specified). + - `PATH` is set to `/path-not-set` to prevent shells from + initialising it to their built-in default value. - - If the build was successful, Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + - `HOME` is set to `/homeless-shelter` to prevent programs from + using `/etc/passwd` or the like to find the user's home + directory, which could cause impurity. Usually, when `HOME` is + set, it is used as the location of the home directory, even if + it points to a non-existent path. - - After the build, Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Note that possible `setuid` and `setgid` - bits are cleared. Setuid and setgid programs are not currently - supported by Nix. This is because the Nix archives used in - deployment have no concept of ownership information, and because it - makes the build result dependent on the user performing the build. + - `NIX_STORE` is set to the path of the top-level Nix store + directory (typically, `/nix/store`). + + - For each output declared in `outputs`, the corresponding + environment variable is set to point to the intended path in the + Nix store for that output. Each output path is a concatenation + of the cryptographic hash of all build inputs, the `name` + attribute and the output name. (The output name is omitted if + it’s `out`.) + +- If an output path already exists, it is removed. Also, locks are + acquired to prevent multiple Nix instances from performing the same + build at the same time. + +- A log of the combined standard output and error is written to + `/nix/var/log/nix`. + +- The builder is executed with the arguments specified by the + attribute `args`. If it exits with exit code 0, it is considered to + have succeeded. + +- The temporary directory is removed (unless the `-K` option was + specified). + +- If the build was successful, Nix scans each output path for + references to input paths by looking for the hash parts of the input + paths. Since these are potential runtime dependencies, Nix registers + them as dependencies of the output paths. + +- After the build, Nix sets the last-modified timestamp on all files + in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to + the default group, and sets the mode of the file to 0444 or 0555 + (i.e., read-only, with execute permission enabled if the file was + originally executable). Note that possible `setuid` and `setgid` + bits are cleared. Setuid and setgid programs are not currently + supported by Nix. This is because the Nix archives used in + deployment have no concept of ownership information, and because it + makes the build result dependent on the user performing the build. From e2f118efeda5c0a9b1380db736eb2b86f691b1fa Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 14:03:17 +0200 Subject: [PATCH 068/402] restructure and reword input attributes section on `derivation` --- doc/manual/src/language/derivations.md | 208 +++++++++++++++---------- 1 file changed, 123 insertions(+), 85 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index df4673cf5..56c6818ca 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,97 +1,135 @@ # Derivations -The most important built-in function is `derivation`, which is used to -describe a single derivation (a build task). It takes as input a set, -the attributes of which specify the inputs of the build. +The most important built-in function is `derivation`, which is used to describe a single derivation: +a build task to run a process on precisely defined input files and repeatably produce output files at uniquely determined file system paths. - - There must be an attribute named [`system`]{#attr-system} whose value must be a - string specifying a Nix system type, such as `"i686-linux"` or - `"x86_64-darwin"`. (To figure out your system type, run `nix -vv - --version`.) The build can only be performed on a machine and - operating system matching the system type. (Nix can automatically - [forward builds for other - platforms](../advanced-topics/distributed-builds.md) by forwarding - them to other machines.) +It takes as input an attribute set, the attributes of which specify the inputs to the build. +It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. - - There must be an attribute named `name` whose value must be a - string. This is used as a symbolic name for the package by - `nix-env`, and it is appended to the output paths of the derivation. + - - There must be an attribute named [`builder`]{#attr-builder} that identifies the - program that is executed to perform the build. It can be either a - derivation or a source (a local file reference, e.g., - `./builder.sh`). +## Input attributes - - Every attribute is passed as an environment variable to the builder. - Attribute values are translated to environment variables as follows: - - - Strings and numbers are just passed verbatim. - - - A *path* (e.g., `../foo/sources.tar`) causes the referenced file - to be copied to the store; its location in the store is put in - the environment variable. The idea is that all sources should - reside in the Nix store, since all inputs to a derivation should - reside in the Nix store. - - - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. - - - Lists of the previous types are also allowed. They are simply - concatenated, separated by spaces. - - - `true` is passed as the string `1`, `false` and `null` are - passed as an empty string. +### Required - - The optional attribute `args` specifies command-line arguments to be - passed to the builder. It should be a list. +- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) - - The optional attribute `outputs` specifies a list of symbolic - outputs of the derivation. By default, a derivation produces a - single output path, denoted as `out`. However, derivations can - produce multiple output paths. This is useful because it allows - outputs to be downloaded or garbage-collected separately. For - instance, imagine a library package that provides a dynamic library, - header files, and documentation. A program that links against the - library doesn’t need the header files and documentation at runtime, - and it doesn’t need the documentation at build time. Thus, the - library package could specify: - - ```nix - outputs = [ "lib" "headers" "doc" ]; - ``` - - This will cause Nix to pass environment variables `lib`, `headers` - and `doc` to the builder containing the intended store paths of each - output. The builder would typically do something like - - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$headers/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.headers ]; - ``` - - The first element of `outputs` determines the *default output*. - Thus, you could also write - - ```nix - buildInputs = [ pkg pkg.headers ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. + Symbolic name for the derivation. + It is appended to the [store paths](@docroot@/glossary.md#gloss-store-path) of the resulting store derivation and its [output paths][output path]. -The function `mkDerivation` in the Nixpkgs standard environment is a -wrapper around `derivation` that adds a default value for `system` and -always uses Bash as the builder, to which the supplied builder is passed -as a command-line argument. See the Nixpkgs manual for details. + Example: `name = "hello";` + +- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) + + The system type on which the [`builder`](#attr-builder) executable can be run. + + Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. + It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. + + Examples: + + `system = "x86_64-linux";` + + `system = builtins.currentSystem;` + + [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + + [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system + +- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) + + Path to an executable that will perform the build. + + Examples: + + `builder = "/bin/bash";` + + `builder = ./builder.sh;` + + `builder = "${pkgs.python}/bin/python";` + +### Optional + +- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ ]` + + Command-line arguments to be passed to the [`builder`](#attr-builder) executable. + + Example: `args = [ "-c" "echo hello world > $out" ];` + +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ "out" ]` + + Symbolic outputs of the derivation. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [output path]. + + [output path]: @docroot@/glossary.md#gloss-output-path + + By default, a derivation produces a single output path called `out`. + However, derivations can produce multiple output paths. + This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + + Examples: + + Imagine a library package that provides a dynamic library, header files, and documentation. + A program that links against the library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + Thus, the library package could specify: + + ```nix + outputs = [ "lib" "headers" "doc" ]; + ``` + + This will cause Nix to pass environment variables `lib`, `headers`, and `doc` to the builder containing the intended store paths of each output. + The builder would typically do something like + + ```bash + ./configure \ + --libdir=$lib/lib \ + --includedir=$headers/include \ + --docdir=$doc/share/doc + ``` + + for an Autoconf-style package. + + You can refer to each output of a + derivation by selecting it as an attribute, e.g. + + ```nix + buildInputs = [ pkg.lib pkg.headers ]; + ``` + + + The first element of `outputs` determines the *default output*. + Thus, you could also write + + ```nix + buildInputs = [ pkg pkg.headers ]; + ``` + + since `pkg` is equivalent to `pkg.lib`. + +- See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. + + + +- Every other attribute is passed as an environment variable to the builder. + Attribute values are translated to environment variables as follows: + + - Strings and numbers are just passed verbatim. + + - A *path* (e.g., `../foo/sources.tar`) causes the referenced file + to be copied to the store; its location in the store is put in + the environment variable. The idea is that all sources should + reside in the Nix store, since all inputs to a derivation should + reside in the Nix store. + + - A *derivation* causes that derivation to be built prior to the + present derivation; its default output path is put in the + environment variable. + + - Lists of the previous types are also allowed. They are simply + concatenated, separated by spaces. + + - `true` is passed as the string `1`, `false` and `null` are + passed as an empty string. ## Builder execution From d621dd17f23023d4d6a84daba90ef38e848a1375 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 15 Sep 2023 10:31:31 +0200 Subject: [PATCH 069/402] more precise wording Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 56c6818ca..dfefe446f 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -14,14 +14,14 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar - [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) - Symbolic name for the derivation. - It is appended to the [store paths](@docroot@/glossary.md#gloss-store-path) of the resulting store derivation and its [output paths][output path]. + A symbolic name for the derivation. + It is added to the [store derivation]'s [path](@docroot@/glossary.md#gloss-store-path) and its [output paths][output path]. Example: `name = "hello";` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) - The system type on which the [`builder`](#attr-builder) executable can be run. + The system type on which the [`builder`](#attr-builder) executable is meant to be run. Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. From 026c24e3787d7b1cf81c49fdcb66818fd5dad2e4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 15 Sep 2023 15:59:07 +0200 Subject: [PATCH 070/402] add example for store path using the given name --- doc/manual/src/language/derivations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index dfefe446f..3ced976f6 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -19,6 +19,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Example: `name = "hello";` + The store derivation's path will be `/nix/store/-hello.drv`, and the output paths will be of the form `/nix/store/-hello[-]` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. From 17884f54d18b6d69dbfefe6c08272ea51b8879f8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Sep 2023 09:53:29 +0200 Subject: [PATCH 071/402] clarification on extra attributes Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 3ced976f6..77eac03b6 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -114,7 +114,11 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar - Every other attribute is passed as an environment variable to the builder. Attribute values are translated to environment variables as follows: - - Strings and numbers are just passed verbatim. + - Strings are passed unchanged. + + - Integral numbers are converted to decimal notation. + + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. - A *path* (e.g., `../foo/sources.tar`) causes the referenced file to be copied to the store; its location in the store is put in From 5e4734a08b8fb3f80b9617c190a2b5dfaba4a1da Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:13:56 +0200 Subject: [PATCH 072/402] reword to avoid saying "build" derivations are about data transformation, so the term "build" does not add any information. there was also some feedback that "build task" is not more helpful than "derivation" if you have no prior experience with Nix or build systems, while existing associations may be misleading. --- doc/manual/src/language/derivations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 77eac03b6..da3cca4ad 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,9 +1,9 @@ # Derivations The most important built-in function is `derivation`, which is used to describe a single derivation: -a build task to run a process on precisely defined input files and repeatably produce output files at uniquely determined file system paths. +a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. -It takes as input an attribute set, the attributes of which specify the inputs to the build. +It takes as input an attribute set, the attributes of which specify the inputs to the process. It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. From 75a231147f2d6acd32321e088a5030a69f188854 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:43:41 +0200 Subject: [PATCH 073/402] be more precise about `system` semantics --- doc/manual/src/language/derivations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index da3cca4ad..c13c81e7d 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -24,7 +24,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar The system type on which the [`builder`](#attr-builder) executable is meant to be run. - Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. + A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. Examples: From 7de66f19f84edc0d617e00f6e0cc37f2dbc4f5f6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:46:26 +0200 Subject: [PATCH 074/402] example: headers -> dev make the example more realistic, since `headers` is not an output name used in Nixpkgs Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index c13c81e7d..6e6d3298b 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -75,16 +75,16 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Thus, the library package could specify: ```nix - outputs = [ "lib" "headers" "doc" ]; + outputs = [ "lib" "dev" "doc" ]; ``` - This will cause Nix to pass environment variables `lib`, `headers`, and `doc` to the builder containing the intended store paths of each output. + This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. The builder would typically do something like ```bash ./configure \ --libdir=$lib/lib \ - --includedir=$headers/include \ + --includedir=$dev/include \ --docdir=$doc/share/doc ``` @@ -94,7 +94,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar derivation by selecting it as an attribute, e.g. ```nix - buildInputs = [ pkg.lib pkg.headers ]; + buildInputs = [ pkg.lib pkg.dev ]; ``` @@ -102,7 +102,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Thus, you could also write ```nix - buildInputs = [ pkg pkg.headers ]; + buildInputs = [ pkg pkg.dev ]; ``` since `pkg` is equivalent to `pkg.lib`. From 5b0336b3b14551762f37126423ea8b4590ebb560 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:58:50 +0200 Subject: [PATCH 075/402] reword example for clarity --- doc/manual/src/language/derivations.md | 28 ++++++++++---------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 6e6d3298b..e0b9d021a 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -71,11 +71,15 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Examples: Imagine a library package that provides a dynamic library, header files, and documentation. - A program that links against the library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. Thus, the library package could specify: ```nix - outputs = [ "lib" "dev" "doc" ]; + derivation { + # ... + outputs = [ "lib" "dev" "doc" ]; + # ... + } ``` This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. @@ -90,22 +94,12 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar for an Autoconf-style package. - You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.dev ]; - ``` - + You can refer to each output of a derivation by selecting it as an attribute, e.g. `myPackage.lib` or `myPackage.doc`. The first element of `outputs` determines the *default output*. - Thus, you could also write + Therefore, in the given example, `myPackage` is equivalent to `myPackage.lib`. - ```nix - buildInputs = [ pkg pkg.dev ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. + - See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. @@ -115,9 +109,9 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Attribute values are translated to environment variables as follows: - Strings are passed unchanged. - + - Integral numbers are converted to decimal notation. - + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. - A *path* (e.g., `../foo/sources.tar`) causes the referenced file From 887cbcd3958abed7cf726f7eb23ba530e021ddfb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:53:50 +0200 Subject: [PATCH 076/402] add contributing guide for documentation --- doc/manual/local.mk | 6 +- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/contributing/contributing.md | 9 +- doc/manual/src/contributing/documentation.md | 181 +++++++++++++++++++ doc/manual/src/contributing/hacking.md | 51 ------ 5 files changed, 195 insertions(+), 53 deletions(-) create mode 100644 doc/manual/src/contributing/documentation.md diff --git a/doc/manual/local.mk b/doc/manual/local.mk index abdfd6a62..ab95af53c 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -173,6 +173,10 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ +# the `! -name 'contributing.md'` filter excludes the one place where +# `@docroot@` is to be preserved for documenting the mechanism +# FIXME: maybe contributing guides should live right next to the code +# instead of in the manual $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ @@ -180,7 +184,7 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/ find "$$tmp" -name '*.md' | while read -r file; do \ $(call process-includes,$$file,$$file); \ done; \ - find "$$tmp" -name '*.md' | while read -r file; do \ + find "$$tmp" -name '*.md' ! -name 'documentation.md' | while read -r file; do \ docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ sed -i "s,@docroot@,$$docroot,g" "$$file"; \ done; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c23186511..5c3667690 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -105,6 +105,7 @@ - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) - [Testing](contributing/testing.md) + - [Documentation](contributing/documentation.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md index 854139a31..4d55c17a4 100644 --- a/doc/manual/src/contributing/contributing.md +++ b/doc/manual/src/contributing/contributing.md @@ -1 +1,8 @@ -# Contributing +# Development + +Nix is developed on GitHub. +Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md) if you want to get involved. + +This chapter is a collection of guides for making changes to the code and documentation. + +If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md new file mode 100644 index 000000000..f73ab2149 --- /dev/null +++ b/doc/manual/src/contributing/documentation.md @@ -0,0 +1,181 @@ +# Contributing documentation + +Improvements to documentation are very much appreciated, and a good way to start out with contributing to Nix. + +This is how you can help: +- Address [open issues with documentation](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) +- Review [pull requests concerning documentation](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+label%3Adocumentation) + +Incremental refactorings of the documentation build setup to make it faster or easier to understand and maintain are also welcome. + +## Building the manual + +Build the manual from scratch: + +```console +nix-build $(nix-instantiate)'!doc' +``` + +or + +```console +nix build .#^doc +``` + +and open `./result-doc/share/doc/nix/manual/index.html`. + +To build the manual incrementally, [enter the development shell](./hacking.md) and run: + +```console +make manual-html -j $NIX_BUILD_CORES +``` + +and open `./outputs/out/share/doc/nix/manual/language/index.html`. + +In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: + +[Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES +``` + +## Style guide + +The goal of this style guide is to make it such that +- The manual is easy to search and skim for relevant information +- Documentation sources are easy to edit +- Changes to documentation are easy to review + +You will notice that this is not implemented consistently yet. +Please follow the guide when making additions or changes to existing documentation. +Do not make sweeping changes, unless they are programmatic and can be validated easily. + +### Language + +This manual is [reference documentation](https://diataxis.fr/reference/). +The typical usage pattern is to look up isolated pieces of information. +It should therefore aim to be correct, consistent, complete, and easy to navigate at a glance. + +- Aim for clarity and brevity. + + Please take the time to read the [plain language guidelines](https://www.plainlanguage.gov/guidelines/) for details. + +- Describe the subject factually. + + In particular, do not make value judgements or recommendations. + Check the code or add tests if in doubt. + +- Provide complete, minimal examples, and explain them. + + Readers should be able to try examples verbatim and get the same results as shown in the manual. + Always describe in words what a given example does. + + Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. + +- Use British English. + + This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. + +### Links and anchors + +Reference documentation must be readable in arbitrary order. +Readers cannot be expected to have any particular prerequisite knowledge about Nix. +While the table of contents can provide guidance and full-text search can help, they are most likely to find what they need by following sensible cross-references. + +- Link to technical terms + + When mentioning Nix-specific concepts, commands, options, settings, etc., link to appropriate documentation. + Also link to external tools or concepts, especially if their meaning may be ambiguous. + You may also want to link to definitions of less common technical terms. + + Then readers won't have to actively search for definitions and are more likely to discover relevant information on their own. + + > **Note** + > + > `man` and `--help` pages don't display links. + > Use appropriate link texts such that readers of terminal output can infer search terms. + +- Do not break existing URLs between releases. + + There are countless links in the wild pointing to old versions of the manual. + We want people to find up-to-date documentation when following popular advice. + + - When moving files, update [redirects on nixos.org](https://github.com/NixOS/nixos-homepage/blob/master/netlify.toml). + + This is especially important when moving information out of the Nix manual to other resources. + + - When changing anchors, update [client-side redirects](https://github.com/NixOS/nix/blob/master/doc/manual/redirects.js) + + The current setup is cumbersome, and help making better automation is appreciated. + +The build checks for broken internal links with. +This happens late in the process, so [building the whole manual](#building-the-manual) is not suitable for iterating quickly. +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment + +### Markdown conventions + +The manual is written in markdown, and rendered with [mdBook](https://github.com/rust-lang/mdBook) for the web and with [lowdown](https://github.com/kristapsdz/lowdown) for `man` pages and `--help` output. + +For supported markdown features, refer to: +- [mdBook documentation](https://rust-lang.github.io/mdBook/format/markdown.html) +- [lowdown documentation](https://kristaps.bsd.lv/lowdown/) + +Please observe these guidelines to ease reviews: + +- Write one sentence per line. + + This makes long sentences immediately visible, and makes it easier to review changes and make direct suggestions. + +- Use reference links – sparingly – to ease source readability. + Put definitions close to their first use. + + Example: + + ``` + A [store object] contains a [file system object] and [references] to other store objects. + + [store object]: @docroot@/glossary.md#gloss-store-object + [file system object]: @docroot@/architecture/file-system-object.md + [references]: @docroot@/glossary.md#gloss-reference + ``` + +- Use admonitions of the following form: + + ``` + > **Note** + > + > This is a note. + ``` + +### The `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the [`mdbook-linkcheck`] tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile for the manual]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. + +## API documentation + +[Doxygen API documentation] is available online. +You can also build and view it yourself: + +[Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs + +```console +# nix build .#hydraJobs.internal-api-docs +# xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +``` +# make internal-api-html +# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +``` diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index a2446b53a..db393942c 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -220,54 +220,3 @@ Configure your editor to use the `clangd` from the shell, either by running it i > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. - -### Checking links in the manual - -The build checks for broken internal links. -This happens late in the process, so `nix build` is not suitable for iterating. -To build the manual incrementally, run: - -```console -make manual-html -j $NIX_BUILD_CORES -``` - -In order to reflect changes to the [Makefile], clear all generated files before re-building: - -[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk - -```console -rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES -``` - -[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. - -[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck -[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment - -#### `@docroot@` variable - -`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. - -If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. - -If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. -See existing `@docroot@` logic in the [Makefile]. -Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. - -## API documentation - -Doxygen API documentation is [available -online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You -can also build and view it yourself: - -```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html -``` - -or inside a `nix develop` shell by running: - -``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html -``` From 45de35bcf14d1bdad5dca2e24a113d26d1985aed Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 01:24:07 +0200 Subject: [PATCH 077/402] fix broken redirects script --- doc/manual/redirects.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index b43622ed6..b2fad19bb 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -343,7 +343,7 @@ const redirects = { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", - } + }, "contributing/hacking.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", From c6f8247032612b4cc4a9f79213a887aa9ddddfda Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 03:10:12 +0200 Subject: [PATCH 078/402] fix broken reference link --- doc/manual/src/language/derivations.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index e0b9d021a..4a1b70fd1 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -4,7 +4,9 @@ The most important built-in function is `derivation`, which is used to describe a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. It takes as input an attribute set, the attributes of which specify the inputs to the process. -It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. +It outputs an attribute set, and produces a [store derivation] as a side effect of evaluation. + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation From 7e24dc606b1cf7fd4e4dfd7027ba1469a59784d5 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 22 Sep 2023 13:05:55 -0400 Subject: [PATCH 079/402] fix(tests): fix assumption that string.s is a char* --- src/libexpr/tests/libexpr.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index b8e65aafe..97b05ac46 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -71,7 +71,7 @@ namespace nix { if (arg.type() != nString) { return false; } - return std::string_view(arg.string.s) == s; + return std::string_view(arg.c_str()) == s; } MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) { @@ -106,8 +106,8 @@ namespace nix { if (arg.type() != nPath) { *result_listener << "Expected a path got " << arg.type(); return false; - } else if (std::string_view(arg.string.s) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.string.s; + } else if (std::string_view(arg._path) != p) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str(); return false; } return true; From b17f200b11b3901c4f975c9901f1e2c6fc54124f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 03:49:03 +0200 Subject: [PATCH 080/402] Document "Import From Derivation" (#7332) * document "Import From Derivation" Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/glossary.md | 5 +- .../src/language/import-from-derivation.md | 139 ++++++++++++++++++ src/libexpr/eval-settings.hh | 11 +- 4 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 doc/manual/src/language/import-from-derivation.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c23186511..9e2ad0618 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -33,6 +33,7 @@ - [Operators](language/operators.md) - [Derivations](language/derivations.md) - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) - [Advanced Topics](advanced-topics/advanced-topics.md) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b7c340d36..c23466351 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -105,12 +105,15 @@ - [store object]{#gloss-store-object} - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. It can be referred to by a [store path]. [store object]: #gloss-store-object +- [IFD]{#gloss-ifd} + + [Import From Derivation](./language/import-from-derivation.md) + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md new file mode 100644 index 000000000..03b3f9d91 --- /dev/null +++ b/doc/manual/src/language/import-from-derivation.md @@ -0,0 +1,139 @@ +# Import From Derivation + +The value of a Nix expression can depend on the contents of a [store object](@docroot@/glossary.md#gloss-store-object). + +Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): + +- [`import`](./builtins.md#builtins-import)` expr` +- [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` +- [`builtins.readFileType`](./builtins.md#builtins-readFileType)` expr` +- [`builtins.readDir`](./builtins.md#builtins-readDir)` expr` +- [`builtins.pathExists`](./builtins.md#builtins-pathExists)` expr` +- [`builtins.filterSource`](./builtins.md#builtins-filterSource)` f expr` +- [`builtins.path`](./builtins.md#builtins-path)` { path = expr; }` +- [`builtins.hashFile`](./builtins.md#builtins-hashFile)` t expr` +- `builtins.scopedImport x drv` + +When the store path needs to be accessed, evaluation will be paused, the corresponding store object [realised], and then evaluation resumed. + +[realised]: @docroot@/glossary.md#gloss-realise + +This has performance implications: +Evaluation can only finish when all required store objects are realised. +Since the Nix language evaluator is sequential, it only finds store paths to read from one at a time. +While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. + +Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. +Without IFD it is ensured that evaluation is complete and Nix can produce a build plan before starting any realisation. + +## Example + +In the following Nix expression, the inner derivation `drv` produces a file with contents `hello`. + +```nix +# IFD.nix +let + drv = derivation { + name = "hello"; + builder = "/bin/sh"; + args = [ "-c" "echo -n hello > $out" ]; + system = builtins.currentSystem; + }; +in "${builtins.readFile drv} world" +``` + +```shellSession +nix-instantiate IFD.nix --eval --read-write-mode +``` + +``` +building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... +"hello world" +``` + +The contents of the derivation's output have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). +Only then evaluation can continue to produce the final result. + +## Illustration + +As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a [store object]. +Boxes are data structures, arrow labels are transformations. + +``` ++----------------------+ +------------------------+ +| Nix evaluator | | Nix store | +| .----------------. | | | +| | Nix expression | | | | +| '----------------' | | | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | .------------------. | +| | derivation |----|-instantiate-|->| store derivation | | +| '------------' | | '------------------' | +| | | | | +| | | realise | +| | | | | +| | | V | +| .----------------. | | .--------------. | +| | Nix expression |<-|----read-----|----| store object | | +| '----------------' | | '--------------' | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | | +| | value | | | | +| '------------' | | | ++----------------------+ +------------------------+ +``` + +In more detail, the following sequence diagram shows how the expression is evaluated step by step, and where evaluation is blocked to wait for the build output to appear. + +``` +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' + | | | + |evaluate IFD.nix| | + |--------------->| | + | | | + | evaluate `"${readFile drv} world"` | + | | | + | evaluate `readFile drv` | + | | | + | evaluate `drv` as string | + | | | + | |instantiate /nix/store/...-hello.drv| + | |----------------------------------->| + | : | + | : realise /nix/store/...-hello.drv | + | :----------------------------------->| + | : | + | |--------. + | : | | + | (evaluation blocked) | echo hello > $out + | : | | + | |<-------' + | : /nix/store/...-hello | + | |<-----------------------------------| + | | | + | resume `readFile /nix/store/...-hello` | + | | | + | | readFile /nix/store/...-hello | + | |----------------------------------->| + | | | + | | hello | + | |<-----------------------------------| + | | | + | resume `"${"hello"} world"` | + | | | + | resume `"hello world"` | + | | | + | "hello world" | | + |<---------------| | +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' +``` diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 19122bc31..19ba679be 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -47,11 +47,12 @@ struct EvalSettings : Config Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. + By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). + + With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature, + even when the required store object is readily available. + This ensures that evaluation will not require any builds to take place, + regardless of the state of the store. )"}; Setting allowedUris{this, {}, "allowed-uris", From a757749fcfa810f699fa096f98821efccba86522 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 06:28:17 +0200 Subject: [PATCH 081/402] reword for readability --- doc/manual/src/release-notes/release-notes.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md index 1fa2f5580..cc805e631 100644 --- a/doc/manual/src/release-notes/release-notes.md +++ b/doc/manual/src/release-notes/release-notes.md @@ -6,6 +6,7 @@ Notable changes and additions are announced in the release notes for each versio Bugfixes can be backported on request to previous Nix releases. We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). -Backports never skip releases (if a feature is backported to `x.y`, it must also be available in `x.(y+1)`; +Backports never skip releases. +If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. From 5b902ce9d60fca33c0f5bb76fcd31792c0bb03a6 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Tue, 26 Sep 2023 23:30:32 +0800 Subject: [PATCH 082/402] libexpr: construct ExprPath by move ctor, not copy cotr --- src/libexpr/parser.y | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde..509f4fcc4 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -520,7 +520,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(path); + $$ = new ExprPath(std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -530,7 +530,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(path); + $$ = new ExprPath(std::move(path)); } ; From 1eeea01931d20d32c917e92d860513b7cb32ba10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christina=20S=C3=B8rensen?= Date: Wed, 27 Sep 2023 02:07:43 +0000 Subject: [PATCH 083/402] Fix repl.md duplicate typo Seems like `legacyPackages.x86_64-linux.emacs.name` is accidentally shown twice. --- src/nix/repl.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/nix/repl.md b/src/nix/repl.md index c5113be61..9dc61fd35 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -43,9 +43,6 @@ R""( nix-repl> legacyPackages.x86_64-linux.emacs.name "emacs-27.1" - nix-repl> legacyPackages.x86_64-linux.emacs.name - "emacs-27.1" - nix-repl> :q # nix repl --expr 'import {}' From 399ef8442012308f480ea5841fb6ac732207e03d Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 25 Sep 2023 21:30:41 -0400 Subject: [PATCH 084/402] refactor: use string accessors Create context, string_view, and c_str, accessors throughout in order to better support improvements to the underlying string representation. --- src/libcmd/repl.cc | 2 +- src/libexpr/eval-cache.cc | 8 ++++---- src/libexpr/eval.cc | 18 +++++++++--------- src/libexpr/flake/flake.cc | 10 +++++----- src/libexpr/get-drvs.cc | 12 ++++++------ src/libexpr/primops.cc | 10 +++++----- src/libexpr/primops/fetchClosure.cc | 2 +- src/libexpr/tests/primops.cc | 4 ++-- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 6 +++--- src/libexpr/value.hh | 20 +++++++++++++++----- src/nix-env/nix-env.cc | 6 +++--- src/nix/eval.cc | 2 +- 13 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 944609f22..2e17a29a7 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -922,7 +922,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m case nString: str << ANSI_WARNING; - printLiteralString(str, v.string.s); + printLiteralString(str, v.string_view()); str << ANSI_NORMAL; break; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2c9aa5532..391b32a77 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -440,8 +440,8 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), - string_t{v.string.s, {}}}; + cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), + string_t{v.c_str(), {}}}; else if (v.type() == nPath) { auto path = v.path().path; cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; @@ -582,7 +582,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); - return v.type() == nString ? v.string.s : v.path().to_string(); + return v.type() == nString ? v.c_str() : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -624,7 +624,7 @@ string_t AttrCursor::getStringWithContext() if (v.type() == nString) { NixStringContext context; copyContext(v, context); - return {v.string.s, std::move(context)}; + return {v.c_str(), std::move(context)}; } else if (v.type() == nPath) return {v.path().to_string(), {}}; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index afa864730..a78c0afb1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -114,7 +114,7 @@ void Value::print(const SymbolTable &symbols, std::ostream &str, printLiteralBool(str, boolean); break; case tString: - printLiteralString(str, string.s); + printLiteralString(str, string_view()); break; case tPath: str << path().to_string(); // !!! escaping? @@ -339,7 +339,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) Value nameValue; name.expr->eval(state, env, nameValue); state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); - return state.symbols.create(nameValue.string.s); + return state.symbols.create(nameValue.string_view()); } } @@ -1343,7 +1343,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); - auto nameSym = state.symbols.create(nameVal.string.s); + auto nameSym = state.symbols.create(nameVal.string_view()); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); @@ -2155,7 +2155,7 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string forceValue(v, pos); if (v.type() != nString) error("value is %1% while a string was expected", showType(v)).debugThrow(); - return v.string.s; + return v.string_view(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -2182,8 +2182,8 @@ std::string_view EvalState::forceString(Value & v, NixStringContext & context, c std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); - if (v.string.context) { - error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]).withTrace(pos, errorCtx).debugThrow(); + if (v.context()) { + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2196,7 +2196,7 @@ bool EvalState::isDerivation(Value & v) if (i == v.attrs->end()) return false; forceValue(*i->value, i->pos); if (i->value->type() != nString) return false; - return strcmp(i->value->string.s, "derivation") == 0; + return i->value->string_view().compare("derivation") == 0; } @@ -2228,7 +2228,7 @@ BackedStringView EvalState::coerceToString( if (v.type() == nString) { copyContext(v, context); - return std::string_view(v.string.s); + return v.string_view(); } if (v.type() == nPath) { @@ -2426,7 +2426,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean == v2.boolean; case nString: - return strcmp(v1.string.s, v2.string.s) == 0; + return v1.string_view().compare(v2.string_view()) == 0; case nPath: return strcmp(v1._path, v2._path) == 0; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 90427e064..a6212c12f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -113,7 +113,7 @@ static FlakeInput parseFlakeInput(EvalState & state, try { if (attr.name == sUrl) { expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string.s; + url = attr.value->string_view(); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); @@ -122,7 +122,7 @@ static FlakeInput parseFlakeInput(EvalState & state, input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); - auto follows(parseInputPath(attr.value->string.s)); + auto follows(parseInputPath(attr.value->c_str())); follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { @@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state, #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(state.symbols[attr.name], attr.value->string.s); + attrs.emplace(state.symbols[attr.name], attr.value->c_str()); break; case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); @@ -229,7 +229,7 @@ static Flake getFlake( if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); - flake.description = description->value->string.s; + flake.description = description->value->c_str(); } auto sInputs = state.symbols.create("inputs"); @@ -850,7 +850,7 @@ static void prim_flakeRefToString( Explicit { attr.value->boolean }); } else if (t == nString) { attrs.emplace(state.symbols[attr.name], - std::string(attr.value->str())); + std::string(attr.value->string_view())); } else { state.error( "flake reference attribute sets may only contain integers, Booleans, " diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 506a63677..fe3e6f7ee 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -156,7 +156,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall Outputs result; for (auto elem : outTI->listItems()) { if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->string.s); + auto out = outputs.find(elem->c_str()); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -230,7 +230,7 @@ std::string DrvInfo::queryMetaString(const std::string & name) { Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; - return v->string.s; + return v->c_str(); } @@ -242,7 +242,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->string.s)) + if (auto n = string2Int(v->c_str())) return *n; } return def; @@ -256,7 +256,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - if (auto n = string2Float(v->string.s)) + if (auto n = string2Float(v->c_str())) return *n; } return def; @@ -271,8 +271,8 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def) if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ - if (strcmp(v->string.s, "true") == 0) return true; - if (strcmp(v->string.s, "false") == 0) return false; + if (v->string_view() == "true") return true; + if (v->string_view() == "false") return false; } return def; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cd9a05bb2..1cb5cdf30 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -590,7 +590,7 @@ struct CompareValues case nFloat: return v1->fpoint < v2->fpoint; case nString: - return strcmp(v1->string.s, v2->string.s) < 0; + return v1->string_view().compare(v2->string_view()) < 0; case nPath: return strcmp(v1->_path, v2->_path) < 0; case nList: @@ -982,7 +982,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu { state.forceValue(*args[0], pos); if (args[0]->type() == nString) - printError("trace: %1%", args[0]->string.s); + printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", printValue(state, *args[0])); state.forceValue(*args[1], pos); @@ -1528,7 +1528,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString && arg.str().ends_with("/"); + auto mustBeDir = arg.type() == nString && arg.string_view().ends_with("/"); try { auto checked = state.checkSourcePath(path); @@ -2400,7 +2400,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; }); } static RegisterPrimOp primop_attrNames({ @@ -2541,7 +2541,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); - names.emplace_back(state.symbols.create(elem->string.s), nullptr); + names.emplace_back(state.symbols.create(elem->string_view()), nullptr); } std::sort(names.begin(), names.end()); diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 7fe8203f4..b86ef6b93 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -133,7 +133,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string(""); + bool isEmptyString = attr.value->type() == nString && attr.value->string_view() == ""; if (isEmptyString) { toPath = StorePathOrGap {}; } diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index ce3b5d11f..d820b860e 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -711,14 +711,14 @@ namespace nix { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\""); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(v.string.s, std::string_view("fabir")); + ASSERT_EQ(v.string_view(), "fabir"); } TEST_F(PrimOpTest, concatStringsSep) { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]"); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(std::string_view(v.string.s), "foo%bar%baz"); + ASSERT_EQ(v.string_view(), "foo%bar%baz"); } TEST_F(PrimOpTest, split1) { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index ac3986c87..cbc91f509 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -31,7 +31,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nString: copyContext(v, context); - out = v.string.s; + out = v.c_str(); break; case nPath: diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 2539ad1c1..bd7a4ae30 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -74,7 +74,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nString: /* !!! show the context? */ copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + doc.writeEmptyElement("string", singletonAttrs("value", v.c_str())); break; case nPath: @@ -96,14 +96,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["drvPath"] = drvPath = a->value->string.s; + xmlAttrs["drvPath"] = drvPath = a->value->c_str(); } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["outPath"] = a->value->string.s; + xmlAttrs["outPath"] = a->value->c_str(); } XMLOpenElement _(doc, "derivation", xmlAttrs); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index c44683e50..7a1165cac 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -186,10 +186,9 @@ public: * For canonicity, the store paths should be in sorted order. */ struct { - const char * s; + const char * c_str; const char * * context; // must be in sorted order } string; - const char * _path; Bindings * attrs; struct { @@ -270,7 +269,7 @@ public: inline void mkString(const char * s, const char * * context = 0) { internalType = tString; - string.s = s; + string.c_str = s; string.context = context; } @@ -441,10 +440,21 @@ public: return SourcePath{CanonPath(_path)}; } - std::string_view str() const + std::string_view string_view() const { assert(internalType == tString); - return std::string_view(string.s); + return std::string_view(string.c_str); + } + + const char * const c_str() const + { + assert(internalType == tString); + return string.c_str; + } + + const char * * context() const + { + return string.context; } }; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index b112e8cb3..e455a50fd 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1228,7 +1228,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else { if (v->type() == nString) { attrs2["type"] = "string"; - attrs2["value"] = v->string.s; + attrs2["value"] = v->c_str(); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; @@ -1248,7 +1248,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) for (auto elem : v->listItems()) { if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = elem->string.s; + attrs3["value"] = elem->c_str(); xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { @@ -1260,7 +1260,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if(a.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = a.value->string.s; + attrs3["value"] = a.value->c_str(); xml.writeEmptyElement("string", attrs3); } } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index d880bef0a..b34af34e0 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path, v.string.s); + writeFile(path, v.string_view()); else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); From 5bc540a8ca7ada282839ab58225a1a24cc723c4e Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Wed, 27 Sep 2023 14:49:52 -0500 Subject: [PATCH 085/402] Respect `NOCOLOR` While `nix` has always been respectful towards requests for `NO_COLOR=1`, this change asks represents a new stage of maturity for `nix` - making it also respect quests for `NOCOLOR=1`. This ideally makes the tool more accessible to folks like me, who are exhausted by guessing whether `NO_COLOR` or `NOCOLOR` is the right environment variable to set. <3 --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5a10c69e2..3b4c181e5 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1515,7 +1515,7 @@ bool shouldANSI() { return isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" - && !getEnv("NO_COLOR").has_value(); + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); } std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) From dafa38213bda956525fb692eea33f2ec4fed7ff3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 27 Sep 2023 23:10:39 +0100 Subject: [PATCH 086/402] Update doc/manual/src/advanced-topics/post-build-hook.md Co-authored-by: Valentin Gagarin --- doc/manual/src/advanced-topics/post-build-hook.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index 17b67111c..f7b3b1317 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -69,7 +69,7 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. -> If you want to upload the .drv file too, the `$DRV_PATH` variable +> If you want to upload the `.drv` file too, the `$DRV_PATH` variable > is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: From 13ed5d710674a9467c8959fed12641b40e38c791 Mon Sep 17 00:00:00 2001 From: Ilan Joselevich Date: Tue, 26 Sep 2023 00:31:00 +0200 Subject: [PATCH 087/402] flakes: adopt repl-flake behavior as default --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libutil/experimental-features.cc | 2 ++ src/nix/repl.cc | 2 +- src/nix/repl.md | 2 +- tests/repl.sh | 11 +++-------- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2467e7ccc..a69cad1db 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -3,3 +3,5 @@ - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 6d92222b9..203455b63 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -163,6 +163,8 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReplFlake, .name = "repl-flake", .description = R"( + *Enabled with [`flakes`](#xp-feature-flakes) since 2.19* + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. )", }, diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9677c1b48..63fe3044b 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -47,7 +47,7 @@ struct CmdRepl : RawInstallablesCommand void applyDefaultInstallables(std::vector & rawInstallables) override { - if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes) && !(file) && rawInstallables.size() >= 1) { warn("future versions of Nix will require using `--file` to load a file"); if (rawInstallables.size() > 1) warn("more than one input file is not currently supported"); diff --git a/src/nix/repl.md b/src/nix/repl.md index 9dc61fd35..32c08e24b 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,7 +36,7 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs + # nix repl --extra-experimental-features 'flakes' nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. diff --git a/tests/repl.sh b/tests/repl.sh index bb8b60e50..1b779c1f5 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -105,18 +105,13 @@ testReplResponseNoRegex ' testReplResponse ' drvPath ' '".*-simple.drv"' \ -$testDir/simple.nix +--file $testDir/simple.nix testReplResponse ' drvPath ' '".*-simple.drv"' \ --file $testDir/simple.nix --experimental-features 'ca-derivations' -testReplResponse ' -drvPath -' '".*-simple.drv"' \ ---file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations' - mkdir -p flake && cat < flake/flake.nix { outputs = { self }: { @@ -130,7 +125,7 @@ EOF testReplResponse ' foo + baz ' "3" \ - ./flake ./flake\#bar --experimental-features 'flakes repl-flake' + ./flake ./flake\#bar --experimental-features 'flakes' # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` @@ -143,7 +138,7 @@ sleep 1 # Leave the repl the time to eval 'foo' sed -i 's/beforeChange/afterChange/' flake/flake.nix echo ":reload" echo "changingThing" -) | nix repl ./flake --experimental-features 'flakes repl-flake') +) | nix repl ./flake --experimental-features 'flakes') echo "$replResult" | grepQuiet -s beforeChange echo "$replResult" | grepQuiet -s afterChange From add7c99c3b559e531b7426e7a0a987231e0028ed Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 18 Sep 2023 16:57:18 +0800 Subject: [PATCH 088/402] Include "original" and "locked" in nix flake prefetch --json --- src/nix/flake.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1b..f8df032c8 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1342,6 +1342,8 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(tree.storePath); res["hash"] = hash.to_string(SRI, true); + res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); + res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", From cede94dbf7733c2c64bdc4a4f53645cffdd73379 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 20:51:25 -0400 Subject: [PATCH 089/402] `builtins.fetchTree`: Mark experimental the new way This helps ensure uniform docs/error message. --- src/libexpr/primops/fetchTree.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 46d31a4e9..124165896 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -195,7 +195,6 @@ static void fetchTree( static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - experimentalFeatureSettings.require(Xp::Flakes); fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } @@ -237,12 +236,9 @@ static RegisterPrimOp primop_fetchTree({ ```nix builtins.fetchTree "https://example.com/" ``` - - > **Note** - > - > This function requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. )", - .fun = prim_fetchTree + .fun = prim_fetchTree, + .experimentalFeature = Xp::Flakes, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, From b912f3a9377ce0b7302b8833934bc1af448ea996 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 20:55:41 -0400 Subject: [PATCH 090/402] Move `flakeIdRegex{,S}` from `libutil` to `flakeref.{cc,hh` It isn't used, and doesn't belong in `libutil`. --- src/libexpr/flake/flakeref.cc | 2 ++ src/libexpr/flake/flakeref.hh | 3 +++ src/libutil/url-parts.hh | 3 --- src/libutil/url.cc | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index f7ef19b3f..93ad33a33 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -289,4 +289,6 @@ std::tuple parseFlakeRefWithFragment return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a7c9208c0..ffb2e50de 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -6,6 +6,7 @@ #include "fetchers.hh" #include "outputs-spec.hh" +#include #include namespace nix { @@ -91,5 +92,7 @@ std::tuple parseFlakeRefWithFragment bool allowMissing = false, bool isFlake = true); +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; } diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 98162b0f7..4712c0c91 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -41,7 +41,4 @@ extern std::regex revRegex; /// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index e700c8eaf..2970f8d2d 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -8,7 +8,6 @@ namespace nix { std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript); -std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { From c816c67eed6d8508a84f40ca00ac93f5926b44be Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 21:10:17 -0400 Subject: [PATCH 091/402] Reword some comments/API docs to reflect `libfetcher`'s multiple users It's not just flakes, but also `builtins.fetchTree`. Also try to provide some more info in general. --- src/libfetchers/attrs.hh | 6 ++++++ src/libfetchers/fetchers.cc | 3 ++- src/libfetchers/fetchers.hh | 31 ++++++++++++++++++++----------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 9f885a793..b9a2c824e 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,6 +13,12 @@ namespace nix::fetchers { typedef std::variant> Attr; + +/** + * An `Attrs` can be thought of a JSON object restricted or simplified + * to be "flat", not containing any subcontainers (arrays or objects) + * and also not containing any `null`s. + */ typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e683b9f80..7bffb04ff 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -254,7 +254,8 @@ std::optional Input::getRev() const try { hash = Hash::parseAnyPrefixed(*s); } catch (BadHash &e) { - // Default to sha1 for backwards compatibility with existing flakes + // Default to sha1 for backwards compatibility with existing + // usages (e.g. `builtins.fetchTree` calls or flake inputs). hash = Hash::parseAny(*s, htSHA1); } } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 6e10e9513..b27ac4128 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -22,12 +22,11 @@ struct Tree struct InputScheme; /** - * The Input object is generated by a specific fetcher, based on the - * user-supplied input attribute in the flake.nix file, and contains + * The `Input` object is generated by a specific fetcher, based on + * user-supplied information, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the - * "fromURL()" or "fromAttrs()" static functions which are provided - * the url or attrset specified in the flake file. + * `fromURL()` or `fromAttrs()` static functions. */ struct Input { @@ -44,10 +43,20 @@ struct Input std::optional parent; public: + /** + * Create an `Input` from a URL. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ static Input fromURL(const std::string & url, bool requireTree = true); static Input fromURL(const ParsedURL & url, bool requireTree = true); + /** + * Create an `Input` from a an `Attrs`. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ static Input fromAttrs(Attrs && attrs); ParsedURL toURL() const; @@ -116,13 +125,13 @@ public: /** - * The InputScheme represents a type of fetcher. Each fetcher - * registers with nix at startup time. When processing an input for a - * flake, each scheme is given an opportunity to "recognize" that - * input from the url or attributes in the flake file's specification - * and return an Input object to represent the input if it is - * recognized. The Input object contains the information the fetcher - * needs to actually perform the "fetch()" when called. + * The `InputScheme` represents a type of fetcher. Each fetcher + * registers with nix at startup time. When processing an `Input`, + * each scheme is given an opportunity to "recognize" that + * input from the user-provided url or attributes + * and return an `Input` object to represent the input if it is + * recognized. The `Input` object contains the information the fetcher + * needs to actually perform the `fetch()` when called. */ struct InputScheme { From bfe1308d3ff7f3dfd4facf4318adcb459e57ece9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 21:21:58 -0400 Subject: [PATCH 092/402] Add infra for `InputSchemes` to be experimental --- src/libfetchers/fetchers.cc | 7 +++++++ src/libfetchers/fetchers.hh | 5 +++++ src/libutil/args.hh | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7bffb04ff..e3d3ac562 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -36,6 +36,7 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromURL(url, requireTree); if (res) { + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; fixupInput(*res); return std::move(*res); @@ -50,6 +51,7 @@ Input Input::fromAttrs(Attrs && attrs) for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromAttrs(attrs); if (res) { + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; fixupInput(*res); return std::move(*res); @@ -309,4 +311,9 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } +std::optional InputScheme::experimentalFeature() +{ + return {}; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b27ac4128..b55aabb6f 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -158,6 +158,11 @@ struct InputScheme virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); virtual std::pair fetch(ref store, const Input & input) = 0; + + /** + * Is this `InputScheme` part of an experimental feature? + */ + virtual std::optional experimentalFeature(); }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index d90129796..1e5bac650 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -236,7 +236,7 @@ struct Command : virtual public Args static constexpr Category catDefault = 0; - virtual std::optional experimentalFeature (); + virtual std::optional experimentalFeature(); virtual Category category() { return catDefault; } }; From 89b39520630726b7c48e8cbc065643134352e43d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 21:35:36 -0400 Subject: [PATCH 093/402] Make the indirect fetcher input scheme part of the Flakes XP feature I don't know much about it, but by the number of times "flake" appears in the code it seems like is part of flakes, at least for now. --- src/libfetchers/indirect.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 4874a43ff..e09d8bfb8 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -98,6 +98,11 @@ struct IndirectInputScheme : InputScheme { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } + + std::optional experimentalFeature() override + { + return Xp::Flakes; + } }; static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From 1dd03c62ad8f09b50dd77c18b973d4aee2112f7a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Sep 2023 10:46:42 +0200 Subject: [PATCH 094/402] add hint for troubleshooting tests --- doc/manual/src/contributing/testing.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 881118505..4a7f1c15c 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -130,6 +130,15 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. +### Troubleshooting + +Sometimes running tests in the development shell may leave artefacts in the local repository. +To remove any traces of that: + +```console +git clean -x --force tests +``` + ### Characterization testing Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. From 08145a5be5cbc963a7006f65d9417f710b3fb8b5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Sep 2023 16:17:51 +0200 Subject: [PATCH 095/402] contributor guide: emphasize solving a well-specified problem with each pull request this moves the orientation step to the beginning, and adds notes how to make sure that a problem is well-spefified and the according change more likely to get accepted Co-authored-by: Robert Hensing --- CONTRIBUTING.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index facbf0eb0..4a9cff3b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,25 +24,30 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). ## Making changes to Nix -1. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. - There are many open pull requests that might already do what you intent to work on. - You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. - -2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. +1. Search for related issues that cover what you're going to work on. + It could help to mention there that you will work on the issue. Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. - Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. + Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. + + If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. + +2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. + There are many open pull requests that might already do what you intend to work on. + You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). -4. Make your changes! +4. Make your change! 5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes. - * Link related issues in your pull request to inform interested parties and future contributors about your change. + * Clearly explain the problem that you're solving. + + Link related issues to inform interested parties and future contributors about your change. + If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). - If your pull request closes one or multiple issues, note that in the description using `Closes: #`, as it will then happen automatically when your change is merged. * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. 6. Do not expect your pull request to be reviewed immediately. From f8a3893e8d77ce4a6e23719a0b2d88464cb84b9c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 2 Sep 2023 22:47:38 +0200 Subject: [PATCH 096/402] pathExists: isDir when endswith /. --- src/libexpr/primops.cc | 4 +++- tests/lang/eval-okay-pathexists.nix | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cb1f99acb..5de1b2828 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1528,7 +1528,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString && arg.string_view().ends_with("/"); + auto mustBeDir = arg.type() == nString + && (arg.string_view().ends_with("/") + || arg.string_view().ends_with("/.")); try { auto checked = state.checkSourcePath(path); diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix index e1246e370..c5e7a62de 100644 --- a/tests/lang/eval-okay-pathexists.nix +++ b/tests/lang/eval-okay-pathexists.nix @@ -2,6 +2,27 @@ builtins.pathExists (./lib.nix) && builtins.pathExists (builtins.toPath ./lib.nix) && builtins.pathExists (builtins.toString ./lib.nix) && !builtins.pathExists (builtins.toString ./lib.nix + "/") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/.") +# FIXME +# && !builtins.pathExists (builtins.toString ./lib.nix + "/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/a/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/../lib.nix") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./.") +&& builtins.pathExists (builtins.toString ./.. + "/lang/lib.nix") +&& !builtins.pathExists (builtins.toString ./.. + "lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./lib.nix") +&& builtins.pathExists (builtins.toString ./.) +&& builtins.pathExists (builtins.toString ./. + "/") +&& builtins.pathExists (builtins.toString ./. + "/../lang") +&& builtins.pathExists (builtins.toString ./. + "/../lang/") +&& builtins.pathExists (builtins.toString ./. + "/../lang/.") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./") +&& builtins.pathExists (builtins.toString ./. + "/../lang//./") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..") +&& builtins.pathExists (builtins.toString ./. + "/../lang/../") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..//") && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) && builtins.pathExists ./lib.nix From bfdd908f7dee5ae04c6a501167ea0fe1f4e08591 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 24 Sep 2023 13:19:04 +0200 Subject: [PATCH 097/402] structured attrs: improve support / usage of NIX_ATTRS_{SH,JSON}_FILE In #4770 I implemented proper `nix-shell(1)` support for derivations using `__structuredAttrs = true;`. Back then we decided to introduce two new environment variables, `NIX_ATTRS_SH_FILE` for `.attrs.sh` and `NIX_ATTRS_JSON_FILE` for `.attrs.json`. This was to avoid having to copy these files to `$NIX_BUILD_TOP` in a `nix-shell(1)` session which effectively meant copying these files to the project dir without cleaning up afterwords[1]. On last NixCon I resumed hacking on `__structuredAttrs = true;` by default for `nixpkgs` with a few other folks and getting back to it, I identified a few problems with the how it's used in `nixpkgs`: * A lot of builders in `nixpkgs` don't care about the env vars and assume that `.attrs.sh` and `.attrs.json` are in `$NIX_BUILD_TOP`. The sole reason why this works is that `nix-shell(1)` sources the contents of `.attrs.sh` and then sources `$stdenv/setup` if it exists. This may not be pretty, but it mostly works. One notable difference when using nixpkgs' stdenv as of now is however that `$__structuredAttrs` is set to `1` on regular builds, but set to an empty string in a shell session. Also, `.attrs.json` cannot be used in shell sessions because it can only be accessed by `$NIX_ATTRS_JSON_FILE` and not by `$NIX_BUILD_TOP/.attrs.json`. I considered changing Nix to be compatible with what nixpkgs effectively does, but then we'd have to either move $NIX_BUILD_TOP for shell sessions to a temporary location (and thus breaking a lot of assumptions) or we'd reintroduce all the problems we solved back then by using these two env vars. This is partly because I didn't document these variables back then (mea culpa), so I decided to drop all mentions of `.attrs.{json,sh}` in the manual and only refer to `$NIX_ATTRS_SH_FILE` and `$NIX_ATTRS_JSON_FILE`. The same applies to all our integration tests. Theoretically we could deprecated using `"$NIX_BUILD_TOP"/.attrs.sh` in the future now. * `nix develop` and `nix print-dev-env` don't support this environment variable at all even though they're supposed to be part of the replacement for `nix-shell` - for the drv debugging part to be precise. This isn't a big deal for the vast majority of derivations, i.e. derivations relying on nixpkgs' `stdenv` wiring things together properly. This is because `nix develop` effectively "clones" the derivation and replaces the builder with a script that dumps all of the environment, shell variables, functions etc, so the state of structured attrs being "sourced" is transmitted into the dev shell and most of the time you don't need to worry about `.attrs.sh` not existing because the shell is correctly configured and the if [ -e .attrs.sh ]; then source .attrs.sh; fi is simply omitted. However, this will break when having a derivation that reads e.g. from `.attrs.json` like with import {}; runCommand "foo" { __structuredAttrs = true; foo.bar = 23; } '' cat $NIX_ATTRS_JSON_FILE # doesn't work because it points to /build/.attrs.json '' To work around this I employed a similar approach as it exists for `nix-shell`: the `NIX_ATTRS_{JSON,SH}_FILE` vars are replaced with temporary locations. The contents of `.attrs.sh` and `.attrs.json` are now written into the JSON by `get-env.sh`, the builder that `nix develop` injects into the derivation it's debugging. So finally the exact file contents are present and exported by `nix develop`. I also made `.attrs.json` a JSON string in the JSON printed by `get-env.sh` on purpose because then it's not necessary to serialize the object structure again. `nix develop` only needs the JSON as string because it's only written into the temporary file. I'm not entirely sure if it makes sense to also use a temporary location for `nix print-dev-env` (rather than just skipping the rewrite in there), but this would probably break certain cases where it's relied upon `$NIX_ATTRS_SH_FILE` to exist (prime example are the `nix print-dev-env` test-cases I wrote in this patch using `tests/shell.nix`, these would fail because the env var exists, but it cannot read from it). [1] https://github.com/NixOS/nix/pull/4770#issuecomment-836799719 --- .../src/language/advanced-attributes.md | 17 ++-- src/nix/develop.cc | 86 +++++++++++++++++-- src/nix/get-env.sh | 20 ++++- tests/build-hook-ca-fixed.nix | 5 +- tests/build-hook.nix | 5 +- tests/config.nix.in | 5 +- tests/failing.nix | 5 +- tests/hermetic.nix | 5 +- tests/structured-attrs.sh | 14 ++- 9 files changed, 141 insertions(+), 21 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 86f0f24b5..7d97b24b6 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -273,18 +273,21 @@ Derivations can declare some infrequently used optional attributes. - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation - attributes are serialised in JSON format and made available to the - builder via the file `.attrs.json` in the builder’s temporary - directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files - have no size restrictions, unlike process environments. + attributes are serialised into a file in JSON format. The environment variable + `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build + and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for + [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, + unlike process environments. It also makes it possible to tweak derivation settings in a structured way; see [`outputChecks`](#adv-attr-outputChecks) for example. As a convenience to Bash builders, - Nix writes a script named `.attrs.sh` to the builder’s directory - that initialises shell variables corresponding to all attributes - that are representable in Bash. This includes non-nested + Nix writes a script that initialises shell variables + corresponding to all attributes that are representable in Bash. The + environment variable `NIX_ATTRS_SH_FILE` points to the exact + location of the script, both in a build and a + [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. diff --git a/src/nix/develop.cc b/src/nix/develop.cc index c01ef1a2a..b080a3939 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,9 +8,12 @@ #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" +#include "util.hh" +#include #include #include +#include using namespace nix; @@ -51,6 +54,7 @@ struct BuildEnvironment std::map vars; std::map bashFunctions; + std::optional> structuredAttrs; static BuildEnvironment fromJSON(std::string_view in) { @@ -74,6 +78,10 @@ struct BuildEnvironment res.bashFunctions.insert({name, def}); } + if (json.contains("structuredAttrs")) { + res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]}; + } + return res; } @@ -102,6 +110,13 @@ struct BuildEnvironment res["bashFunctions"] = bashFunctions; + if (providesStructuredAttrs()) { + auto contents = nlohmann::json::object(); + contents[".attrs.sh"] = getAttrsSH(); + contents[".attrs.json"] = getAttrsJSON(); + res["structuredAttrs"] = std::move(contents); + } + auto json = res.dump(); assert(BuildEnvironment::fromJSON(json) == *this); @@ -109,6 +124,23 @@ struct BuildEnvironment return json; } + bool providesStructuredAttrs() const + { + return structuredAttrs.has_value(); + } + + std::string getAttrsJSON() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->first; + } + + std::string getAttrsSH() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->second; + } + void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { @@ -291,6 +323,7 @@ struct Common : InstallableCommand, MixProfile std::string makeRcScript( ref store, const BuildEnvironment & buildEnvironment, + const Path & tmpDir, const Path & outputsDir = absPath(".") + "/outputs") { // A list of colon-separated environment variables that should be @@ -353,9 +386,48 @@ struct Common : InstallableCommand, MixProfile } } + if (buildEnvironment.providesStructuredAttrs()) { + fixupStructuredAttrs( + "sh", + "NIX_ATTRS_SH_FILE", + buildEnvironment.getAttrsSH(), + rewrites, + buildEnvironment, + tmpDir + ); + fixupStructuredAttrs( + "json", + "NIX_ATTRS_JSON_FILE", + buildEnvironment.getAttrsJSON(), + rewrites, + buildEnvironment, + tmpDir + ); + } + return rewriteStrings(script, rewrites); } + /** + * Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file + * that's accessible from the interactive shell session. + */ + void fixupStructuredAttrs( + const std::string & ext, + const std::string & envVar, + const std::string & content, + StringMap & rewrites, + const BuildEnvironment & buildEnvironment, + const Path & tmpDir) + { + auto targetFilePath = tmpDir + "/.attrs." + ext; + writeFile(targetFilePath, content); + + auto fileInBuilderEnv = buildEnvironment.vars.find(envVar); + assert(fileInBuilderEnv != buildEnvironment.vars.end()); + rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath}); + } + Strings getDefaultFlakeAttrPaths() override { Strings paths{ @@ -487,7 +559,9 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - auto script = makeRcScript(store, buildEnvironment); + AutoDelete tmpDir(createTempDir("", "nix-develop"), true); + + auto script = makeRcScript(store, buildEnvironment, (Path) tmpDir); if (verbosity >= lvlDebug) script += "set -x\n"; @@ -615,10 +689,12 @@ struct CmdPrintDevEnv : Common, MixJSON stopProgressBar(); - logger->writeToStdout( - json - ? buildEnvironment.toJSON() - : makeRcScript(store, buildEnvironment)); + if (json) { + logger->writeToStdout(buildEnvironment.toJSON()); + } else { + AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true); + logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir)); + } } }; diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a7a8a01b9..832cc2f11 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,5 +1,5 @@ set -e -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 @@ -101,7 +101,21 @@ __dumpEnv() { printf "}" done < <(printf "%s\n" "$__vars") - printf '\n }\n}' + printf '\n }' + + if [ -e "$NIX_ATTRS_SH_FILE" ]; then + printf ',\n "structuredAttrs": {\n ' + __escapeString ".attrs.sh" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_SH_FILE")" + printf ',\n ' + __escapeString ".attrs.json" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_JSON_FILE")" + printf '\n }' + fi + + printf '\n}' } __escapeString() { @@ -117,7 +131,7 @@ __escapeString() { # In case of `__structuredAttrs = true;` the list of outputs is an associative # array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` # must contain the array's keys (hence `${!...[@]}`) in this case. -if [ -e .attrs.sh ]; then +if [ -e "$NIX_ATTRS_SH_FILE" ]; then __olist="${!outputs[@]}" else __olist=$outputs diff --git a/tests/build-hook-ca-fixed.nix b/tests/build-hook-ca-fixed.nix index 4cb9e85d1..0ce6d9b12 100644 --- a/tests/build-hook-ca-fixed.nix +++ b/tests/build-hook-ca-fixed.nix @@ -8,7 +8,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; outputHashMode = "recursive"; outputHashAlgo = "sha256"; } // removeAttrs args ["builder" "meta" "passthru"]) diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 7effd7903..99a13aee4 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/config.nix.in b/tests/config.nix.in index 7facbdcbc..00dc007e1 100644 --- a/tests/config.nix.in +++ b/tests/config.nix.in @@ -20,7 +20,10 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; PATH = path; } // caArgs // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/failing.nix b/tests/failing.nix index 2a0350d4d..d25e2d6b6 100644 --- a/tests/failing.nix +++ b/tests/failing.nix @@ -6,7 +6,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; in diff --git a/tests/hermetic.nix b/tests/hermetic.nix index 4c9d7a51f..0513540f0 100644 --- a/tests/hermetic.nix +++ b/tests/hermetic.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index 378dbc735..f2835a761 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -15,9 +15,21 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result export NIX_BUILD_SHELL=$SHELL env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ - --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + +nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' + +nix print-dev-env -f structured-attrs-shell.nix | grep -q 'NIX_ATTRS_JSON_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grep -q 'NIX_ATTRS_SH_FILE=' +nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE' + +jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" + +test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")" + +test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" From 42e3c6d658ebdb60eeef179d91e71ca311f69da8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 29 Sep 2023 17:14:31 +0200 Subject: [PATCH 098/402] doc: reference NIX_ATTRS_*_FILE vars at the env var reference for drvs --- doc/manual/src/language/derivations.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 4a1b70fd1..d80a7dd48 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -165,6 +165,11 @@ The [`builder`](#attr-builder) is executed as follows: - `NIX_STORE` is set to the path of the top-level Nix store directory (typically, `/nix/store`). + - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` + is set to `true` for the dervation. A detailed explanation of this + behavior can be found in the + [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs). + - For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the Nix store for that output. Each output path is a concatenation From 7a0886e3ccbab7da645dae6ecd015b3084aadc54 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 30 Sep 2023 22:04:38 +0100 Subject: [PATCH 099/402] tests/structured-attrs.sh: grep -q -> grepQuiet --- tests/structured-attrs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index f2835a761..f11992dcd 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -24,8 +24,8 @@ nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list # (which is an associative array in thsi case) being fine. nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' -nix print-dev-env -f structured-attrs-shell.nix | grep -q 'NIX_ATTRS_JSON_FILE=' -nix print-dev-env -f structured-attrs-shell.nix | grep -q 'NIX_ATTRS_SH_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE=' nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE' jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" From 78e886bc5fd9e4d85f8503799540c0b71bb270be Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Sep 2023 15:59:11 +0200 Subject: [PATCH 100/402] refine the maintainer's process to unblock discussions more quickly this addresses that we're too often running into open-ended discussions about attempts to solve problems where neither the problem nor the solution is well-understood enough to make decisions in a reasonable amount of time. this also prevents us from doing more work asynchronously. Co-authored-by: Robert Hensing --- maintainers/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 0d520cb0c..8eeb47e8b 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -96,8 +96,10 @@ What constitutes a trivial pull request is up to maintainers' judgement. Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. This may be where the merit of the change itself or the implementation strategy is contested by a team member. +Whenever the discussion opens up questions about the process or this team's goals, this may indicate that the change is too large in scope. +In that case it is taken off the board to be reconsidered by the author or broken down into smaller pieces that are less far-reaching and can be reviewed independently. -As a general guideline, the order of items is determined as follows: +As a general guideline, the order of items to discuss is determined as follows: - Prioritise pull requests over issues From 8440afbed756254784d9fea3eaab06649dffd390 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 1 Oct 2023 23:29:45 -0400 Subject: [PATCH 101/402] Revert "Adapt scheduler to work with dynamic derivations" This reverts commit 5e3986f59cb58f48186a49dcec7aa317b4787522. This un-implements RFC 92 but fixes the critical bug #9052 which many people are hitting. This is a decent stop-gap until a minimal reproduction of that bug is found and a proper fix can be made. Mostly fixed #9052, but I would like to leave that issue open until we have a regression test, so I can then properly fix the bug (unbreaking RFC 92) later. --- .../create-derivation-and-realise-goal.cc | 157 ------------------ .../create-derivation-and-realise-goal.hh | 96 ----------- src/libstore/build/derivation-goal.cc | 33 +++- src/libstore/build/derivation-goal.hh | 5 +- src/libstore/build/entry-points.cc | 11 +- src/libstore/build/goal.hh | 10 -- src/libstore/build/worker.cc | 110 +++--------- src/libstore/build/worker.hh | 24 --- src/libstore/derived-path-map.cc | 3 - src/libstore/derived-path-map.hh | 7 +- tests/dyn-drv/build-built-drv.sh | 4 +- tests/dyn-drv/dep-built-drv.sh | 4 +- 12 files changed, 55 insertions(+), 409 deletions(-) delete mode 100644 src/libstore/build/create-derivation-and-realise-goal.cc delete mode 100644 src/libstore/build/create-derivation-and-realise-goal.hh diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc deleted file mode 100644 index 60f67956d..000000000 --- a/src/libstore/build/create-derivation-and-realise-goal.cc +++ /dev/null @@ -1,157 +0,0 @@ -#include "create-derivation-and-realise-goal.hh" -#include "worker.hh" - -namespace nix { - -CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref drvReq, - const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs }) - , drvReq(drvReq) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &CreateDerivationAndRealiseGoal::getDerivation; - name = fmt( - "outer obtaining drv from '%s' and then building outputs %s", - drvReq->to_string(worker.store), - std::visit(overloaded { - [&](const OutputsSpec::All) -> std::string { - return "* (all of them)"; - }, - [&](const OutputsSpec::Names os) { - return concatStringsSep(", ", quoteStrings(os)); - }, - }, wantedOutputs.raw)); - trace("created outer"); - - worker.updateProgress(); -} - - -CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal() -{ -} - - -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit(overloaded { - [&](const SingleDerivedPath::Opaque & bo) { - return bo.path; - }, - [&](const SingleDerivedPath::Built & bfd) { - return pathPartOfReq(*bfd.drvPath); - }, - }, req.raw()); -} - - -std::string CreateDerivationAndRealiseGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before "baboon". And - substitution goals and inner derivation goals always happen before - derivation goals (due to "b$"). */ - return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); -} - - -void CreateDerivationAndRealiseGoal::timedOut(Error && ex) -{ -} - - -void CreateDerivationAndRealiseGoal::work() -{ - (this->*state)(); -} - - -void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - auto newWanted = wantedOutputs.union_(outputs); - bool needRestart = !newWanted.isSubsetOf(wantedOutputs); - wantedOutputs = newWanted; - - if (!needRestart) return; - - if (!optDrvPath) - // haven't started steps where the outputs matter yet - return; - worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); -} - - -void CreateDerivationAndRealiseGoal::getDerivation() -{ - trace("outer init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (auto optDrvPath = [this]() -> std::optional { - if (buildMode != bmNormal) return std::nullopt; - - auto drvPath = StorePath::dummy; - try { - drvPath = resolveDerivedPath(worker.store, *drvReq); - } catch (MissingRealisation &) { - return std::nullopt; - } - return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) - ? std::optional { drvPath } - : std::nullopt; - }()) { - trace(fmt("already have drv '%s' for '%s', can go straight to building", - worker.store.printStorePath(*optDrvPath), - drvReq->to_string(worker.store))); - - loadAndBuildDerivation(); - } else { - trace("need to obtain drv we want to build"); - - addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); - - state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation; - if (waitees.empty()) work(); - } -} - - -void CreateDerivationAndRealiseGoal::loadAndBuildDerivation() -{ - trace("outer load and build derivation"); - - if (nrFailed != 0) { - amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); - return; - } - - StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); - /* Build this step! */ - concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); - addWaitee(upcast_goal(concreteDrvGoal)); - state = &CreateDerivationAndRealiseGoal::buildDone; - optDrvPath = std::move(drvPath); - if (waitees.empty()) work(); -} - - -void CreateDerivationAndRealiseGoal::buildDone() -{ - trace("outer build done"); - - buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built { - .drvPath = drvReq, - .outputs = wantedOutputs, - }); - - if (buildResult.success()) - amDone(ecSuccess); - else - amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store))); -} - - -} diff --git a/src/libstore/build/create-derivation-and-realise-goal.hh b/src/libstore/build/create-derivation-and-realise-goal.hh deleted file mode 100644 index ca936fc95..000000000 --- a/src/libstore/build/create-derivation-and-realise-goal.hh +++ /dev/null @@ -1,96 +0,0 @@ -#pragma once - -#include "parsed-derivations.hh" -#include "lock.hh" -#include "store-api.hh" -#include "pathlocks.hh" -#include "goal.hh" - -namespace nix { - -struct DerivationGoal; - -/** - * This goal type is essentially the serial composition (like function - * composition) of a goal for getting a derivation, and then a - * `DerivationGoal` using the newly-obtained derivation. - * - * In the (currently experimental) general inductive case of derivations - * that are themselves build outputs, that first goal will be *another* - * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case - * where the derivation has no provence and is just referred to by - * (content-addressed) store path, that first goal is a - * `SubstitutionGoal`. - * - * If we already have the derivation (e.g. if the evalutator has created - * the derivation locally and then instructured the store to build it), - * we can skip the first goal entirely as a small optimization. - */ -struct CreateDerivationAndRealiseGoal : public Goal -{ - /** - * How to obtain a store path of the derivation to build. - */ - ref drvReq; - - /** - * The path of the derivation, once obtained. - **/ - std::optional optDrvPath; - - /** - * The goal for the corresponding concrete derivation. - **/ - std::shared_ptr concreteDrvGoal; - - /** - * The specific outputs that we need to build. - */ - OutputsSpec wantedOutputs; - - typedef void (CreateDerivationAndRealiseGoal::*GoalState)(); - GoalState state; - - /** - * The final output paths of the build. - * - * - For input-addressed derivations, always the precomputed paths - * - * - For content-addressed derivations, calcuated from whatever the - * hash ends up being. (Note that fixed outputs derivations that - * produce the "wrong" output still install that data under its - * true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - CreateDerivationAndRealiseGoal(ref drvReq, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - virtual ~CreateDerivationAndRealiseGoal(); - - void timedOut(Error && ex) override; - - std::string key() override; - - void work() override; - - /** - * Add wanted outputs to an already existing derivation goal. - */ - void addWantedOutputs(const OutputsSpec & outputs); - - /** - * The states. - */ - void getDerivation(); - void loadAndBuildDerivation(); - void buildDone(); - - JobCategory jobCategory() const override { - return JobCategory::Administration; - }; -}; - -} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6472ecd99..83c0a3135 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::loadDerivation; + state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -164,6 +164,24 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } +void DerivationGoal::getDerivation() +{ + trace("init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { + loadDerivation(); + return; + } + + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); + + state = &DerivationGoal::loadDerivation; +} + + void DerivationGoal::loadDerivation() { trace("loading derivation"); @@ -1498,24 +1516,23 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation) return; auto & fullDrv = *dynamic_cast(drv.get()); - std::optional info = tryGetConcreteDrvGoal(waitee); - if (!info) return; - const auto & [dg, drvReq] = *info; + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; - auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get()); + auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); if (!nodeP) return; auto & outputs = nodeP->value; for (auto & outputName : outputs) { - auto buildResult = dg.get().getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg.get().drvPath), + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { auto i = buildResult.builtOutputs.find(outputName); if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg.get().drvPath, outputName }, + { dg->drvPath, outputName }, i->second.outPath); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 62b122c27..ddb5ee1e3 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -52,10 +52,6 @@ struct InitialOutput { /** * A goal for building some or all of the outputs of a derivation. - * - * The derivation must already be present, either in the store in a drv - * or in memory. If the derivation itself needs to be gotten first, a - * `CreateDerivationAndRealiseGoal` goal must be used instead. */ struct DerivationGoal : public Goal { @@ -235,6 +231,7 @@ struct DerivationGoal : public Goal /** * The states. */ + void getDerivation(); void loadDerivation(); void haveDerivation(); void outputsSubstitutionTried(); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f0f0e5519..13ff22f45 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,6 +1,5 @@ #include "worker.hh" #include "substitution-goal.hh" -#include "create-derivation-and-realise-goal.hh" #include "derivation-goal.hh" #include "local-store.hh" @@ -16,7 +15,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StringSet failed; + StorePathSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -26,10 +25,10 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->drvReq->to_string(*this)); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(i2->drvPath); else if (auto i2 = dynamic_cast(i.get())) - failed.insert(printStorePath(i2->storePath)); + failed.insert(i2->storePath); } } @@ -38,7 +37,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); + throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); } } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 01d3c3491..9af083230 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -49,16 +49,6 @@ enum struct JobCategory { * A substitution an arbitrary store object; it will use network resources. */ Substitution, - /** - * A goal that does no "real" work by itself, and just exists to depend on - * other goals which *do* do real work. These goals therefore are not - * limited. - * - * These goals cannot infinitely create themselves, so there is no risk of - * a "fork bomb" type situation (which would be a problem even though the - * goal do no real work) either. - */ - Administration, }; struct Goal : public std::enable_shared_from_this diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b4a634e7b..37cb86b91 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -2,7 +2,6 @@ #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" -#include "create-derivation-and-realise-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" @@ -42,24 +41,6 @@ Worker::~Worker() } -std::shared_ptr Worker::makeCreateDerivationAndRealiseGoal( - ref drvReq, - const OutputsSpec & wantedOutputs, - BuildMode buildMode) -{ - std::weak_ptr & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value; - std::shared_ptr goal = goal_weak.lock(); - if (!goal) { - goal = std::make_shared(drvReq, wantedOutputs, *this, buildMode); - goal_weak = goal; - wakeUp(goal); - } else { - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -130,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -139,46 +123,24 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) } -template -static void cullMap(std::map & goalMap, F f) -{ - for (auto i = goalMap.begin(); i != goalMap.end();) - if (!f(i->second)) - i = goalMap.erase(i); - else ++i; -} - - template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - cullMap(goalMap, [&](const std::weak_ptr & gp) -> bool { - return gp.lock() != goal; - }); -} - -template -static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap); - -template -static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap) -{ - /* !!! inefficient */ - cullMap(goalMap, [&](DerivedPathMap>::ChildNode & node) -> bool { - if (node.value.lock() == goal) - node.value.reset(); - removeGoal(goal, node.childMap); - return !node.value.expired() || !node.childMap.empty(); - }); + for (auto i = goalMap.begin(); + i != goalMap.end(); ) + if (i->second.lock() == goal) { + auto j = i; ++j; + goalMap.erase(i); + i = j; + } + else ++i; } void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, outerDerivationGoals.map); - else if (auto drvGoal = std::dynamic_pointer_cast(goal)) + if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); @@ -236,19 +198,8 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - nrSubstitutions++; - break; - case JobCategory::Build: - nrLocalBuilds++; - break; - case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; - default: - abort(); - } + if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; + else nrLocalBuilds++; } } @@ -260,20 +211,12 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: + if (goal->jobCategory() == JobCategory::Substitution) { assert(nrSubstitutions > 0); nrSubstitutions--; - break; - case JobCategory::Build: + } else { assert(nrLocalBuilds > 0); nrLocalBuilds--; - break; - case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; - default: - abort(); } } @@ -324,9 +267,9 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { - .drvPath = goal->drvReq, + .drvPath = makeConstantStorePathRef(goal->drvPath), .outputs = goal->wantedOutputs, }); } else if (auto goal = dynamic_cast(i.get())) { @@ -589,19 +532,4 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } -GoalPtr upcast_goal(std::shared_ptr subGoal) -{ - return subGoal; -} - -std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee) -{ - auto * odg = dynamic_cast(&*waitee); - if (!odg) return std::nullopt; - return {{ - std::cref(*odg->concreteDrvGoal), - std::cref(*odg->drvReq), - }}; -} - } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6f6d25d7d..23ad87914 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,7 +4,6 @@ #include "types.hh" #include "lock.hh" #include "store-api.hh" -#include "derived-path-map.hh" #include "goal.hh" #include "realisation.hh" @@ -14,7 +13,6 @@ namespace nix { /* Forward definition. */ -struct CreateDerivationAndRealiseGoal; struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -33,25 +31,9 @@ class DrvOutputSubstitutionGoal; */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); -GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -/** - * The current implementation of impure derivations has - * `DerivationGoal`s accumulate realisations from their waitees. - * Unfortunately, `DerivationGoal`s don't directly depend on other - * goals, but instead depend on `CreateDerivationAndRealiseGoal`s. - * - * We try not to share any of the details of any goal type with any - * other, for sake of modularity and quicker rebuilds. This means we - * cannot "just" downcast and fish out the field. So as an escape hatch, - * we have made the function, written in `worker.cc` where all the goal - * types are visible, and use it instead. - */ - -std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee); - /** * A mapping used to remember for each child process to what goal it * belongs, and file descriptors for receiving log data and output @@ -119,9 +101,6 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ - - DerivedPathMap> outerDerivationGoals; - std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -209,9 +188,6 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::shared_ptr makeCreateDerivationAndRealiseGoal( - ref drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 437b6a71a..5982c04b3 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -51,11 +51,8 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single // instantiations -#include "create-derivation-and-realise-goal.hh" namespace nix { -template struct DerivedPathMap>; - GENERATE_CMP_EXT( template<>, DerivedPathMap>::ChildNode, diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 4a2c90733..4d72b301e 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -20,11 +20,8 @@ namespace nix { * * @param V A type to instantiate for each output. It should probably * should be an "optional" type so not every interior node has to have a - * value. For example, the scheduler uses - * `DerivedPathMap>` to - * remember which goals correspond to which outputs. `* const Something` - * or `std::optional` would also be good choices for - * "optional" types. + * value. `* const Something` or `std::optional` would be + * good choices for "optional" types. */ template struct DerivedPathMap { diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh index 94f3550bd..647be9457 100644 --- a/tests/dyn-drv/build-built-drv.sh +++ b/tests/dyn-drv/build-built-drv.sh @@ -18,6 +18,4 @@ clearStore drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) -out2=$(nix build "${drvDep}^out^out" --no-link) - -test $out1 == $out2 +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/dyn-drv/dep-built-drv.sh index c3daf2c59..4f6e9b080 100644 --- a/tests/dyn-drv/dep-built-drv.sh +++ b/tests/dyn-drv/dep-built-drv.sh @@ -6,6 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) clearStore -out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) +expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" -diff -r $out1 $out2 +# diff -r $out1 $out2 From 632f24166d9d22e329d9d8626fa7dcd6b9ac83ce Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 26 Sep 2023 14:44:15 -0400 Subject: [PATCH 102/402] Test the rest of the worker protocol serializers Part of the `BuildResult` test is commented out because we have caught a roundtrip bug! A future PR will fix the bug and uncomment that test. --- src/libstore/build-result.cc | 18 ++ src/libstore/build-result.hh | 3 + src/libstore/tests/worker-protocol.cc | 189 ++++++++++++++++++ src/libstore/worker-protocol.hh | 4 +- .../libstore/worker-protocol/build-result.bin | Bin 0 -> 144 bytes .../libstore/worker-protocol/drv-output.bin | Bin 0 -> 176 bytes .../worker-protocol/keyed-build-result.bin | Bin 0 -> 264 bytes .../optional-content-address.bin | Bin 0 -> 64 bytes .../worker-protocol/optional-store-path.bin | Bin 0 -> 72 bytes .../worker-protocol/optional-trusted-flag.bin | Bin 0 -> 24 bytes .../libstore/worker-protocol/realisation.bin | Bin 0 -> 520 bytes .../libstore/worker-protocol/set.bin | Bin 0 -> 152 bytes .../libstore/worker-protocol/vector.bin | Bin 0 -> 152 bytes 13 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 src/libstore/build-result.cc create mode 100644 unit-test-data/libstore/worker-protocol/build-result.bin create mode 100644 unit-test-data/libstore/worker-protocol/drv-output.bin create mode 100644 unit-test-data/libstore/worker-protocol/keyed-build-result.bin create mode 100644 unit-test-data/libstore/worker-protocol/optional-content-address.bin create mode 100644 unit-test-data/libstore/worker-protocol/optional-store-path.bin create mode 100644 unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin create mode 100644 unit-test-data/libstore/worker-protocol/realisation.bin create mode 100644 unit-test-data/libstore/worker-protocol/set.bin create mode 100644 unit-test-data/libstore/worker-protocol/vector.bin diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc new file mode 100644 index 000000000..18f519c5c --- /dev/null +++ b/src/libstore/build-result.cc @@ -0,0 +1,18 @@ +#include "build-result.hh" + +namespace nix { + +GENERATE_CMP_EXT( + , + BuildResult, + me->status, + me->errorMsg, + me->timesBuilt, + me->isNonDeterministic, + me->builtOutputs, + me->startTime, + me->stopTime, + me->cpuUser, + me->cpuSystem); + +} diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index b7a56e791..8840fa7e3 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -3,6 +3,7 @@ #include "realisation.hh" #include "derived-path.hh" +#include "comparator.hh" #include #include @@ -100,6 +101,8 @@ struct BuildResult */ std::optional cpuUser, cpuSystem; + DECLARE_CMP(BuildResult); + bool success() { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 4a6ccf7c0..24a06503c 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -6,6 +6,7 @@ #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "derived-path.hh" +#include "build-result.hh" #include "tests/libstore.hh" namespace nix { @@ -136,4 +137,192 @@ CHARACTERIZATION_TEST( }, })) +CHARACTERIZATION_TEST( + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + buildResult, + "build-result", + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, +#if 0 + // This is commented because this test would fail! + // FIXME uncomment this and fix the underlying bug. + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), + }, +#endif + }; + t; + })) + +CHARACTERIZATION_TEST( + keyedBuildResult, + "keyed-build-result", + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + KeyedBuildResult { + { + .status = KeyedBuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + /* .path = */ DerivedPath::Opaque { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx" }, + }, + }, + KeyedBuildResult { + { + .status = KeyedBuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + /* .path = */ DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "out" }, + }, + }, + }; + t; + })) + +CHARACTERIZATION_TEST( + optionalTrustedFlag, + "optional-trusted-flag", + (std::tuple, std::optional, std::optional> { + std::nullopt, + std::optional { Trusted }, + std::optional { NotTrusted }, + })) + +CHARACTERIZATION_TEST( + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 70a5bddb9..b7f42f24d 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -208,10 +208,10 @@ MAKE_WORKER_PROTO(ContentAddress); template<> MAKE_WORKER_PROTO(DerivedPath); template<> -MAKE_WORKER_PROTO(Realisation); -template<> MAKE_WORKER_PROTO(DrvOutput); template<> +MAKE_WORKER_PROTO(Realisation); +template<> MAKE_WORKER_PROTO(BuildResult); template<> MAKE_WORKER_PROTO(KeyedBuildResult); diff --git a/unit-test-data/libstore/worker-protocol/build-result.bin b/unit-test-data/libstore/worker-protocol/build-result.bin new file mode 100644 index 0000000000000000000000000000000000000000..f981fd91f7f32724dc2b9ce1938398aad3d73cdf GIT binary patch literal 144 tcmZQ&fB%U7Sq@}q)U|6h9n2j1!MT?w+C~^gO WO9=?G&ojrYjy`x4ZufnEe#tjMpDPdm literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result.bin b/unit-test-data/libstore/worker-protocol/keyed-build-result.bin new file mode 100644 index 0000000000000000000000000000000000000000..c5b3c7f3669a906fef3ef450e8fc8cf8d87d1d04 GIT binary patch literal 264 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NMMVVzD^wphl+Mdn$V^F1R4C7=go+SH onM0M4Vt-O%kzPtsnPPrv2?GyQKQ4DLL**Hvv>cQ+g3>Sw03m8EMgRZ+ literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/unit-test-data/libstore/worker-protocol/optional-content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..f8cfe65ba27fd78ec7787eed7ebec8dfdead2fba GIT binary patch literal 64 zcmZQzfB#ECTBU06lLKJOBUy literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin new file mode 100644 index 0000000000000000000000000000000000000000..51b239409bc815c19b00cb97f62996fb782f24c3 GIT binary patch literal 24 PcmZQzfB;4)%><3#3x z?rW}!>Uc!$VBY>Sr2mUC`<2<)qY;)1KN Po44)luetw6d9AunL$;dR literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/unit-test-data/libstore/worker-protocol/set.bin new file mode 100644 index 0000000000000000000000000000000000000000..ce11ede7fe7aa061a30b211a18cd7a336f879de8 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obCnXkvMPU52{CpHXOdBEdVDg4g4KThDln(&+h63;a literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/unit-test-data/libstore/worker-protocol/vector.bin new file mode 100644 index 0000000000000000000000000000000000000000..7a37c8cd1093201c56f384747ae9c2aa8aa700f9 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obr{(8^MPU4-#3B?vOdBEdVDg4g4KThDln(&-b^`DK literal 0 HcmV?d00001 From e1af175707007591113b0d6fc56036782b0e347c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 4 Oct 2023 17:36:49 -0400 Subject: [PATCH 103/402] Enable most of the third `BuildResult` worker protocol test This was somewhat of a false alarm. The problem was not that the protocol implementation actually failed to round trip, but that two of the fields were ignored entirely --- not serialized and deserialized at all. For reference, those fields were added in fa68eb367e79297bb1c0451cd92ad18a06edce96. --- src/libstore/tests/worker-protocol.cc | 11 ++++++----- .../libstore/worker-protocol/build-result.bin | Bin 144 -> 744 bytes 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 24a06503c..fa7cbe121 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -187,7 +187,7 @@ CHARACTERIZATION_TEST( "build-result", ({ using namespace std::literals::chrono_literals; - std::tuple t { + std::tuple t { BuildResult { .status = BuildResult::OutputRejected, .errorMsg = "no idea why", @@ -200,9 +200,6 @@ CHARACTERIZATION_TEST( .startTime = 30, .stopTime = 50, }, -#if 0 - // This is commented because this test would fail! - // FIXME uncomment this and fix the underlying bug. BuildResult { .status = BuildResult::Built, .timesBuilt = 1, @@ -230,10 +227,14 @@ CHARACTERIZATION_TEST( }, .startTime = 30, .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. .cpuUser = std::chrono::milliseconds(500s), .cpuSystem = std::chrono::milliseconds(604s), - }, #endif + }, }; t; })) diff --git a/unit-test-data/libstore/worker-protocol/build-result.bin b/unit-test-data/libstore/worker-protocol/build-result.bin index f981fd91f7f32724dc2b9ce1938398aad3d73cdf..b02c706eab8de4d60eadd6afdcc2c888ab605ad3 100644 GIT binary patch literal 744 zcmc(b&uRiO5XSolPxTp!=B%)rxXHS&(2IC2JXe~O6CwQ^%woR?}>#sX4il$bfs;n zmv%`YOQ~vvTo;t-%xH@l(n4vSK8bRZQHc`kI^B)Ih0TkNGRhXy8rqZN5BnYj(ieFo zAJ+t*u7l`;??iPt&V)lzi91dfGZFf@g4iVAZN4+jUVRU7o>ol_o!fedeM@Pl_mAVf T^ROZOQyyvZZF!s Date: Thu, 7 Sep 2023 00:31:33 +0200 Subject: [PATCH 104/402] always show anchors on setting listings refactor the templates for readability --- doc/manual/generate-manpage.nix | 55 ++++++-------------------- doc/manual/generate-settings.nix | 63 ++++++++++++++++++++++++++++++ doc/manual/generate-store-info.nix | 45 +++++++++++++++++++++ doc/manual/local.mk | 6 +-- doc/manual/utils.nix | 57 --------------------------- 5 files changed, 124 insertions(+), 102 deletions(-) create mode 100644 doc/manual/generate-settings.nix create mode 100644 doc/manual/generate-store-info.nix diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 41725aed6..425eea002 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -2,7 +2,8 @@ let inherit (builtins) attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique showSettings; + inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique; + showStoreDocs = import ./generate-store-info.nix; in commandDump: @@ -30,7 +31,7 @@ let ${maybeSubcommands} - ${maybeDocumentation} + ${maybeStoreDocs} ${maybeOptions} ''; @@ -40,15 +41,15 @@ let showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' - `${command}` [*option*...] ${arguments} + `${command}` [*option*...] ${arguments} ''; maybeSubcommands = optionalString (details ? commands && details.commands != {}) - '' - where *subcommand* is one of the following: + '' + where *subcommand* is one of the following: - ${subcommands} - ''; + ${subcommands} + ''; subcommands = if length categories > 1 then listCategories @@ -70,10 +71,11 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - # TODO: move this confusing special case out of here when implementing #8496 - maybeDocumentation = optionalString - (details ? doc) - (replaceStrings ["@stores@"] [storeDocs] details.doc); + # FIXME: this is a hack. + # store parameters should not be part of command documentation to begin + # with, but instead be rendered on separate pages. + maybeStoreDocs = optionalString (details ? doc) + (replaceStrings [ "@stores@" ] [ (showStoreDocs commandInfo.stores) ] details.doc); maybeOptions = let allVisibleOptions = filterAttrs @@ -147,35 +149,4 @@ let " - [${page.command}](command-ref/new-cli/${page.name})"; in concatStringsSep "\n" (map showEntry manpages) + "\n"; - storeDocs = - let - showStore = name: { settings, doc, experimentalFeature }: - let - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To use this store, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): - - ``` - extra-experimental-features = ${experimentalFeature} - ``` - ''; - in '' - ## ${name} - - ${doc} - - ${experimentalFeatureNote} - - **Settings**: - - ${showSettings { useAnchors = false; } settings} - ''; - in concatStrings (attrValues (mapAttrs showStore commandInfo.stores)); - in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; } diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix new file mode 100644 index 000000000..450771f73 --- /dev/null +++ b/doc/manual/generate-settings.nix @@ -0,0 +1,63 @@ +let + inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; + inherit (import ./utils.nix) concatStrings indent optionalString squash; +in + +prefix: settingsInfo: + +let + + showSetting = prefix: setting: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: + let + result = squash '' + - [`${setting}`](#${prefix}-${setting}) + + ${indent " " body} + ''; + + # separate body to cleanly handle indentation + body = '' + ${description} + + ${experimentalFeatureNote} + + **Default:** ${showDefault documentDefault defaultValue} + + ${showAliases aliases} + ''; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To change this setting, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ${setting} = ... + ``` + ''; + + showDefault = documentDefault: defaultValue: + if documentDefault then + # a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is `null` in + # JSON, but that converts to `{ }` here. + if defaultValue == "" || defaultValue == [] || isAttrs defaultValue + then "*empty*" + else if isBool defaultValue then + if defaultValue then "`true`" else "`false`" + else "`${toString defaultValue}`" + else "*machine-specific*"; + + showAliases = aliases: + optionalString (aliases != []) + "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; + + in result; + +in concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix new file mode 100644 index 000000000..7bb2ebad3 --- /dev/null +++ b/doc/manual/generate-store-info.nix @@ -0,0 +1,45 @@ +let + inherit (builtins) attrValues mapAttrs; + inherit (import ./utils.nix) concatStrings optionalString; + showSettings = import ./generate-settings.nix; +in + +storesInfo: + +let + + showStore = name: { settings, doc, experimentalFeature }: + let + + result = '' + ## ${name} + + ${doc} + + ${experimentalFeatureNote} + + ### Settings + + ${showSettings "store-${slug}" settings} + ''; + + # markdown doesn't like spaces in URLs + slug = builtins.replaceStrings [ " " ] [ "-" ] name; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To use this store, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ``` + ''; + in result; + +in concatStrings (attrValues (mapAttrs showStore storesInfo)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index abdfd6a62..927854066 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -96,14 +96,14 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix "opt-" (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 9043dd8cd..849832b2c 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -44,63 +44,6 @@ rec { optionalString = cond: string: if cond then string else ""; - showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: - let - result = squash '' - - ${if useAnchors - then ''[`${name}`](#conf-${name})'' - else ''`${name}`''} - - ${indent " " body} - ''; - - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To change this setting, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ${name} = ... - ``` - ''; - - # separate body to cleanly handle indentation - body = '' - ${description} - - ${experimentalFeatureNote} - - **Default:** ${showDefault documentDefault defaultValue} - - ${showAliases aliases} - ''; - - showDefault = documentDefault: defaultValue: - if documentDefault then - # a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is `null` in - # JSON, but that converts to `{ }` here. - if defaultValue == "" || defaultValue == [] || isAttrs defaultValue - then "*empty*" - else if isBool defaultValue then - if defaultValue then "`true`" else "`false`" - else "`${toString defaultValue}`" - else "*machine-specific*"; - - showAliases = aliases: - optionalString (aliases != []) - "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - - in result; - indent = prefix: s: concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - - showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } From 8232711c9fa0ba831c3554e1d19e6707dd093c39 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 4 Oct 2023 23:07:13 +0200 Subject: [PATCH 105/402] fix wiring of baked-in Nix expressions --- src/nix/local.mk | 2 +- src/nix/main.cc | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/nix/local.mk b/src/nix/local.mk index 20ea29d10..57f8259c4 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -31,7 +31,7 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh +src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md @mkdir -p $$(dirname $@) diff --git a/src/nix/main.cc b/src/nix/main.cc index d05bac68e..031dc2348 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -214,6 +214,22 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) , CanonPath::root), *vUtils); + auto vSettingsInfo = state.allocValue(); + state.cacheFile( + CanonPath("/generate-settings.nix"), CanonPath("/generate-settings.nix"), + state.parseExprFromString( + #include "generate-settings.nix.gen.hh" + , CanonPath::root), + *vSettingsInfo); + + auto vStoreInfo = state.allocValue(); + state.cacheFile( + CanonPath("/generate-store-info.nix"), CanonPath("/generate-store-info.nix"), + state.parseExprFromString( + #include "generate-store-info.nix.gen.hh" + , CanonPath::root), + *vStoreInfo); + auto vDump = state.allocValue(); vDump->mkString(toplevel.dumpCli()); From 24bda0c7b381e1a017023c6f7cb9661fae8560bd Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 4 Oct 2023 16:50:24 +0000 Subject: [PATCH 106/402] StorePath: reject names starting with '.' This has been the behaviour before Nix 2.4. It was dropped in a rewrite in 759947bf72c134592f0ce23d385e48095bd0a301, allowing the creation of store paths that aren't considered valid by older Nix versions or other Nix tooling. Nix 2.4 didn't ship in NixOS until 22.05, and stdenv.mkDerivation in nixpkgs drops leading periods since April 2022, so it's unlikely anyone is relying on the current lax behaviour. Closes #9091. Change-Id: I4a57bd9899e1b0dba56870ae5a1b680918a18ce9 --- src/libstore/path-regex.hh | 2 +- src/libstore/path.cc | 2 ++ src/libstore/tests/path.cc | 9 +++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 4f8dc4c1f..a44e6a2eb 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -3,6 +3,6 @@ namespace nix { -static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)"; +static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)"; } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 552e83114..3c6b9fc10 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -11,6 +11,8 @@ static void checkName(std::string_view path, std::string_view name) if (name.size() > StorePath::MaxPathLen) throw BadStorePath("store path '%s' has a name longer than %d characters", path, StorePath::MaxPathLen); + if (name[0] == '.') + throw BadStorePath("store path '%s' starts with illegal character '.'", path); // See nameRegexStr for the definition for (auto c : name) if (!((c >= '0' && c <= '9') diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index efa35ef2b..5a84d646c 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -39,6 +39,7 @@ TEST_DONT_PARSE(double_star, "**") TEST_DONT_PARSE(star_first, "*,foo") TEST_DONT_PARSE(star_second, "foo,*") TEST_DONT_PARSE(bang, "foo!o") +TEST_DONT_PARSE(dotfile, ".gitignore") #undef TEST_DONT_PARSE @@ -101,8 +102,12 @@ Gen Arbitrary::arbitrary() pre += '-'; break; case 64: - pre += '.'; - break; + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } case 65: pre += '_'; break; From e0e47c0a68f7e1b6b0f9e5065e8113b32b12c212 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 5 Oct 2023 01:20:26 +0200 Subject: [PATCH 107/402] accommodate inconsistent output from `lowdown` the `term` output mode leaves inline HTML around verbatim, while `nroff` mode (used for `man` pages) does not. the correct solution would be to pre-render all output with a more benign tool so we have less liabilities in our own code, but this has to do for now. --- doc/manual/generate-manpage.nix | 22 +++++++++++++--------- doc/manual/generate-settings.nix | 9 ++++++--- doc/manual/generate-store-info.nix | 4 ++-- doc/manual/local.mk | 4 ++-- src/nix/main.cc | 3 ++- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 425eea002..d81eac182 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -6,7 +6,7 @@ let showStoreDocs = import ./generate-store-info.nix; in -commandDump: +inlineHTML: commandDump: let @@ -75,7 +75,7 @@ let # store parameters should not be part of command documentation to begin # with, but instead be rendered on separate pages. maybeStoreDocs = optionalString (details ? doc) - (replaceStrings [ "@stores@" ] [ (showStoreDocs commandInfo.stores) ] details.doc); + (replaceStrings [ "@stores@" ] [ (showStoreDocs inlineHTML commandInfo.stores) ] details.doc); maybeOptions = let allVisibleOptions = filterAttrs @@ -84,14 +84,14 @@ let in optionalString (allVisibleOptions != {}) '' # Options - ${showOptions allVisibleOptions} + ${showOptions inlineHTML allVisibleOptions} > **Note** > > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; - showOptions = allOptions: + showOptions = inlineHTML: allOptions: let showCategory = cat: opts: '' ${optionalString (cat != "") "## ${cat}"} @@ -100,17 +100,21 @@ let ''; showOption = name: option: let + result = trim '' + - ${item} + + ${option.description} + ''; + item = if inlineHTML + then ''[`--${name}`](#opt-${name}) ${shortName} ${labels}'' + else "`--${name}` ${shortName} ${labels}"; shortName = optionalString (option ? shortName) ("/ `-${option.shortName}`"); labels = optionalString (option ? labels) (concatStringsSep " " (map (s: "*${s}*") option.labels)); - in trim '' - - [`--${name}`](#opt-${name}) ${shortName} ${labels} - - ${option.description} - ''; + in result; categories = mapAttrs # Convert each group from a list of key-value pairs back to an attrset (_: listToAttrs) diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 450771f73..8736bb793 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -3,18 +3,21 @@ let inherit (import ./utils.nix) concatStrings indent optionalString squash; in -prefix: settingsInfo: +# `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` +{ prefix, inlineHTML ? true }: settingsInfo: let showSetting = prefix: setting: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: let result = squash '' - - [`${setting}`](#${prefix}-${setting}) + - ${item} ${indent " " body} ''; - + item = if inlineHTML + then ''[`${setting}`](#${prefix}-${setting})'' + else "`${setting}`"; # separate body to cleanly handle indentation body = '' ${description} diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 7bb2ebad3..36215aadf 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -4,7 +4,7 @@ let showSettings = import ./generate-settings.nix; in -storesInfo: +inlineHTML: storesInfo: let @@ -20,7 +20,7 @@ let ### Settings - ${showSettings "store-${slug}" settings} + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} ''; # markdown doesn't like spaces in URLs diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 927854066..2d1db66f3 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -98,12 +98,12 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix @rm -rf $@ $@.tmp - $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix "opt-" (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "opt-"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix diff --git a/src/nix/main.cc b/src/nix/main.cc index 031dc2348..879baa5f5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -234,7 +234,8 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos); + state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos); + state.callFunction(*vRes, *vDump, *vRes, noPos); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); if (!attr) From 80f734a803d3170eadd51758c71390ff87be22f9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 5 Oct 2023 09:20:42 +0200 Subject: [PATCH 108/402] more specific links to nix.dev --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1440239bc..65daa22e4 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ for more details. ## Installation and first steps -Visit [nix.dev](https://nix.dev) for installation instructions and beginner tutorials. +Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). From eb68454be61bd6e8baf40136ae69c2b5fec3006a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 4 Oct 2023 17:10:54 +0200 Subject: [PATCH 109/402] Don't run the tests that require building if we're not building A couple of tests require building some libraries that depend on Nix, and assume it to be built locally. Don't run these if we only want to run the install tests. This prevents the CI from rebuilding several times Nix (like in https://github.com/NixOS/nix/actions/runs/6404422275/job/17384964033#step:6:6412), thus removing a fair amount of build time. --- flake.nix | 1 + tests/local.mk | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index e3b9477a5..d74085db7 100644 --- a/flake.nix +++ b/flake.nix @@ -260,6 +260,7 @@ testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; + HAVE_LOCAL_NIX_BUILD = false; name = "nix-tests" + optionalString diff --git a/tests/local.mk b/tests/local.mk index 7479b1576..ac418dc0d 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,3 +1,7 @@ +# whether to run the tests that assume that we have a local build of +# Nix +HAVE_LOCAL_NIX_BUILD ?= 1 + nix_tests = \ test-infra.sh \ init.sh \ @@ -104,7 +108,6 @@ nix_tests = \ case-hack.sh \ placeholders.sh \ ssh-relay.sh \ - plugins.sh \ build.sh \ build-delete.sh \ output-normalization.sh \ @@ -120,7 +123,6 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - test-libstoreconsumer.sh \ toString-path.sh \ read-only-store.sh \ nested-sandboxing.sh @@ -129,6 +131,17 @@ ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif +ifeq ($(HAVE_LOCAL_NIX_BUILD), 1) + nix_tests += test-libstoreconsumer.sh + + ifeq ($(BUILD_SHARED_LIBS), 1) + nix_tests += plugins.sh + endif +endif + +tests/test-libstoreconsumer.sh.test: tests/test-libstoreconsumer/test-libstoreconsumer +tests/plugins.sh: tests/plugins/libplugintest.$(SO_EXT) + install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) clean-files += \ @@ -137,9 +150,4 @@ clean-files += \ test-deps += \ tests/common/vars-and-functions.sh \ - tests/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer - -ifeq ($(BUILD_SHARED_LIBS), 1) - test-deps += tests/plugins/libplugintest.$(SO_EXT) -endif + tests/config.nix From 92e8e1b1bbc59d3a7e9efd5bff146b0a2726c7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 4 Oct 2023 17:22:45 +0200 Subject: [PATCH 110/402] Poison the build on the test derivation Make sure that we're not accidentally rebuilding Nix here as it's just wasteful and awful for CI times. --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index d74085db7..4de155362 100644 --- a/flake.nix +++ b/flake.nix @@ -280,9 +280,13 @@ enableParallelBuilding = true; configureFlags = testConfigureFlags; # otherwise configure fails - dontBuild = true; doInstallCheck = true; + buildPhase = '' + # Remove the source files to make sure that we're not accidentally rebuilding Nix + rm src/**/*.cc + ''; + installPhase = '' mkdir -p $out ''; From f95364a803ddf694e4d7a7f6841101b93d9741fe Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 1 Sep 2023 12:20:54 +0200 Subject: [PATCH 111/402] eval: Run a full GC before printing stats This makes the numbers more deterministic, especially when it comes to the final heap size. --- src/libexpr/eval.cc | 212 +++++++++++++++++++++++++------------------- src/libexpr/eval.hh | 18 +++- 2 files changed, 136 insertions(+), 94 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a78c0afb1..600444955 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2477,10 +2477,37 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v } } +bool EvalState::fullGC() { +#if HAVE_BOEHMGC + GC_gcollect(); + // Check that it ran. We might replace this with a version that uses more + // of the boehm API to get this reliably, at a maintenance cost. + // We use a 1K margin because technically this has a race condtion, but we + // probably won't encounter it in practice, because the CLI isn't concurrent + // like that. + return GC_get_bytes_since_gc() < 1024; +#else + return false; +#endif +} + void EvalState::printStats() { bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; + if (showStats) { + // Make the final heap size more deterministic. +#if HAVE_BOEHMGC + if (!fullGC()) { + warn("failed to perform a full GC before reporting stats"); + } +#endif + printStatistics(); + } +} + +void EvalState::printStatistics() +{ struct rusage buf; getrusage(RUSAGE_SELF, &buf); float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); @@ -2494,106 +2521,105 @@ void EvalState::printStats() GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); #endif - if (showStats) { - auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); - std::fstream fs; - if (outPath != "-") - fs.open(outPath, std::fstream::out); - json topObj = json::object(); - topObj["cpuTime"] = cpuTime; - topObj["envs"] = { - {"number", nrEnvs}, - {"elements", nrValuesInEnvs}, - {"bytes", bEnvs}, - }; - topObj["nrExprs"] = Expr::nrExprs; - topObj["list"] = { - {"elements", nrListElems}, - {"bytes", bLists}, - {"concats", nrListConcats}, - }; - topObj["values"] = { - {"number", nrValues}, - {"bytes", bValues}, - }; - topObj["symbols"] = { - {"number", symbols.size()}, - {"bytes", symbols.totalSize()}, - }; - topObj["sets"] = { - {"number", nrAttrsets}, - {"bytes", bAttrsets}, - {"elements", nrAttrsInAttrsets}, - }; - topObj["sizes"] = { - {"Env", sizeof(Env)}, - {"Value", sizeof(Value)}, - {"Bindings", sizeof(Bindings)}, - {"Attr", sizeof(Attr)}, - }; - topObj["nrOpUpdates"] = nrOpUpdates; - topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; - topObj["nrThunks"] = nrThunks; - topObj["nrAvoided"] = nrAvoided; - topObj["nrLookups"] = nrLookups; - topObj["nrPrimOpCalls"] = nrPrimOpCalls; - topObj["nrFunctionCalls"] = nrFunctionCalls; + + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + json topObj = json::object(); + topObj["cpuTime"] = cpuTime; + topObj["envs"] = { + {"number", nrEnvs}, + {"elements", nrValuesInEnvs}, + {"bytes", bEnvs}, + }; + topObj["nrExprs"] = Expr::nrExprs; + topObj["list"] = { + {"elements", nrListElems}, + {"bytes", bLists}, + {"concats", nrListConcats}, + }; + topObj["values"] = { + {"number", nrValues}, + {"bytes", bValues}, + }; + topObj["symbols"] = { + {"number", symbols.size()}, + {"bytes", symbols.totalSize()}, + }; + topObj["sets"] = { + {"number", nrAttrsets}, + {"bytes", bAttrsets}, + {"elements", nrAttrsInAttrsets}, + }; + topObj["sizes"] = { + {"Env", sizeof(Env)}, + {"Value", sizeof(Value)}, + {"Bindings", sizeof(Bindings)}, + {"Attr", sizeof(Attr)}, + }; + topObj["nrOpUpdates"] = nrOpUpdates; + topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; + topObj["nrThunks"] = nrThunks; + topObj["nrAvoided"] = nrAvoided; + topObj["nrLookups"] = nrLookups; + topObj["nrPrimOpCalls"] = nrPrimOpCalls; + topObj["nrFunctionCalls"] = nrFunctionCalls; #if HAVE_BOEHMGC - topObj["gc"] = { - {"heapSize", heapSize}, - {"totalBytes", totalBytes}, - }; + topObj["gc"] = { + {"heapSize", heapSize}, + {"totalBytes", totalBytes}, + }; #endif - if (countCalls) { - topObj["primops"] = primOpCalls; - { - auto& list = topObj["functions"]; - list = json::array(); - for (auto & [fun, count] : functionCalls) { - json obj = json::object(); - if (fun->name) - obj["name"] = (std::string_view) symbols[fun->name]; - else - obj["name"] = nullptr; - if (auto pos = positions[fun->pos]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = path->to_string(); - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = count; - list.push_back(obj); - } - } - { - auto list = topObj["attributes"]; - list = json::array(); - for (auto & i : attrSelects) { - json obj = json::object(); - if (auto pos = positions[i.first]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = path->to_string(); - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = i.second; - list.push_back(obj); + if (countCalls) { + topObj["primops"] = primOpCalls; + { + auto& list = topObj["functions"]; + list = json::array(); + for (auto & [fun, count] : functionCalls) { + json obj = json::object(); + if (fun->name) + obj["name"] = (std::string_view) symbols[fun->name]; + else + obj["name"] = nullptr; + if (auto pos = positions[fun->pos]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } + obj["count"] = count; + list.push_back(obj); } } + { + auto list = topObj["attributes"]; + list = json::array(); + for (auto & i : attrSelects) { + json obj = json::object(); + if (auto pos = positions[i.first]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; + } + obj["count"] = i.second; + list.push_back(obj); + } + } + } - if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { - // XXX: overrides earlier assignment - topObj["symbols"] = json::array(); - auto &list = topObj["symbols"]; - symbols.dump([&](const std::string & s) { list.emplace_back(s); }); - } - if (outPath == "-") { - std::cerr << topObj.dump(2) << std::endl; - } else { - fs << topObj.dump(2) << std::endl; - } + if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { + // XXX: overrides earlier assignment + topObj["symbols"] = json::array(); + auto &list = topObj["symbols"]; + symbols.dump([&](const std::string & s) { list.emplace_back(s); }); + } + if (outPath == "-") { + std::cerr << topObj.dump(2) << std::endl; + } else { + fs << topObj.dump(2) << std::endl; } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index fa8fa462b..3bfa34ef6 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -709,10 +709,26 @@ public: void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** - * Print statistics. + * Print statistics, if enabled. + * + * Performs a full memory GC before printing the statistics, so that the + * GC statistics are more accurate. */ void printStats(); + /** + * Print statistics, unconditionally, cheaply, without performing a GC first. + */ + void printStatistics(); + + /** + * Perform a full memory garbage collection - not incremental. + * + * @return true if Nix was built with GC and a GC was performed, false if not. + * The return value is currently not thread safe - just the return value. + */ + bool fullGC(); + /** * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path From 369b076986531f3fbf6933539a0379b5d3fb1970 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 6 Oct 2023 11:46:41 +0200 Subject: [PATCH 112/402] add links and anchors --- doc/manual/src/command-ref/nix-env/install.md | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index 95648e9e0..c1fff50e8 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -14,19 +14,21 @@ # Description -The install operation creates a new user environment, based on the -current generation of the active profile, to which a set of store paths -described by *args* is added. The arguments *args* map to store paths in -a number of possible ways: +The install operation creates a new user environment. +It is based on the current generation of the active [profile](@docroot@/command-ref/files/profiles.md), to which a set of [store paths] described by *args* is added. + +[store paths]: @docroot@/glossary.md#gloss-store-path + +The arguments *args* map to store paths in a number of possible ways: - - By default, *args* is a set of [derivation] names denoting derivations - in the active Nix expression. These are realised, and the resulting - output paths are installed. Currently installed derivations with a - name equal to the name of a derivation being added are removed - unless the option `--preserve-installed` is specified. + - By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. + These are [realised], and the resulting output paths are installed. + Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. - [derivation]: @docroot@/language/derivations.md + [derivation]: @docroot@/glossary.md#gloss-derivation + [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md + [realised]: @docroot@/glossary.md#gloss-realise If there are multiple derivations matching a name in *args* that have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the @@ -43,40 +45,33 @@ a number of possible ways: gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will probably cause a user environment conflict\!). - - If `--attr` (`-A`) is specified, the arguments are *attribute - paths* that select attributes from the top-level Nix - expression. This is faster than using derivation names and - unambiguous. To find out the attribute paths of available - packages, use `nix-env --query --available --attr-path `. + - If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. + This is faster than using derivation names and unambiguous. + Show the attribute paths of available packages with [`nix-env --query`](./query.md): + + ```console + nix-env --query --available --attr-path` + ``` - If `--from-profile` *path* is given, *args* is a set of names - denoting installed store paths in the profile *path*. This is an + denoting installed [store paths] in the profile *path*. This is an easy way to copy user environment elements from one profile to another. - - If `--from-expression` is given, *args* are Nix - [functions](@docroot@/language/constructs.md#functions) - that are called with the active Nix expression as their single - argument. The derivations returned by those function calls are - installed. This allows derivations to be specified in an - unambiguous way, which is necessary if there are multiple - derivations with the same name. + - If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/constructs.md#functions) that are called with the [default Nix expression] as their single argument. + The derivations returned by those function calls are installed. + This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. - - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are - [realised](@docroot@/command-ref/nix-store/realise.md), and the resulting output paths - are installed. + - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. - - If *args* are store paths that are not store derivations, then these - are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. + - If *args* are [store paths] that are not store derivations, then these are [realised] and installed. - - By default all outputs are installed for each derivation. + - By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. - - Example: - The file `example.nix` defines a [derivation] with two outputs `foo` and `bar`, each containing a file. + The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. ```nix # example.nix @@ -123,15 +118,17 @@ a number of possible ways: manifest.nix ``` -# Flags +# Options + + - `--prebuilt-only` / `-b` - - `--prebuilt-only` / `-b`\ Use only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can be downloaded in lieu of building the derivation. Thus, no packages will be built from source. - - `--preserve-installed` / `-P`\ + - `--preserve-installed` / `-P` + Do not remove derivations with a name matching one of the derivations being installed. Usually, trying to have two versions of the same package installed in the same generation of a profile will @@ -139,7 +136,8 @@ a number of possible ways: clashes between the two versions. However, this is not the case for all packages. - - `--remove-all` / `-r`\ + - `--remove-all` / `-r` + Remove all previously installed packages first. This is equivalent to running `nix-env --uninstall '.*'` first, except that everything happens in a single transaction. From 68c81c737571794f7246db53fb4774e94fcf4b7e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 5 Oct 2023 12:12:18 -0400 Subject: [PATCH 113/402] Put functional tests in `tests/functional` I think it is bad for these reasons when `tests/` contains a mix of functional and integration tests - Concepts is harder to understand, the documentation makes a good unit vs functional vs integration distinction, but when the integration tests are just two subdirs within `tests/` this is not clear. - Source filtering in the `flake.nix` is more complex. We need to filter out some of the dirs from `tests/`, rather than simply pick the dirs we want and take all of them. This is a good sign the structure of what we are trying to do is not matching the structure of the files. With this change we have a clean: ```shell-session $ git show 'HEAD:tests' tree HEAD:tests functional/ installer/ nixos/ ``` --- .github/labeler.yml | 2 +- .gitignore | 34 ++++++------ CONTRIBUTING.md | 2 +- Makefile | 10 ++-- doc/manual/src/contributing/testing.md | 26 ++++----- flake.nix | 52 ++++++++---------- mk/common-test.sh | 8 ++- src/libstore/gc.cc | 2 +- src/libutil/url-parts.hh | 2 +- tests/{ => functional}/add.sh | 0 tests/{ => functional}/bad.tar.xz | Bin tests/{ => functional}/bash-profile.sh | 2 +- .../{ => functional}/big-derivation-attr.nix | 0 .../binary-cache-build-remote.sh | 0 tests/{ => functional}/binary-cache.sh | 0 tests/{ => functional}/brotli.sh | 0 tests/{ => functional}/build-delete.sh | 0 tests/{ => functional}/build-dry.sh | 0 .../{ => functional}/build-hook-ca-fixed.nix | 0 .../build-hook-ca-floating.nix | 0 tests/{ => functional}/build-hook.nix | 0 .../build-remote-content-addressed-fixed.sh | 0 ...build-remote-content-addressed-floating.sh | 0 .../build-remote-input-addressed.sh | 0 .../build-remote-trustless-after.sh | 0 .../build-remote-trustless-should-fail-0.sh | 0 .../build-remote-trustless-should-pass-0.sh | 0 .../build-remote-trustless-should-pass-1.sh | 0 .../build-remote-trustless-should-pass-2.sh | 0 .../build-remote-trustless-should-pass-3.sh | 0 .../build-remote-trustless.sh | 2 +- tests/{ => functional}/build-remote.sh | 0 tests/{ => functional}/build.sh | 0 tests/{ => functional}/ca-shell.nix | 0 tests/{ => functional}/ca/build-cache.sh | 0 tests/{ => functional}/ca/build-dry.sh | 0 .../ca/build-with-garbage-path.sh | 0 tests/{ => functional}/ca/build.sh | 0 tests/{ => functional}/ca/common.sh | 0 .../{ => functional}/ca/concurrent-builds.sh | 0 tests/{ => functional}/ca/config.nix.in | 0 .../{ => functional}/ca/content-addressed.nix | 0 tests/{ => functional}/ca/derivation-json.sh | 0 .../ca/duplicate-realisation-in-closure.sh | 0 tests/{ => functional}/ca/flake.nix | 0 tests/{ => functional}/ca/gc.sh | 0 .../{ => functional}/ca/import-derivation.sh | 0 tests/{ => functional}/ca/local.mk | 2 +- tests/{ => functional}/ca/new-build-cmd.sh | 0 tests/{ => functional}/ca/nix-copy.sh | 0 tests/{ => functional}/ca/nix-run.sh | 0 tests/{ => functional}/ca/nix-shell.sh | 0 .../{ => functional}/ca/nondeterministic.nix | 0 tests/{ => functional}/ca/post-hook.sh | 0 tests/{ => functional}/ca/racy.nix | 0 tests/{ => functional}/ca/recursive.sh | 0 tests/{ => functional}/ca/repl.sh | 0 tests/{ => functional}/ca/selfref-gc.sh | 0 tests/{ => functional}/ca/signatures.sh | 0 tests/{ => functional}/ca/substitute.sh | 0 tests/{ => functional}/ca/why-depends.sh | 0 tests/{ => functional}/case-hack.sh | 0 tests/{ => functional}/case.nar | Bin tests/{ => functional}/check-refs.nix | 0 tests/{ => functional}/check-refs.sh | 0 tests/{ => functional}/check-reqs.nix | 0 tests/{ => functional}/check-reqs.sh | 0 tests/{ => functional}/check.nix | 0 tests/{ => functional}/check.sh | 0 tests/{ => functional}/common.sh | 0 .../common/vars-and-functions.sh.in | 2 +- tests/{ => functional}/completions.sh | 0 tests/{ => functional}/compression-levels.sh | 0 tests/{ => functional}/compute-levels.sh | 0 tests/{ => functional}/config.nix.in | 0 tests/{ => functional}/config.sh | 0 .../config/nix-with-substituters.conf | 0 tests/{ => functional}/db-migration.sh | 0 .../{ => functional}/dependencies.builder0.sh | 0 tests/{ => functional}/dependencies.nix | 0 tests/{ => functional}/dependencies.sh | 0 tests/{ => functional}/derivation-json.sh | 0 tests/{ => functional}/dummy | 0 tests/{ => functional}/dump-db.sh | 0 .../dyn-drv/build-built-drv.sh | 0 tests/{ => functional}/dyn-drv/common.sh | 0 tests/{ => functional}/dyn-drv/config.nix.in | 0 .../{ => functional}/dyn-drv/dep-built-drv.sh | 0 .../{ => functional}/dyn-drv/eval-outputOf.sh | 0 tests/{ => functional}/dyn-drv/local.mk | 2 +- .../dyn-drv/old-daemon-error-hack.nix | 0 .../dyn-drv/old-daemon-error-hack.sh | 0 .../dyn-drv/recursive-mod-json.nix | 0 .../dyn-drv/recursive-mod-json.sh | 0 .../dyn-drv/text-hashed-output.nix | 0 .../dyn-drv/text-hashed-output.sh | 0 tests/{ => functional}/eval-store.sh | 0 tests/{ => functional}/eval.nix | 0 tests/{ => functional}/eval.sh | 0 .../{ => functional}/experimental-features.sh | 0 tests/{ => functional}/export-graph.nix | 0 tests/{ => functional}/export-graph.sh | 0 tests/{ => functional}/export.sh | 0 tests/{ => functional}/failing.nix | 0 tests/{ => functional}/fetchClosure.sh | 0 tests/{ => functional}/fetchGit.sh | 0 tests/{ => functional}/fetchGitRefs.sh | 0 tests/{ => functional}/fetchGitSubmodules.sh | 0 tests/{ => functional}/fetchMercurial.sh | 0 tests/{ => functional}/fetchPath.sh | 0 tests/{ => functional}/fetchTree-file.sh | 0 tests/{ => functional}/fetchurl.sh | 0 tests/{ => functional}/filter-source.nix | 0 tests/{ => functional}/filter-source.sh | 0 tests/{ => functional}/fixed.builder1.sh | 0 tests/{ => functional}/fixed.builder2.sh | 0 tests/{ => functional}/fixed.nix | 0 tests/{ => functional}/fixed.sh | 0 .../flakes/absolute-attr-paths.sh | 0 .../{ => functional}/flakes/absolute-paths.sh | 0 tests/{ => functional}/flakes/build-paths.sh | 0 tests/{ => functional}/flakes/bundle.sh | 0 tests/{ => functional}/flakes/check.sh | 0 tests/{ => functional}/flakes/circular.sh | 0 tests/{ => functional}/flakes/common.sh | 0 tests/{ => functional}/flakes/config.sh | 0 .../flakes/flake-in-submodule.sh | 0 tests/{ => functional}/flakes/flakes.sh | 0 tests/{ => functional}/flakes/follow-paths.sh | 0 tests/{ => functional}/flakes/init.sh | 0 tests/{ => functional}/flakes/inputs.sh | 0 tests/{ => functional}/flakes/mercurial.sh | 0 tests/{ => functional}/flakes/run.sh | 0 tests/{ => functional}/flakes/search-root.sh | 0 tests/{ => functional}/flakes/show.sh | 0 .../flakes/unlocked-override.sh | 0 tests/{ => functional}/fmt.sh | 0 tests/{ => functional}/fmt.simple.sh | 0 tests/{ => functional}/function-trace.sh | 0 tests/{ => functional}/gc-auto.sh | 0 .../{ => functional}/gc-concurrent.builder.sh | 0 tests/{ => functional}/gc-concurrent.nix | 0 tests/{ => functional}/gc-concurrent.sh | 0 .../gc-concurrent2.builder.sh | 0 tests/{ => functional}/gc-non-blocking.sh | 0 tests/{ => functional}/gc-runtime.nix | 0 tests/{ => functional}/gc-runtime.sh | 0 tests/{ => functional}/gc.sh | 0 tests/{ => functional}/hash-check.nix | 0 tests/{ => functional}/hash.sh | 0 tests/{ => functional}/hermetic.nix | 0 tests/{ => functional}/import-derivation.nix | 0 tests/{ => functional}/import-derivation.sh | 0 tests/{ => functional}/impure-derivations.nix | 0 tests/{ => functional}/impure-derivations.sh | 0 tests/{ => functional}/init.sh | 0 tests/{ => functional}/install-darwin.sh | 0 tests/{ => functional}/lang-test-infra.sh | 0 tests/{ => functional}/lang.sh | 2 +- tests/{ => functional}/lang/binary-data | Bin tests/{ => functional}/lang/data | 0 tests/{ => functional}/lang/dir1/a.nix | 0 tests/{ => functional}/lang/dir2/a.nix | 0 tests/{ => functional}/lang/dir2/b.nix | 0 tests/{ => functional}/lang/dir3/a.nix | 0 tests/{ => functional}/lang/dir3/b.nix | 0 tests/{ => functional}/lang/dir3/c.nix | 0 tests/{ => functional}/lang/dir4/a.nix | 0 tests/{ => functional}/lang/dir4/c.nix | 0 tests/{ => functional}/lang/empty.exp | 0 .../lang/eval-fail-abort.err.exp | 0 .../{ => functional}/lang/eval-fail-abort.nix | 0 .../lang/eval-fail-antiquoted-path.err.exp | 0 .../lang/eval-fail-assert.err.exp | 0 .../lang/eval-fail-assert.nix | 0 .../lang/eval-fail-bad-antiquote-1.err.exp | 0 .../lang/eval-fail-bad-antiquote-2.err.exp | 0 .../lang/eval-fail-bad-antiquote-3.err.exp | 0 ...al-fail-bad-string-interpolation-1.err.exp | 0 .../eval-fail-bad-string-interpolation-1.nix | 0 ...al-fail-bad-string-interpolation-2.err.exp | 0 .../eval-fail-bad-string-interpolation-2.nix | 0 ...al-fail-bad-string-interpolation-3.err.exp | 0 .../eval-fail-bad-string-interpolation-3.nix | 0 .../lang/eval-fail-blackhole.err.exp | 0 .../lang/eval-fail-blackhole.nix | 0 .../lang/eval-fail-deepseq.err.exp | 0 .../lang/eval-fail-deepseq.nix | 0 .../lang/eval-fail-dup-dynamic-attrs.err.exp | 0 .../lang/eval-fail-dup-dynamic-attrs.nix | 0 ...-foldlStrict-strict-op-application.err.exp | 0 ...fail-foldlStrict-strict-op-application.nix | 0 .../eval-fail-fromTOML-timestamps.err.exp | 0 .../lang/eval-fail-fromTOML-timestamps.nix | 0 .../lang/eval-fail-hashfile-missing.err.exp | 0 .../lang/eval-fail-hashfile-missing.nix | 0 .../lang/eval-fail-list.err.exp | 0 .../{ => functional}/lang/eval-fail-list.nix | 0 .../lang/eval-fail-missing-arg.err.exp | 0 .../lang/eval-fail-missing-arg.nix | 0 .../lang/eval-fail-nonexist-path.err.exp | 0 .../lang/eval-fail-nonexist-path.nix | 0 .../lang/eval-fail-path-slash.err.exp | 0 .../lang/eval-fail-path-slash.nix | 0 .../lang/eval-fail-recursion.err.exp | 0 .../lang/eval-fail-recursion.nix | 0 .../lang/eval-fail-remove.err.exp | 0 .../lang/eval-fail-remove.nix | 0 .../lang/eval-fail-scope-5.err.exp | 0 .../lang/eval-fail-scope-5.nix | 0 .../lang/eval-fail-seq.err.exp | 0 tests/{ => functional}/lang/eval-fail-seq.nix | 0 .../lang/eval-fail-set-override.err.exp | 0 .../lang/eval-fail-set-override.nix | 0 .../lang/eval-fail-set.err.exp | 0 tests/{ => functional}/lang/eval-fail-set.nix | 0 .../lang/eval-fail-substring.err.exp | 0 .../lang/eval-fail-substring.nix | 0 .../lang/eval-fail-to-path.err.exp | 0 .../lang/eval-fail-to-path.nix | 0 .../lang/eval-fail-toJSON.err.exp | 0 .../lang/eval-fail-toJSON.nix | 0 .../lang/eval-fail-undeclared-arg.err.exp | 0 .../lang/eval-fail-undeclared-arg.nix | 0 .../lang/eval-okay-any-all.exp | 0 .../lang/eval-okay-any-all.nix | 0 .../lang/eval-okay-arithmetic.exp | 0 .../lang/eval-okay-arithmetic.nix | 0 .../lang/eval-okay-attrnames.exp | 0 .../lang/eval-okay-attrnames.nix | 0 .../{ => functional}/lang/eval-okay-attrs.exp | 0 .../{ => functional}/lang/eval-okay-attrs.nix | 0 .../lang/eval-okay-attrs2.exp | 0 .../lang/eval-okay-attrs2.nix | 0 .../lang/eval-okay-attrs3.exp | 0 .../lang/eval-okay-attrs3.nix | 0 .../lang/eval-okay-attrs4.exp | 0 .../lang/eval-okay-attrs4.nix | 0 .../lang/eval-okay-attrs5.exp | 0 .../lang/eval-okay-attrs5.nix | 0 .../lang/eval-okay-attrs6.exp | 0 .../lang/eval-okay-attrs6.nix | 0 .../lang/eval-okay-autoargs.exp | 0 .../lang/eval-okay-autoargs.flags | 0 .../lang/eval-okay-autoargs.nix | 0 .../lang/eval-okay-backslash-newline-1.exp | 0 .../lang/eval-okay-backslash-newline-1.nix | 0 .../lang/eval-okay-backslash-newline-2.exp | 0 .../lang/eval-okay-backslash-newline-2.nix | 0 .../lang/eval-okay-builtins-add.exp | 0 .../lang/eval-okay-builtins-add.nix | 0 .../lang/eval-okay-builtins.exp | 0 .../lang/eval-okay-builtins.nix | 0 .../lang/eval-okay-callable-attrs.exp | 0 .../lang/eval-okay-callable-attrs.nix | 0 .../lang/eval-okay-catattrs.exp | 0 .../lang/eval-okay-catattrs.nix | 0 .../lang/eval-okay-closure.exp | 0 .../lang/eval-okay-closure.exp.xml | 0 .../lang/eval-okay-closure.nix | 0 .../lang/eval-okay-comments.exp | 0 .../lang/eval-okay-comments.nix | 0 .../lang/eval-okay-concat.exp | 0 .../lang/eval-okay-concat.nix | 0 .../lang/eval-okay-concatmap.exp | 0 .../lang/eval-okay-concatmap.nix | 0 .../lang/eval-okay-concatstringssep.exp | 0 .../lang/eval-okay-concatstringssep.nix | 0 .../lang/eval-okay-context-introspection.exp | 0 .../lang/eval-okay-context-introspection.nix | 0 .../lang/eval-okay-context.exp | 0 .../lang/eval-okay-context.nix | 0 .../lang/eval-okay-curpos.exp | 0 .../lang/eval-okay-curpos.nix | 0 .../lang/eval-okay-deepseq.exp | 0 .../lang/eval-okay-deepseq.nix | 0 .../lang/eval-okay-delayed-with-inherit.exp | 0 .../lang/eval-okay-delayed-with-inherit.nix | 0 .../lang/eval-okay-delayed-with.exp | 0 .../lang/eval-okay-delayed-with.nix | 0 .../lang/eval-okay-dynamic-attrs-2.exp | 0 .../lang/eval-okay-dynamic-attrs-2.nix | 0 .../lang/eval-okay-dynamic-attrs-bare.exp | 0 .../lang/eval-okay-dynamic-attrs-bare.nix | 0 .../lang/eval-okay-dynamic-attrs.exp | 0 .../lang/eval-okay-dynamic-attrs.nix | 0 .../{ => functional}/lang/eval-okay-elem.exp | 0 .../{ => functional}/lang/eval-okay-elem.nix | 0 .../lang/eval-okay-empty-args.exp | 0 .../lang/eval-okay-empty-args.nix | 0 .../lang/eval-okay-eq-derivations.exp | 0 .../lang/eval-okay-eq-derivations.nix | 0 tests/{ => functional}/lang/eval-okay-eq.exp | 0 tests/{ => functional}/lang/eval-okay-eq.nix | 0 .../lang/eval-okay-filter.exp | 0 .../lang/eval-okay-filter.nix | 0 .../lang/eval-okay-flake-ref-to-string.exp | 0 .../lang/eval-okay-flake-ref-to-string.nix | 0 .../lang/eval-okay-flatten.exp | 0 .../lang/eval-okay-flatten.nix | 0 .../{ => functional}/lang/eval-okay-float.exp | 0 .../{ => functional}/lang/eval-okay-float.nix | 0 .../lang/eval-okay-floor-ceil.exp | 0 .../lang/eval-okay-floor-ceil.nix | 0 .../eval-okay-foldlStrict-lazy-elements.exp | 0 .../eval-okay-foldlStrict-lazy-elements.nix | 0 ...y-foldlStrict-lazy-initial-accumulator.exp | 0 ...y-foldlStrict-lazy-initial-accumulator.nix | 0 .../lang/eval-okay-foldlStrict.exp | 0 .../lang/eval-okay-foldlStrict.nix | 0 .../lang/eval-okay-fromTOML-timestamps.exp | 0 .../lang/eval-okay-fromTOML-timestamps.flags | 0 .../lang/eval-okay-fromTOML-timestamps.nix | 0 .../lang/eval-okay-fromTOML.exp | 0 .../lang/eval-okay-fromTOML.nix | 0 .../lang/eval-okay-fromjson-escapes.exp | 0 .../lang/eval-okay-fromjson-escapes.nix | 0 .../lang/eval-okay-fromjson.exp | 0 .../lang/eval-okay-fromjson.nix | 0 .../lang/eval-okay-functionargs.exp | 0 .../lang/eval-okay-functionargs.exp.xml | 0 .../lang/eval-okay-functionargs.nix | 0 .../eval-okay-getattrpos-functionargs.exp | 0 .../eval-okay-getattrpos-functionargs.nix | 0 .../lang/eval-okay-getattrpos-undefined.exp | 0 .../lang/eval-okay-getattrpos-undefined.nix | 0 .../lang/eval-okay-getattrpos.exp | 0 .../lang/eval-okay-getattrpos.nix | 0 .../lang/eval-okay-getenv.exp | 0 .../lang/eval-okay-getenv.nix | 0 .../lang/eval-okay-groupBy.exp | 0 .../lang/eval-okay-groupBy.nix | 0 .../{ => functional}/lang/eval-okay-hash.exp | 0 .../lang/eval-okay-hashfile.exp | 0 .../lang/eval-okay-hashfile.nix | 0 .../lang/eval-okay-hashstring.exp | 0 .../lang/eval-okay-hashstring.nix | 0 tests/{ => functional}/lang/eval-okay-if.exp | 0 tests/{ => functional}/lang/eval-okay-if.nix | 0 .../lang/eval-okay-import.exp | 0 .../lang/eval-okay-import.nix | 0 .../lang/eval-okay-ind-string.exp | 0 .../lang/eval-okay-ind-string.nix | 0 .../lang/eval-okay-intersectAttrs.exp | 0 .../lang/eval-okay-intersectAttrs.nix | 0 tests/{ => functional}/lang/eval-okay-let.exp | 0 tests/{ => functional}/lang/eval-okay-let.nix | 0 .../{ => functional}/lang/eval-okay-list.exp | 0 .../{ => functional}/lang/eval-okay-list.nix | 0 .../lang/eval-okay-listtoattrs.exp | 0 .../lang/eval-okay-listtoattrs.nix | 0 .../{ => functional}/lang/eval-okay-logic.exp | 0 .../{ => functional}/lang/eval-okay-logic.nix | 0 tests/{ => functional}/lang/eval-okay-map.exp | 0 tests/{ => functional}/lang/eval-okay-map.nix | 0 .../lang/eval-okay-mapattrs.exp | 0 .../lang/eval-okay-mapattrs.nix | 0 .../lang/eval-okay-merge-dynamic-attrs.exp | 0 .../lang/eval-okay-merge-dynamic-attrs.nix | 0 .../lang/eval-okay-nested-with.exp | 0 .../lang/eval-okay-nested-with.nix | 0 .../lang/eval-okay-new-let.exp | 0 .../lang/eval-okay-new-let.nix | 0 .../lang/eval-okay-null-dynamic-attrs.exp | 0 .../lang/eval-okay-null-dynamic-attrs.nix | 0 .../lang/eval-okay-overrides.exp | 0 .../lang/eval-okay-overrides.nix | 0 .../lang/eval-okay-parse-flake-ref.exp | 0 .../lang/eval-okay-parse-flake-ref.nix | 0 .../lang/eval-okay-partition.exp | 0 .../lang/eval-okay-partition.nix | 0 .../eval-okay-path-string-interpolation.exp | 0 .../eval-okay-path-string-interpolation.nix | 0 .../{ => functional}/lang/eval-okay-path.exp | 0 .../{ => functional}/lang/eval-okay-path.nix | 0 .../lang/eval-okay-pathexists.exp | 0 .../lang/eval-okay-pathexists.nix | 0 .../lang/eval-okay-patterns.exp | 0 .../lang/eval-okay-patterns.nix | 0 .../lang/eval-okay-print.err.exp | 0 .../{ => functional}/lang/eval-okay-print.exp | 0 .../{ => functional}/lang/eval-okay-print.nix | 0 .../lang/eval-okay-readDir.exp | 0 .../lang/eval-okay-readDir.nix | 0 .../lang/eval-okay-readFileType.exp | 0 .../lang/eval-okay-readFileType.nix | 0 .../lang/eval-okay-readfile.exp | 0 .../lang/eval-okay-readfile.nix | 0 .../lang/eval-okay-redefine-builtin.exp | 0 .../lang/eval-okay-redefine-builtin.nix | 0 .../lang/eval-okay-regex-match.exp | 0 .../lang/eval-okay-regex-match.nix | 0 .../lang/eval-okay-regex-split.exp | 0 .../lang/eval-okay-regex-split.nix | 0 .../lang/eval-okay-regression-20220122.exp | 0 .../lang/eval-okay-regression-20220122.nix | 0 .../lang/eval-okay-regression-20220125.exp | 0 .../lang/eval-okay-regression-20220125.nix | 0 .../lang/eval-okay-remove.exp | 0 .../lang/eval-okay-remove.nix | 0 .../lang/eval-okay-replacestrings.exp | 0 .../lang/eval-okay-replacestrings.nix | 0 .../lang/eval-okay-scope-1.exp | 0 .../lang/eval-okay-scope-1.nix | 0 .../lang/eval-okay-scope-2.exp | 0 .../lang/eval-okay-scope-2.nix | 0 .../lang/eval-okay-scope-3.exp | 0 .../lang/eval-okay-scope-3.nix | 0 .../lang/eval-okay-scope-4.exp | 0 .../lang/eval-okay-scope-4.nix | 0 .../lang/eval-okay-scope-6.exp | 0 .../lang/eval-okay-scope-6.nix | 0 .../lang/eval-okay-scope-7.exp | 0 .../lang/eval-okay-scope-7.nix | 0 .../lang/eval-okay-search-path.exp | 0 .../lang/eval-okay-search-path.flags | 0 .../lang/eval-okay-search-path.nix | 0 tests/{ => functional}/lang/eval-okay-seq.exp | 0 tests/{ => functional}/lang/eval-okay-seq.nix | 0 .../{ => functional}/lang/eval-okay-sort.exp | 0 .../{ => functional}/lang/eval-okay-sort.nix | 0 .../lang/eval-okay-splitversion.exp | 0 .../lang/eval-okay-splitversion.nix | 0 .../lang/eval-okay-string.exp | 0 .../lang/eval-okay-string.nix | 0 .../lang/eval-okay-strings-as-attrs-names.exp | 0 .../lang/eval-okay-strings-as-attrs-names.nix | 0 .../lang/eval-okay-substring.exp | 0 .../lang/eval-okay-substring.nix | 0 .../lang/eval-okay-tail-call-1.exp-disabled | 0 .../lang/eval-okay-tail-call-1.nix | 0 .../lang/eval-okay-tojson.exp | 0 .../lang/eval-okay-tojson.nix | 0 .../{ => functional}/lang/eval-okay-toxml.exp | 0 .../{ => functional}/lang/eval-okay-toxml.nix | 0 .../lang/eval-okay-toxml2.exp | 0 .../lang/eval-okay-toxml2.nix | 0 .../lang/eval-okay-tryeval.exp | 0 .../lang/eval-okay-tryeval.nix | 0 .../{ => functional}/lang/eval-okay-types.exp | 0 .../{ => functional}/lang/eval-okay-types.nix | 0 .../lang/eval-okay-versions.exp | 0 .../lang/eval-okay-versions.nix | 0 .../{ => functional}/lang/eval-okay-with.exp | 0 .../{ => functional}/lang/eval-okay-with.nix | 0 .../lang/eval-okay-xml.exp.xml | 0 tests/{ => functional}/lang/eval-okay-xml.nix | 0 .../lang/eval-okay-zipAttrsWith.exp | 0 .../lang/eval-okay-zipAttrsWith.nix | 0 tests/{ => functional}/lang/framework.sh | 0 tests/{ => functional}/lang/imported.nix | 0 tests/{ => functional}/lang/imported2.nix | 0 tests/{ => functional}/lang/lib.nix | 0 .../lang/parse-fail-dup-attrs-1.err.exp | 0 .../lang/parse-fail-dup-attrs-1.nix | 0 .../lang/parse-fail-dup-attrs-2.err.exp | 0 .../lang/parse-fail-dup-attrs-2.nix | 0 .../lang/parse-fail-dup-attrs-3.err.exp | 0 .../lang/parse-fail-dup-attrs-3.nix | 0 .../lang/parse-fail-dup-attrs-4.err.exp | 0 .../lang/parse-fail-dup-attrs-4.nix | 0 .../lang/parse-fail-dup-attrs-6.err.exp | 0 .../lang/parse-fail-dup-attrs-7.err.exp | 0 .../lang/parse-fail-dup-attrs-7.nix | 0 .../lang/parse-fail-dup-formals.err.exp | 0 .../lang/parse-fail-dup-formals.nix | 0 .../lang/parse-fail-eof-in-string.err.exp | 0 .../lang/parse-fail-eof-in-string.nix | 0 .../parse-fail-mixed-nested-attrs1.err.exp | 0 .../lang/parse-fail-mixed-nested-attrs1.nix | 0 .../parse-fail-mixed-nested-attrs2.err.exp | 0 .../lang/parse-fail-mixed-nested-attrs2.nix | 0 .../lang/parse-fail-patterns-1.err.exp | 0 .../lang/parse-fail-patterns-1.nix | 0 .../parse-fail-regression-20060610.err.exp | 0 .../lang/parse-fail-regression-20060610.nix | 0 .../lang/parse-fail-undef-var-2.err.exp | 0 .../lang/parse-fail-undef-var-2.nix | 0 .../lang/parse-fail-undef-var.err.exp | 0 .../lang/parse-fail-undef-var.nix | 0 .../lang/parse-fail-utf8.err.exp | 0 .../{ => functional}/lang/parse-fail-utf8.nix | 0 tests/{ => functional}/lang/parse-okay-1.exp | 0 tests/{ => functional}/lang/parse-okay-1.nix | 0 .../{ => functional}/lang/parse-okay-crlf.exp | 0 .../{ => functional}/lang/parse-okay-crlf.nix | 0 .../lang/parse-okay-dup-attrs-5.exp | 0 .../lang/parse-okay-dup-attrs-5.nix | 0 .../lang/parse-okay-dup-attrs-6.exp | 0 .../lang/parse-okay-dup-attrs-6.nix | 0 .../lang/parse-okay-mixed-nested-attrs-1.exp | 0 .../lang/parse-okay-mixed-nested-attrs-1.nix | 0 .../lang/parse-okay-mixed-nested-attrs-2.exp | 0 .../lang/parse-okay-mixed-nested-attrs-2.nix | 0 .../lang/parse-okay-mixed-nested-attrs-3.exp | 0 .../lang/parse-okay-mixed-nested-attrs-3.nix | 0 .../lang/parse-okay-regression-20041027.exp | 0 .../lang/parse-okay-regression-20041027.nix | 0 .../lang/parse-okay-regression-751.exp | 0 .../lang/parse-okay-regression-751.nix | 0 .../lang/parse-okay-subversion.exp | 0 .../lang/parse-okay-subversion.nix | 0 .../{ => functional}/lang/parse-okay-url.exp | 0 .../{ => functional}/lang/parse-okay-url.nix | 0 tests/{ => functional}/lang/readDir/bar | 0 .../lang/readDir/foo/git-hates-directories | 0 tests/{ => functional}/lang/readDir/ldir | 0 tests/{ => functional}/lang/readDir/linked | 0 tests/{ => functional}/legacy-ssh-store.sh | 0 .../linux-sandbox-cert-test.nix | 0 tests/{ => functional}/linux-sandbox.sh | 0 tests/{ => functional}/local-store.sh | 0 tests/{ => functional}/local.mk | 13 +++-- tests/{ => functional}/logging.sh | 0 tests/{ => functional}/misc.sh | 0 tests/{ => functional}/multiple-outputs.nix | 0 tests/{ => functional}/multiple-outputs.sh | 0 tests/{ => functional}/nar-access.nix | 0 tests/{ => functional}/nar-access.sh | 0 tests/{ => functional}/nested-sandboxing.sh | 2 +- .../nested-sandboxing/command.sh | 0 .../nested-sandboxing/runner.nix | 0 tests/{ => functional}/nix-build-examples.nix | 0 tests/{ => functional}/nix-build.sh | 0 tests/{ => functional}/nix-channel.sh | 0 .../{ => functional}/nix-collect-garbage-d.sh | 0 tests/{ => functional}/nix-copy-ssh-ng.sh | 0 tests/{ => functional}/nix-copy-ssh.sh | 0 .../{ => functional}/nix-daemon-untrusting.sh | 0 tests/{ => functional}/nix-profile.sh | 0 tests/{ => functional}/nix-shell.sh | 0 tests/{ => functional}/nix_path.sh | 0 tests/{ => functional}/optimise-store.sh | 0 .../{ => functional}/output-normalization.sh | 0 tests/{ => functional}/parallel.builder.sh | 0 tests/{ => functional}/parallel.nix | 0 tests/{ => functional}/parallel.sh | 0 tests/{ => functional}/pass-as-file.sh | 0 tests/{ => functional}/path-from-hash-part.sh | 0 tests/{ => functional}/path.nix | 0 tests/{ => functional}/placeholders.sh | 0 tests/{ => functional}/plugins.sh | 0 tests/{ => functional}/plugins/local.mk | 0 tests/{ => functional}/plugins/plugintest.cc | 0 tests/{ => functional}/post-hook.sh | 0 tests/{ => functional}/pure-eval.nix | 0 tests/{ => functional}/pure-eval.sh | 0 tests/{ => functional}/push-to-store-old.sh | 0 tests/{ => functional}/push-to-store.sh | 0 tests/{ => functional}/read-only-store.sh | 0 tests/{ => functional}/readfile-context.nix | 0 tests/{ => functional}/readfile-context.sh | 0 tests/{ => functional}/recursive.nix | 0 tests/{ => functional}/recursive.sh | 0 tests/{ => functional}/referrers.sh | 0 tests/{ => functional}/remote-store.sh | 0 tests/{ => functional}/repair.sh | 0 tests/{ => functional}/repl.sh | 0 tests/{ => functional}/restricted.nix | 0 tests/{ => functional}/restricted.sh | 6 +- tests/{ => functional}/search.nix | 0 tests/{ => functional}/search.sh | 0 tests/{ => functional}/secure-drv-outputs.nix | 0 tests/{ => functional}/secure-drv-outputs.sh | 0 tests/{ => functional}/selfref-gc.sh | 0 tests/{ => functional}/shell-hello.nix | 0 tests/{ => functional}/shell.nix | 0 tests/{ => functional}/shell.sh | 0 tests/{ => functional}/shell.shebang.rb | 0 tests/{ => functional}/shell.shebang.sh | 0 tests/{ => functional}/signing.sh | 0 tests/{ => functional}/simple-failing.nix | 0 tests/{ => functional}/simple.builder.sh | 0 tests/{ => functional}/simple.nix | 0 tests/{ => functional}/simple.sh | 0 tests/{ => functional}/ssh-relay.sh | 0 tests/{ => functional}/store-ping.sh | 0 .../structured-attrs-shell.nix | 0 tests/{ => functional}/structured-attrs.nix | 0 tests/{ => functional}/structured-attrs.sh | 0 .../substitute-with-invalid-ca.sh | 0 tests/{ => functional}/suggestions.sh | 0 .../{ => functional}/supplementary-groups.sh | 0 tests/{ => functional}/tarball.sh | 0 tests/{ => functional}/test-infra.sh | 0 .../{ => functional}/test-libstoreconsumer.sh | 0 .../test-libstoreconsumer/README.md | 0 .../test-libstoreconsumer/local.mk | 0 .../test-libstoreconsumer/main.cc | 0 tests/{ => functional}/timeout.nix | 0 tests/{ => functional}/timeout.sh | 0 tests/{ => functional}/toString-path.sh | 0 tests/{ => functional}/undefined-variable.nix | 0 tests/{ => functional}/user-envs-migration.sh | 0 tests/{ => functional}/user-envs.builder.sh | 0 tests/{ => functional}/user-envs.nix | 0 tests/{ => functional}/user-envs.sh | 0 tests/{ => functional}/why-depends.sh | 0 tests/{ => functional}/zstd.sh | 0 599 files changed, 84 insertions(+), 87 deletions(-) rename tests/{ => functional}/add.sh (100%) rename tests/{ => functional}/bad.tar.xz (100%) rename tests/{ => functional}/bash-profile.sh (78%) rename tests/{ => functional}/big-derivation-attr.nix (100%) rename tests/{ => functional}/binary-cache-build-remote.sh (100%) rename tests/{ => functional}/binary-cache.sh (100%) rename tests/{ => functional}/brotli.sh (100%) rename tests/{ => functional}/build-delete.sh (100%) rename tests/{ => functional}/build-dry.sh (100%) rename tests/{ => functional}/build-hook-ca-fixed.nix (100%) rename tests/{ => functional}/build-hook-ca-floating.nix (100%) rename tests/{ => functional}/build-hook.nix (100%) rename tests/{ => functional}/build-remote-content-addressed-fixed.sh (100%) rename tests/{ => functional}/build-remote-content-addressed-floating.sh (100%) rename tests/{ => functional}/build-remote-input-addressed.sh (100%) rename tests/{ => functional}/build-remote-trustless-after.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-fail-0.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-pass-0.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-pass-1.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-pass-2.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-pass-3.sh (100%) rename tests/{ => functional}/build-remote-trustless.sh (82%) rename tests/{ => functional}/build-remote.sh (100%) rename tests/{ => functional}/build.sh (100%) rename tests/{ => functional}/ca-shell.nix (100%) rename tests/{ => functional}/ca/build-cache.sh (100%) rename tests/{ => functional}/ca/build-dry.sh (100%) rename tests/{ => functional}/ca/build-with-garbage-path.sh (100%) rename tests/{ => functional}/ca/build.sh (100%) rename tests/{ => functional}/ca/common.sh (100%) rename tests/{ => functional}/ca/concurrent-builds.sh (100%) rename tests/{ => functional}/ca/config.nix.in (100%) rename tests/{ => functional}/ca/content-addressed.nix (100%) rename tests/{ => functional}/ca/derivation-json.sh (100%) rename tests/{ => functional}/ca/duplicate-realisation-in-closure.sh (100%) rename tests/{ => functional}/ca/flake.nix (100%) rename tests/{ => functional}/ca/gc.sh (100%) rename tests/{ => functional}/ca/import-derivation.sh (100%) rename tests/{ => functional}/ca/local.mk (94%) rename tests/{ => functional}/ca/new-build-cmd.sh (100%) rename tests/{ => functional}/ca/nix-copy.sh (100%) rename tests/{ => functional}/ca/nix-run.sh (100%) rename tests/{ => functional}/ca/nix-shell.sh (100%) rename tests/{ => functional}/ca/nondeterministic.nix (100%) rename tests/{ => functional}/ca/post-hook.sh (100%) rename tests/{ => functional}/ca/racy.nix (100%) rename tests/{ => functional}/ca/recursive.sh (100%) rename tests/{ => functional}/ca/repl.sh (100%) rename tests/{ => functional}/ca/selfref-gc.sh (100%) rename tests/{ => functional}/ca/signatures.sh (100%) rename tests/{ => functional}/ca/substitute.sh (100%) rename tests/{ => functional}/ca/why-depends.sh (100%) rename tests/{ => functional}/case-hack.sh (100%) rename tests/{ => functional}/case.nar (100%) rename tests/{ => functional}/check-refs.nix (100%) rename tests/{ => functional}/check-refs.sh (100%) rename tests/{ => functional}/check-reqs.nix (100%) rename tests/{ => functional}/check-reqs.sh (100%) rename tests/{ => functional}/check.nix (100%) rename tests/{ => functional}/check.sh (100%) rename tests/{ => functional}/common.sh (100%) rename tests/{ => functional}/common/vars-and-functions.sh.in (99%) rename tests/{ => functional}/completions.sh (100%) rename tests/{ => functional}/compression-levels.sh (100%) rename tests/{ => functional}/compute-levels.sh (100%) rename tests/{ => functional}/config.nix.in (100%) rename tests/{ => functional}/config.sh (100%) rename tests/{ => functional}/config/nix-with-substituters.conf (100%) rename tests/{ => functional}/db-migration.sh (100%) rename tests/{ => functional}/dependencies.builder0.sh (100%) rename tests/{ => functional}/dependencies.nix (100%) rename tests/{ => functional}/dependencies.sh (100%) rename tests/{ => functional}/derivation-json.sh (100%) rename tests/{ => functional}/dummy (100%) rename tests/{ => functional}/dump-db.sh (100%) rename tests/{ => functional}/dyn-drv/build-built-drv.sh (100%) rename tests/{ => functional}/dyn-drv/common.sh (100%) rename tests/{ => functional}/dyn-drv/config.nix.in (100%) rename tests/{ => functional}/dyn-drv/dep-built-drv.sh (100%) rename tests/{ => functional}/dyn-drv/eval-outputOf.sh (100%) rename tests/{ => functional}/dyn-drv/local.mk (87%) rename tests/{ => functional}/dyn-drv/old-daemon-error-hack.nix (100%) rename tests/{ => functional}/dyn-drv/old-daemon-error-hack.sh (100%) rename tests/{ => functional}/dyn-drv/recursive-mod-json.nix (100%) rename tests/{ => functional}/dyn-drv/recursive-mod-json.sh (100%) rename tests/{ => functional}/dyn-drv/text-hashed-output.nix (100%) rename tests/{ => functional}/dyn-drv/text-hashed-output.sh (100%) rename tests/{ => functional}/eval-store.sh (100%) rename tests/{ => functional}/eval.nix (100%) rename tests/{ => functional}/eval.sh (100%) rename tests/{ => functional}/experimental-features.sh (100%) rename tests/{ => functional}/export-graph.nix (100%) rename tests/{ => functional}/export-graph.sh (100%) rename tests/{ => functional}/export.sh (100%) rename tests/{ => functional}/failing.nix (100%) rename tests/{ => functional}/fetchClosure.sh (100%) rename tests/{ => functional}/fetchGit.sh (100%) rename tests/{ => functional}/fetchGitRefs.sh (100%) rename tests/{ => functional}/fetchGitSubmodules.sh (100%) rename tests/{ => functional}/fetchMercurial.sh (100%) rename tests/{ => functional}/fetchPath.sh (100%) rename tests/{ => functional}/fetchTree-file.sh (100%) rename tests/{ => functional}/fetchurl.sh (100%) rename tests/{ => functional}/filter-source.nix (100%) rename tests/{ => functional}/filter-source.sh (100%) rename tests/{ => functional}/fixed.builder1.sh (100%) rename tests/{ => functional}/fixed.builder2.sh (100%) rename tests/{ => functional}/fixed.nix (100%) rename tests/{ => functional}/fixed.sh (100%) rename tests/{ => functional}/flakes/absolute-attr-paths.sh (100%) rename tests/{ => functional}/flakes/absolute-paths.sh (100%) rename tests/{ => functional}/flakes/build-paths.sh (100%) rename tests/{ => functional}/flakes/bundle.sh (100%) rename tests/{ => functional}/flakes/check.sh (100%) rename tests/{ => functional}/flakes/circular.sh (100%) rename tests/{ => functional}/flakes/common.sh (100%) rename tests/{ => functional}/flakes/config.sh (100%) rename tests/{ => functional}/flakes/flake-in-submodule.sh (100%) rename tests/{ => functional}/flakes/flakes.sh (100%) rename tests/{ => functional}/flakes/follow-paths.sh (100%) rename tests/{ => functional}/flakes/init.sh (100%) rename tests/{ => functional}/flakes/inputs.sh (100%) rename tests/{ => functional}/flakes/mercurial.sh (100%) rename tests/{ => functional}/flakes/run.sh (100%) rename tests/{ => functional}/flakes/search-root.sh (100%) rename tests/{ => functional}/flakes/show.sh (100%) rename tests/{ => functional}/flakes/unlocked-override.sh (100%) rename tests/{ => functional}/fmt.sh (100%) rename tests/{ => functional}/fmt.simple.sh (100%) rename tests/{ => functional}/function-trace.sh (100%) rename tests/{ => functional}/gc-auto.sh (100%) rename tests/{ => functional}/gc-concurrent.builder.sh (100%) rename tests/{ => functional}/gc-concurrent.nix (100%) rename tests/{ => functional}/gc-concurrent.sh (100%) rename tests/{ => functional}/gc-concurrent2.builder.sh (100%) rename tests/{ => functional}/gc-non-blocking.sh (100%) rename tests/{ => functional}/gc-runtime.nix (100%) rename tests/{ => functional}/gc-runtime.sh (100%) rename tests/{ => functional}/gc.sh (100%) rename tests/{ => functional}/hash-check.nix (100%) rename tests/{ => functional}/hash.sh (100%) rename tests/{ => functional}/hermetic.nix (100%) rename tests/{ => functional}/import-derivation.nix (100%) rename tests/{ => functional}/import-derivation.sh (100%) rename tests/{ => functional}/impure-derivations.nix (100%) rename tests/{ => functional}/impure-derivations.sh (100%) rename tests/{ => functional}/init.sh (100%) rename tests/{ => functional}/install-darwin.sh (100%) rename tests/{ => functional}/lang-test-infra.sh (100%) rename tests/{ => functional}/lang.sh (98%) rename tests/{ => functional}/lang/binary-data (100%) rename tests/{ => functional}/lang/data (100%) rename tests/{ => functional}/lang/dir1/a.nix (100%) rename tests/{ => functional}/lang/dir2/a.nix (100%) rename tests/{ => functional}/lang/dir2/b.nix (100%) rename tests/{ => functional}/lang/dir3/a.nix (100%) rename tests/{ => functional}/lang/dir3/b.nix (100%) rename tests/{ => functional}/lang/dir3/c.nix (100%) rename tests/{ => functional}/lang/dir4/a.nix (100%) rename tests/{ => functional}/lang/dir4/c.nix (100%) rename tests/{ => functional}/lang/empty.exp (100%) rename tests/{ => functional}/lang/eval-fail-abort.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-abort.nix (100%) rename tests/{ => functional}/lang/eval-fail-antiquoted-path.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-assert.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-assert.nix (100%) rename tests/{ => functional}/lang/eval-fail-bad-antiquote-1.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-antiquote-2.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-antiquote-3.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-1.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-1.nix (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-2.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-2.nix (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-3.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-3.nix (100%) rename tests/{ => functional}/lang/eval-fail-blackhole.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-blackhole.nix (100%) rename tests/{ => functional}/lang/eval-fail-deepseq.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-deepseq.nix (100%) rename tests/{ => functional}/lang/eval-fail-dup-dynamic-attrs.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-dup-dynamic-attrs.nix (100%) rename tests/{ => functional}/lang/eval-fail-foldlStrict-strict-op-application.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-foldlStrict-strict-op-application.nix (100%) rename tests/{ => functional}/lang/eval-fail-fromTOML-timestamps.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-fromTOML-timestamps.nix (100%) rename tests/{ => functional}/lang/eval-fail-hashfile-missing.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-hashfile-missing.nix (100%) rename tests/{ => functional}/lang/eval-fail-list.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-list.nix (100%) rename tests/{ => functional}/lang/eval-fail-missing-arg.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-missing-arg.nix (100%) rename tests/{ => functional}/lang/eval-fail-nonexist-path.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-nonexist-path.nix (100%) rename tests/{ => functional}/lang/eval-fail-path-slash.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-path-slash.nix (100%) rename tests/{ => functional}/lang/eval-fail-recursion.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-recursion.nix (100%) rename tests/{ => functional}/lang/eval-fail-remove.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-remove.nix (100%) rename tests/{ => functional}/lang/eval-fail-scope-5.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-scope-5.nix (100%) rename tests/{ => functional}/lang/eval-fail-seq.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-seq.nix (100%) rename tests/{ => functional}/lang/eval-fail-set-override.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-set-override.nix (100%) rename tests/{ => functional}/lang/eval-fail-set.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-set.nix (100%) rename tests/{ => functional}/lang/eval-fail-substring.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-substring.nix (100%) rename tests/{ => functional}/lang/eval-fail-to-path.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-to-path.nix (100%) rename tests/{ => functional}/lang/eval-fail-toJSON.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-toJSON.nix (100%) rename tests/{ => functional}/lang/eval-fail-undeclared-arg.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-undeclared-arg.nix (100%) rename tests/{ => functional}/lang/eval-okay-any-all.exp (100%) rename tests/{ => functional}/lang/eval-okay-any-all.nix (100%) rename tests/{ => functional}/lang/eval-okay-arithmetic.exp (100%) rename tests/{ => functional}/lang/eval-okay-arithmetic.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrnames.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrnames.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs2.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs2.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs3.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs3.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs4.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs4.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs5.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs5.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs6.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs6.nix (100%) rename tests/{ => functional}/lang/eval-okay-autoargs.exp (100%) rename tests/{ => functional}/lang/eval-okay-autoargs.flags (100%) rename tests/{ => functional}/lang/eval-okay-autoargs.nix (100%) rename tests/{ => functional}/lang/eval-okay-backslash-newline-1.exp (100%) rename tests/{ => functional}/lang/eval-okay-backslash-newline-1.nix (100%) rename tests/{ => functional}/lang/eval-okay-backslash-newline-2.exp (100%) rename tests/{ => functional}/lang/eval-okay-backslash-newline-2.nix (100%) rename tests/{ => functional}/lang/eval-okay-builtins-add.exp (100%) rename tests/{ => functional}/lang/eval-okay-builtins-add.nix (100%) rename tests/{ => functional}/lang/eval-okay-builtins.exp (100%) rename tests/{ => functional}/lang/eval-okay-builtins.nix (100%) rename tests/{ => functional}/lang/eval-okay-callable-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-callable-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-catattrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-catattrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-closure.exp (100%) rename tests/{ => functional}/lang/eval-okay-closure.exp.xml (100%) rename tests/{ => functional}/lang/eval-okay-closure.nix (100%) rename tests/{ => functional}/lang/eval-okay-comments.exp (100%) rename tests/{ => functional}/lang/eval-okay-comments.nix (100%) rename tests/{ => functional}/lang/eval-okay-concat.exp (100%) rename tests/{ => functional}/lang/eval-okay-concat.nix (100%) rename tests/{ => functional}/lang/eval-okay-concatmap.exp (100%) rename tests/{ => functional}/lang/eval-okay-concatmap.nix (100%) rename tests/{ => functional}/lang/eval-okay-concatstringssep.exp (100%) rename tests/{ => functional}/lang/eval-okay-concatstringssep.nix (100%) rename tests/{ => functional}/lang/eval-okay-context-introspection.exp (100%) rename tests/{ => functional}/lang/eval-okay-context-introspection.nix (100%) rename tests/{ => functional}/lang/eval-okay-context.exp (100%) rename tests/{ => functional}/lang/eval-okay-context.nix (100%) rename tests/{ => functional}/lang/eval-okay-curpos.exp (100%) rename tests/{ => functional}/lang/eval-okay-curpos.nix (100%) rename tests/{ => functional}/lang/eval-okay-deepseq.exp (100%) rename tests/{ => functional}/lang/eval-okay-deepseq.nix (100%) rename tests/{ => functional}/lang/eval-okay-delayed-with-inherit.exp (100%) rename tests/{ => functional}/lang/eval-okay-delayed-with-inherit.nix (100%) rename tests/{ => functional}/lang/eval-okay-delayed-with.exp (100%) rename tests/{ => functional}/lang/eval-okay-delayed-with.nix (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs-2.exp (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs-2.nix (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs-bare.exp (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs-bare.nix (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-elem.exp (100%) rename tests/{ => functional}/lang/eval-okay-elem.nix (100%) rename tests/{ => functional}/lang/eval-okay-empty-args.exp (100%) rename tests/{ => functional}/lang/eval-okay-empty-args.nix (100%) rename tests/{ => functional}/lang/eval-okay-eq-derivations.exp (100%) rename tests/{ => functional}/lang/eval-okay-eq-derivations.nix (100%) rename tests/{ => functional}/lang/eval-okay-eq.exp (100%) rename tests/{ => functional}/lang/eval-okay-eq.nix (100%) rename tests/{ => functional}/lang/eval-okay-filter.exp (100%) rename tests/{ => functional}/lang/eval-okay-filter.nix (100%) rename tests/{ => functional}/lang/eval-okay-flake-ref-to-string.exp (100%) rename tests/{ => functional}/lang/eval-okay-flake-ref-to-string.nix (100%) rename tests/{ => functional}/lang/eval-okay-flatten.exp (100%) rename tests/{ => functional}/lang/eval-okay-flatten.nix (100%) rename tests/{ => functional}/lang/eval-okay-float.exp (100%) rename tests/{ => functional}/lang/eval-okay-float.nix (100%) rename tests/{ => functional}/lang/eval-okay-floor-ceil.exp (100%) rename tests/{ => functional}/lang/eval-okay-floor-ceil.nix (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict-lazy-elements.exp (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict-lazy-elements.nix (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict.exp (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict.nix (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML-timestamps.exp (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML-timestamps.flags (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML-timestamps.nix (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML.exp (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML.nix (100%) rename tests/{ => functional}/lang/eval-okay-fromjson-escapes.exp (100%) rename tests/{ => functional}/lang/eval-okay-fromjson-escapes.nix (100%) rename tests/{ => functional}/lang/eval-okay-fromjson.exp (100%) rename tests/{ => functional}/lang/eval-okay-fromjson.nix (100%) rename tests/{ => functional}/lang/eval-okay-functionargs.exp (100%) rename tests/{ => functional}/lang/eval-okay-functionargs.exp.xml (100%) rename tests/{ => functional}/lang/eval-okay-functionargs.nix (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos-functionargs.exp (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos-functionargs.nix (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos-undefined.exp (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos-undefined.nix (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos.exp (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos.nix (100%) rename tests/{ => functional}/lang/eval-okay-getenv.exp (100%) rename tests/{ => functional}/lang/eval-okay-getenv.nix (100%) rename tests/{ => functional}/lang/eval-okay-groupBy.exp (100%) rename tests/{ => functional}/lang/eval-okay-groupBy.nix (100%) rename tests/{ => functional}/lang/eval-okay-hash.exp (100%) rename tests/{ => functional}/lang/eval-okay-hashfile.exp (100%) rename tests/{ => functional}/lang/eval-okay-hashfile.nix (100%) rename tests/{ => functional}/lang/eval-okay-hashstring.exp (100%) rename tests/{ => functional}/lang/eval-okay-hashstring.nix (100%) rename tests/{ => functional}/lang/eval-okay-if.exp (100%) rename tests/{ => functional}/lang/eval-okay-if.nix (100%) rename tests/{ => functional}/lang/eval-okay-import.exp (100%) rename tests/{ => functional}/lang/eval-okay-import.nix (100%) rename tests/{ => functional}/lang/eval-okay-ind-string.exp (100%) rename tests/{ => functional}/lang/eval-okay-ind-string.nix (100%) rename tests/{ => functional}/lang/eval-okay-intersectAttrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-intersectAttrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-let.exp (100%) rename tests/{ => functional}/lang/eval-okay-let.nix (100%) rename tests/{ => functional}/lang/eval-okay-list.exp (100%) rename tests/{ => functional}/lang/eval-okay-list.nix (100%) rename tests/{ => functional}/lang/eval-okay-listtoattrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-listtoattrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-logic.exp (100%) rename tests/{ => functional}/lang/eval-okay-logic.nix (100%) rename tests/{ => functional}/lang/eval-okay-map.exp (100%) rename tests/{ => functional}/lang/eval-okay-map.nix (100%) rename tests/{ => functional}/lang/eval-okay-mapattrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-mapattrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-merge-dynamic-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-merge-dynamic-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-nested-with.exp (100%) rename tests/{ => functional}/lang/eval-okay-nested-with.nix (100%) rename tests/{ => functional}/lang/eval-okay-new-let.exp (100%) rename tests/{ => functional}/lang/eval-okay-new-let.nix (100%) rename tests/{ => functional}/lang/eval-okay-null-dynamic-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-null-dynamic-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-overrides.exp (100%) rename tests/{ => functional}/lang/eval-okay-overrides.nix (100%) rename tests/{ => functional}/lang/eval-okay-parse-flake-ref.exp (100%) rename tests/{ => functional}/lang/eval-okay-parse-flake-ref.nix (100%) rename tests/{ => functional}/lang/eval-okay-partition.exp (100%) rename tests/{ => functional}/lang/eval-okay-partition.nix (100%) rename tests/{ => functional}/lang/eval-okay-path-string-interpolation.exp (100%) rename tests/{ => functional}/lang/eval-okay-path-string-interpolation.nix (100%) rename tests/{ => functional}/lang/eval-okay-path.exp (100%) rename tests/{ => functional}/lang/eval-okay-path.nix (100%) rename tests/{ => functional}/lang/eval-okay-pathexists.exp (100%) rename tests/{ => functional}/lang/eval-okay-pathexists.nix (100%) rename tests/{ => functional}/lang/eval-okay-patterns.exp (100%) rename tests/{ => functional}/lang/eval-okay-patterns.nix (100%) rename tests/{ => functional}/lang/eval-okay-print.err.exp (100%) rename tests/{ => functional}/lang/eval-okay-print.exp (100%) rename tests/{ => functional}/lang/eval-okay-print.nix (100%) rename tests/{ => functional}/lang/eval-okay-readDir.exp (100%) rename tests/{ => functional}/lang/eval-okay-readDir.nix (100%) rename tests/{ => functional}/lang/eval-okay-readFileType.exp (100%) rename tests/{ => functional}/lang/eval-okay-readFileType.nix (100%) rename tests/{ => functional}/lang/eval-okay-readfile.exp (100%) rename tests/{ => functional}/lang/eval-okay-readfile.nix (100%) rename tests/{ => functional}/lang/eval-okay-redefine-builtin.exp (100%) rename tests/{ => functional}/lang/eval-okay-redefine-builtin.nix (100%) rename tests/{ => functional}/lang/eval-okay-regex-match.exp (100%) rename tests/{ => functional}/lang/eval-okay-regex-match.nix (100%) rename tests/{ => functional}/lang/eval-okay-regex-split.exp (100%) rename tests/{ => functional}/lang/eval-okay-regex-split.nix (100%) rename tests/{ => functional}/lang/eval-okay-regression-20220122.exp (100%) rename tests/{ => functional}/lang/eval-okay-regression-20220122.nix (100%) rename tests/{ => functional}/lang/eval-okay-regression-20220125.exp (100%) rename tests/{ => functional}/lang/eval-okay-regression-20220125.nix (100%) rename tests/{ => functional}/lang/eval-okay-remove.exp (100%) rename tests/{ => functional}/lang/eval-okay-remove.nix (100%) rename tests/{ => functional}/lang/eval-okay-replacestrings.exp (100%) rename tests/{ => functional}/lang/eval-okay-replacestrings.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-1.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-1.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-2.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-2.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-3.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-3.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-4.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-4.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-6.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-6.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-7.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-7.nix (100%) rename tests/{ => functional}/lang/eval-okay-search-path.exp (100%) rename tests/{ => functional}/lang/eval-okay-search-path.flags (100%) rename tests/{ => functional}/lang/eval-okay-search-path.nix (100%) rename tests/{ => functional}/lang/eval-okay-seq.exp (100%) rename tests/{ => functional}/lang/eval-okay-seq.nix (100%) rename tests/{ => functional}/lang/eval-okay-sort.exp (100%) rename tests/{ => functional}/lang/eval-okay-sort.nix (100%) rename tests/{ => functional}/lang/eval-okay-splitversion.exp (100%) rename tests/{ => functional}/lang/eval-okay-splitversion.nix (100%) rename tests/{ => functional}/lang/eval-okay-string.exp (100%) rename tests/{ => functional}/lang/eval-okay-string.nix (100%) rename tests/{ => functional}/lang/eval-okay-strings-as-attrs-names.exp (100%) rename tests/{ => functional}/lang/eval-okay-strings-as-attrs-names.nix (100%) rename tests/{ => functional}/lang/eval-okay-substring.exp (100%) rename tests/{ => functional}/lang/eval-okay-substring.nix (100%) rename tests/{ => functional}/lang/eval-okay-tail-call-1.exp-disabled (100%) rename tests/{ => functional}/lang/eval-okay-tail-call-1.nix (100%) rename tests/{ => functional}/lang/eval-okay-tojson.exp (100%) rename tests/{ => functional}/lang/eval-okay-tojson.nix (100%) rename tests/{ => functional}/lang/eval-okay-toxml.exp (100%) rename tests/{ => functional}/lang/eval-okay-toxml.nix (100%) rename tests/{ => functional}/lang/eval-okay-toxml2.exp (100%) rename tests/{ => functional}/lang/eval-okay-toxml2.nix (100%) rename tests/{ => functional}/lang/eval-okay-tryeval.exp (100%) rename tests/{ => functional}/lang/eval-okay-tryeval.nix (100%) rename tests/{ => functional}/lang/eval-okay-types.exp (100%) rename tests/{ => functional}/lang/eval-okay-types.nix (100%) rename tests/{ => functional}/lang/eval-okay-versions.exp (100%) rename tests/{ => functional}/lang/eval-okay-versions.nix (100%) rename tests/{ => functional}/lang/eval-okay-with.exp (100%) rename tests/{ => functional}/lang/eval-okay-with.nix (100%) rename tests/{ => functional}/lang/eval-okay-xml.exp.xml (100%) rename tests/{ => functional}/lang/eval-okay-xml.nix (100%) rename tests/{ => functional}/lang/eval-okay-zipAttrsWith.exp (100%) rename tests/{ => functional}/lang/eval-okay-zipAttrsWith.nix (100%) rename tests/{ => functional}/lang/framework.sh (100%) rename tests/{ => functional}/lang/imported.nix (100%) rename tests/{ => functional}/lang/imported2.nix (100%) rename tests/{ => functional}/lang/lib.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-1.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-1.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-2.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-2.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-3.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-3.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-4.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-4.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-6.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-7.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-7.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-formals.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-formals.nix (100%) rename tests/{ => functional}/lang/parse-fail-eof-in-string.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-eof-in-string.nix (100%) rename tests/{ => functional}/lang/parse-fail-mixed-nested-attrs1.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-mixed-nested-attrs1.nix (100%) rename tests/{ => functional}/lang/parse-fail-mixed-nested-attrs2.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-mixed-nested-attrs2.nix (100%) rename tests/{ => functional}/lang/parse-fail-patterns-1.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-patterns-1.nix (100%) rename tests/{ => functional}/lang/parse-fail-regression-20060610.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-regression-20060610.nix (100%) rename tests/{ => functional}/lang/parse-fail-undef-var-2.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-undef-var-2.nix (100%) rename tests/{ => functional}/lang/parse-fail-undef-var.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-undef-var.nix (100%) rename tests/{ => functional}/lang/parse-fail-utf8.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-utf8.nix (100%) rename tests/{ => functional}/lang/parse-okay-1.exp (100%) rename tests/{ => functional}/lang/parse-okay-1.nix (100%) rename tests/{ => functional}/lang/parse-okay-crlf.exp (100%) rename tests/{ => functional}/lang/parse-okay-crlf.nix (100%) rename tests/{ => functional}/lang/parse-okay-dup-attrs-5.exp (100%) rename tests/{ => functional}/lang/parse-okay-dup-attrs-5.nix (100%) rename tests/{ => functional}/lang/parse-okay-dup-attrs-6.exp (100%) rename tests/{ => functional}/lang/parse-okay-dup-attrs-6.nix (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-1.exp (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-1.nix (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-2.exp (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-2.nix (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-3.exp (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-3.nix (100%) rename tests/{ => functional}/lang/parse-okay-regression-20041027.exp (100%) rename tests/{ => functional}/lang/parse-okay-regression-20041027.nix (100%) rename tests/{ => functional}/lang/parse-okay-regression-751.exp (100%) rename tests/{ => functional}/lang/parse-okay-regression-751.nix (100%) rename tests/{ => functional}/lang/parse-okay-subversion.exp (100%) rename tests/{ => functional}/lang/parse-okay-subversion.nix (100%) rename tests/{ => functional}/lang/parse-okay-url.exp (100%) rename tests/{ => functional}/lang/parse-okay-url.nix (100%) rename tests/{ => functional}/lang/readDir/bar (100%) rename tests/{ => functional}/lang/readDir/foo/git-hates-directories (100%) rename tests/{ => functional}/lang/readDir/ldir (100%) rename tests/{ => functional}/lang/readDir/linked (100%) rename tests/{ => functional}/legacy-ssh-store.sh (100%) rename tests/{ => functional}/linux-sandbox-cert-test.nix (100%) rename tests/{ => functional}/linux-sandbox.sh (100%) rename tests/{ => functional}/local-store.sh (100%) rename tests/{ => functional}/local.mk (90%) rename tests/{ => functional}/logging.sh (100%) rename tests/{ => functional}/misc.sh (100%) rename tests/{ => functional}/multiple-outputs.nix (100%) rename tests/{ => functional}/multiple-outputs.sh (100%) rename tests/{ => functional}/nar-access.nix (100%) rename tests/{ => functional}/nar-access.sh (100%) rename tests/{ => functional}/nested-sandboxing.sh (75%) rename tests/{ => functional}/nested-sandboxing/command.sh (100%) rename tests/{ => functional}/nested-sandboxing/runner.nix (100%) rename tests/{ => functional}/nix-build-examples.nix (100%) rename tests/{ => functional}/nix-build.sh (100%) rename tests/{ => functional}/nix-channel.sh (100%) rename tests/{ => functional}/nix-collect-garbage-d.sh (100%) rename tests/{ => functional}/nix-copy-ssh-ng.sh (100%) rename tests/{ => functional}/nix-copy-ssh.sh (100%) rename tests/{ => functional}/nix-daemon-untrusting.sh (100%) rename tests/{ => functional}/nix-profile.sh (100%) rename tests/{ => functional}/nix-shell.sh (100%) rename tests/{ => functional}/nix_path.sh (100%) rename tests/{ => functional}/optimise-store.sh (100%) rename tests/{ => functional}/output-normalization.sh (100%) rename tests/{ => functional}/parallel.builder.sh (100%) rename tests/{ => functional}/parallel.nix (100%) rename tests/{ => functional}/parallel.sh (100%) rename tests/{ => functional}/pass-as-file.sh (100%) rename tests/{ => functional}/path-from-hash-part.sh (100%) rename tests/{ => functional}/path.nix (100%) rename tests/{ => functional}/placeholders.sh (100%) rename tests/{ => functional}/plugins.sh (100%) rename tests/{ => functional}/plugins/local.mk (100%) rename tests/{ => functional}/plugins/plugintest.cc (100%) rename tests/{ => functional}/post-hook.sh (100%) rename tests/{ => functional}/pure-eval.nix (100%) rename tests/{ => functional}/pure-eval.sh (100%) rename tests/{ => functional}/push-to-store-old.sh (100%) rename tests/{ => functional}/push-to-store.sh (100%) rename tests/{ => functional}/read-only-store.sh (100%) rename tests/{ => functional}/readfile-context.nix (100%) rename tests/{ => functional}/readfile-context.sh (100%) rename tests/{ => functional}/recursive.nix (100%) rename tests/{ => functional}/recursive.sh (100%) rename tests/{ => functional}/referrers.sh (100%) rename tests/{ => functional}/remote-store.sh (100%) rename tests/{ => functional}/repair.sh (100%) rename tests/{ => functional}/repl.sh (100%) rename tests/{ => functional}/restricted.nix (100%) rename tests/{ => functional}/restricted.sh (95%) rename tests/{ => functional}/search.nix (100%) rename tests/{ => functional}/search.sh (100%) rename tests/{ => functional}/secure-drv-outputs.nix (100%) rename tests/{ => functional}/secure-drv-outputs.sh (100%) rename tests/{ => functional}/selfref-gc.sh (100%) rename tests/{ => functional}/shell-hello.nix (100%) rename tests/{ => functional}/shell.nix (100%) rename tests/{ => functional}/shell.sh (100%) rename tests/{ => functional}/shell.shebang.rb (100%) rename tests/{ => functional}/shell.shebang.sh (100%) rename tests/{ => functional}/signing.sh (100%) rename tests/{ => functional}/simple-failing.nix (100%) rename tests/{ => functional}/simple.builder.sh (100%) rename tests/{ => functional}/simple.nix (100%) rename tests/{ => functional}/simple.sh (100%) rename tests/{ => functional}/ssh-relay.sh (100%) rename tests/{ => functional}/store-ping.sh (100%) rename tests/{ => functional}/structured-attrs-shell.nix (100%) rename tests/{ => functional}/structured-attrs.nix (100%) rename tests/{ => functional}/structured-attrs.sh (100%) rename tests/{ => functional}/substitute-with-invalid-ca.sh (100%) rename tests/{ => functional}/suggestions.sh (100%) rename tests/{ => functional}/supplementary-groups.sh (100%) rename tests/{ => functional}/tarball.sh (100%) rename tests/{ => functional}/test-infra.sh (100%) rename tests/{ => functional}/test-libstoreconsumer.sh (100%) rename tests/{ => functional}/test-libstoreconsumer/README.md (100%) rename tests/{ => functional}/test-libstoreconsumer/local.mk (100%) rename tests/{ => functional}/test-libstoreconsumer/main.cc (100%) rename tests/{ => functional}/timeout.nix (100%) rename tests/{ => functional}/timeout.sh (100%) rename tests/{ => functional}/toString-path.sh (100%) rename tests/{ => functional}/undefined-variable.nix (100%) rename tests/{ => functional}/user-envs-migration.sh (100%) rename tests/{ => functional}/user-envs.builder.sh (100%) rename tests/{ => functional}/user-envs.nix (100%) rename tests/{ => functional}/user-envs.sh (100%) rename tests/{ => functional}/why-depends.sh (100%) rename tests/{ => functional}/zstd.sh (100%) diff --git a/.github/labeler.yml b/.github/labeler.yml index 12120bdb3..7544f07a6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -20,4 +20,4 @@ # Unit tests - src/*/tests/**/* # Functional and integration tests - - tests/**/* + - tests/functional/**/* diff --git a/.gitignore b/.gitignore index bf4f33564..2d3314015 100644 --- a/.gitignore +++ b/.gitignore @@ -79,24 +79,24 @@ perl/Makefile.config /src/build-remote/build-remote -# /tests/ -/tests/test-tmp -/tests/common/vars-and-functions.sh -/tests/result* -/tests/restricted-innocent -/tests/shell -/tests/shell.drv -/tests/config.nix -/tests/ca/config.nix -/tests/dyn-drv/config.nix -/tests/repl-result-out -/tests/test-libstoreconsumer/test-libstoreconsumer +# /tests/functional/ +/tests/functional/test-tmp +/tests/functional/common/vars-and-functions.sh +/tests/functional/result* +/tests/functional/restricted-innocent +/tests/functional/shell +/tests/functional/shell.drv +/tests/functional/config.nix +/tests/functional/ca/config.nix +/tests/functional/dyn-drv/config.nix +/tests/functional/repl-result-out +/tests/functional/test-libstoreconsumer/test-libstoreconsumer -# /tests/lang/ -/tests/lang/*.out -/tests/lang/*.out.xml -/tests/lang/*.err -/tests/lang/*.ast +# /tests/functional/lang/ +/tests/functional/lang/*.out +/tests/functional/lang/*.out.xml +/tests/functional/lang/*.err +/tests/functional/lang/*.ast /perl/lib/Nix/Config.pm /perl/lib/Nix/Store.cc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a9cff3b2..73210b303 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue - [ ] Tests, as appropriate: - - Functional tests – [`tests/**.sh`](./tests) + - Functional tests – [`tests/functional/**.sh`](./tests/functional) - Unit tests – [`src/*/tests`](./src/) - Integration tests – [`tests/nixos/*`](./tests/nixos) - [ ] User documentation in the [manual](..doc/manual/src) diff --git a/Makefile b/Makefile index 2eabeb077..6658e3490 100644 --- a/Makefile +++ b/Makefile @@ -27,11 +27,11 @@ makefiles += \ src/libutil/tests/local.mk \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ - tests/local.mk \ - tests/ca/local.mk \ - tests/dyn-drv/local.mk \ - tests/test-libstoreconsumer/local.mk \ - tests/plugins/local.mk + tests/functional/local.mk \ + tests/functional/ca/local.mk \ + tests/functional/dyn-drv/local.mk \ + tests/functional/test-libstoreconsumer/local.mk \ + tests/functional/plugins/local.mk else makefiles += \ mk/disable-tests.mk diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 04bb4343d..1a6388e40 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -82,7 +82,7 @@ The characterization tests will mark themselves "skipped" since they regenerated ## Functional tests -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. ### Running the whole test suite @@ -91,8 +91,8 @@ The whole test suite can be run with: ```shell-session $ make install && make installcheck -ran test tests/foo.sh... [PASS] -ran test tests/bar.sh... [PASS] +ran test tests/functional/foo.sh... [PASS] +ran test tests/functional/bar.sh... [PASS] ... ``` @@ -100,14 +100,14 @@ ran test tests/bar.sh... [PASS] Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. Each test group is in a subdirectory of `tests`. -For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. +For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. That test group can be run like this: ```shell-session $ make ca.test-group -j50 -ran test tests/ca/nix-run.sh... [PASS] -ran test tests/ca/import-derivation.sh... [PASS] +ran test tests/functional/ca/nix-run.sh... [PASS] +ran test tests/functional/ca/import-derivation.sh... [PASS] ... ``` @@ -126,21 +126,21 @@ install-tests-groups += $(test-group-name) Individual tests can be run with `make`: ```shell-session -$ make tests/${testName}.sh.test -ran test tests/${testName}.sh... [PASS] +$ make tests/functional/${testName}.sh.test +ran test tests/functional/${testName}.sh... [PASS] ``` or without `make`: ```shell-session -$ ./mk/run-test.sh tests/${testName}.sh -ran test tests/${testName}.sh... [PASS] +$ ./mk/run-test.sh tests/functional/${testName}.sh +ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh + foo output from foo + bar @@ -175,7 +175,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -206,7 +206,7 @@ It is frequently useful to regenerate the expected output. To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. For example: ```bash -_NIX_TEST_ACCEPT=1 make tests/lang.sh.test +_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test ``` This convention is shared with the [characterization unit tests](#characterization-testing-1) too. diff --git a/flake.nix b/flake.nix index 4de155362..c331b2651 100644 --- a/flake.nix +++ b/flake.nix @@ -61,36 +61,28 @@ nixSrc = fileset.toSource { root = ./.; - fileset = fileset.intersect baseFiles ( - fileset.difference - (fileset.unions [ - ./.version - ./boehmgc-coroutine-sp-fallback.diff - ./bootstrap.sh - ./configure.ac - ./doc - ./local.mk - ./m4 - ./Makefile - ./Makefile.config.in - ./misc - ./mk - ./precompiled-headers.h - ./src - ./tests - ./unit-test-data - ./COPYING - ./scripts/local.mk - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]) - (fileset.unions [ - # Removed file sets - ./tests/nixos - ./tests/installer - ]) - ); + fileset = fileset.intersect baseFiles (fileset.unions [ + ./.version + ./boehmgc-coroutine-sp-fallback.diff + ./bootstrap.sh + ./configure.ac + ./doc + ./local.mk + ./m4 + ./Makefile + ./Makefile.config.in + ./misc + ./mk + ./precompiled-headers.h + ./src + ./tests/functional + ./unit-test-data + ./COPYING + ./scripts/local.mk + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]); }; # Memoize nixpkgs for different platforms for efficiency. diff --git a/mk/common-test.sh b/mk/common-test.sh index 0a2e4c1c2..7ab25febf 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,11 +1,15 @@ +test_dir=tests/functional + +test=$(echo -n "$test" | sed -e "s|^$test_dir/||") + TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') : ${BASH:=/usr/bin/env bash} init_test () { - cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null + cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null } run_test_proper () { - cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) + cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 26c87391c..516cbef83 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -776,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ + /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */ if (auto p = getEnv("_NIX_TEST_GC_SYNC")) readFile(*p); diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 4712c0c91..5c5a30dc2 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -30,7 +30,7 @@ extern std::regex refRegex; /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is /// This is because of the definition of a ref in refs.c in https://github.com/git/git -/// See tests/fetchGitRefs.sh for the full definition +/// See tests/functional/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; diff --git a/tests/add.sh b/tests/functional/add.sh similarity index 100% rename from tests/add.sh rename to tests/functional/add.sh diff --git a/tests/bad.tar.xz b/tests/functional/bad.tar.xz similarity index 100% rename from tests/bad.tar.xz rename to tests/functional/bad.tar.xz diff --git a/tests/bash-profile.sh b/tests/functional/bash-profile.sh similarity index 78% rename from tests/bash-profile.sh rename to tests/functional/bash-profile.sh index e2e0d1090..3faeaaba1 100644 --- a/tests/bash-profile.sh +++ b/tests/functional/bash-profile.sh @@ -1,6 +1,6 @@ source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh user=$(whoami) rm -rf $TEST_HOME $TEST_ROOT/profile-var diff --git a/tests/big-derivation-attr.nix b/tests/functional/big-derivation-attr.nix similarity index 100% rename from tests/big-derivation-attr.nix rename to tests/functional/big-derivation-attr.nix diff --git a/tests/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh similarity index 100% rename from tests/binary-cache-build-remote.sh rename to tests/functional/binary-cache-build-remote.sh diff --git a/tests/binary-cache.sh b/tests/functional/binary-cache.sh similarity index 100% rename from tests/binary-cache.sh rename to tests/functional/binary-cache.sh diff --git a/tests/brotli.sh b/tests/functional/brotli.sh similarity index 100% rename from tests/brotli.sh rename to tests/functional/brotli.sh diff --git a/tests/build-delete.sh b/tests/functional/build-delete.sh similarity index 100% rename from tests/build-delete.sh rename to tests/functional/build-delete.sh diff --git a/tests/build-dry.sh b/tests/functional/build-dry.sh similarity index 100% rename from tests/build-dry.sh rename to tests/functional/build-dry.sh diff --git a/tests/build-hook-ca-fixed.nix b/tests/functional/build-hook-ca-fixed.nix similarity index 100% rename from tests/build-hook-ca-fixed.nix rename to tests/functional/build-hook-ca-fixed.nix diff --git a/tests/build-hook-ca-floating.nix b/tests/functional/build-hook-ca-floating.nix similarity index 100% rename from tests/build-hook-ca-floating.nix rename to tests/functional/build-hook-ca-floating.nix diff --git a/tests/build-hook.nix b/tests/functional/build-hook.nix similarity index 100% rename from tests/build-hook.nix rename to tests/functional/build-hook.nix diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/functional/build-remote-content-addressed-fixed.sh similarity index 100% rename from tests/build-remote-content-addressed-fixed.sh rename to tests/functional/build-remote-content-addressed-fixed.sh diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/functional/build-remote-content-addressed-floating.sh similarity index 100% rename from tests/build-remote-content-addressed-floating.sh rename to tests/functional/build-remote-content-addressed-floating.sh diff --git a/tests/build-remote-input-addressed.sh b/tests/functional/build-remote-input-addressed.sh similarity index 100% rename from tests/build-remote-input-addressed.sh rename to tests/functional/build-remote-input-addressed.sh diff --git a/tests/build-remote-trustless-after.sh b/tests/functional/build-remote-trustless-after.sh similarity index 100% rename from tests/build-remote-trustless-after.sh rename to tests/functional/build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh similarity index 100% rename from tests/build-remote-trustless-should-fail-0.sh rename to tests/functional/build-remote-trustless-should-fail-0.sh diff --git a/tests/build-remote-trustless-should-pass-0.sh b/tests/functional/build-remote-trustless-should-pass-0.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-0.sh rename to tests/functional/build-remote-trustless-should-pass-0.sh diff --git a/tests/build-remote-trustless-should-pass-1.sh b/tests/functional/build-remote-trustless-should-pass-1.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-1.sh rename to tests/functional/build-remote-trustless-should-pass-1.sh diff --git a/tests/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-2.sh rename to tests/functional/build-remote-trustless-should-pass-2.sh diff --git a/tests/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-3.sh rename to tests/functional/build-remote-trustless-should-pass-3.sh diff --git a/tests/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh similarity index 82% rename from tests/build-remote-trustless.sh rename to tests/functional/build-remote-trustless.sh index 9df44e0c5..81e5253bf 100644 --- a/tests/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -6,7 +6,7 @@ unset NIX_STATE_DIR remoteDir=$TEST_ROOT/remote -# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for +# Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for # more details. nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ --arg busybox $busybox \ diff --git a/tests/build-remote.sh b/tests/functional/build-remote.sh similarity index 100% rename from tests/build-remote.sh rename to tests/functional/build-remote.sh diff --git a/tests/build.sh b/tests/functional/build.sh similarity index 100% rename from tests/build.sh rename to tests/functional/build.sh diff --git a/tests/ca-shell.nix b/tests/functional/ca-shell.nix similarity index 100% rename from tests/ca-shell.nix rename to tests/functional/ca-shell.nix diff --git a/tests/ca/build-cache.sh b/tests/functional/ca/build-cache.sh similarity index 100% rename from tests/ca/build-cache.sh rename to tests/functional/ca/build-cache.sh diff --git a/tests/ca/build-dry.sh b/tests/functional/ca/build-dry.sh similarity index 100% rename from tests/ca/build-dry.sh rename to tests/functional/ca/build-dry.sh diff --git a/tests/ca/build-with-garbage-path.sh b/tests/functional/ca/build-with-garbage-path.sh similarity index 100% rename from tests/ca/build-with-garbage-path.sh rename to tests/functional/ca/build-with-garbage-path.sh diff --git a/tests/ca/build.sh b/tests/functional/ca/build.sh similarity index 100% rename from tests/ca/build.sh rename to tests/functional/ca/build.sh diff --git a/tests/ca/common.sh b/tests/functional/ca/common.sh similarity index 100% rename from tests/ca/common.sh rename to tests/functional/ca/common.sh diff --git a/tests/ca/concurrent-builds.sh b/tests/functional/ca/concurrent-builds.sh similarity index 100% rename from tests/ca/concurrent-builds.sh rename to tests/functional/ca/concurrent-builds.sh diff --git a/tests/ca/config.nix.in b/tests/functional/ca/config.nix.in similarity index 100% rename from tests/ca/config.nix.in rename to tests/functional/ca/config.nix.in diff --git a/tests/ca/content-addressed.nix b/tests/functional/ca/content-addressed.nix similarity index 100% rename from tests/ca/content-addressed.nix rename to tests/functional/ca/content-addressed.nix diff --git a/tests/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh similarity index 100% rename from tests/ca/derivation-json.sh rename to tests/functional/ca/derivation-json.sh diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh similarity index 100% rename from tests/ca/duplicate-realisation-in-closure.sh rename to tests/functional/ca/duplicate-realisation-in-closure.sh diff --git a/tests/ca/flake.nix b/tests/functional/ca/flake.nix similarity index 100% rename from tests/ca/flake.nix rename to tests/functional/ca/flake.nix diff --git a/tests/ca/gc.sh b/tests/functional/ca/gc.sh similarity index 100% rename from tests/ca/gc.sh rename to tests/functional/ca/gc.sh diff --git a/tests/ca/import-derivation.sh b/tests/functional/ca/import-derivation.sh similarity index 100% rename from tests/ca/import-derivation.sh rename to tests/functional/ca/import-derivation.sh diff --git a/tests/ca/local.mk b/tests/functional/ca/local.mk similarity index 94% rename from tests/ca/local.mk rename to tests/functional/ca/local.mk index 0852e592e..fd87b8d1f 100644 --- a/tests/ca/local.mk +++ b/tests/functional/ca/local.mk @@ -25,4 +25,4 @@ clean-files += \ $(d)/config.nix test-deps += \ - tests/ca/config.nix + tests/functional/ca/config.nix diff --git a/tests/ca/new-build-cmd.sh b/tests/functional/ca/new-build-cmd.sh similarity index 100% rename from tests/ca/new-build-cmd.sh rename to tests/functional/ca/new-build-cmd.sh diff --git a/tests/ca/nix-copy.sh b/tests/functional/ca/nix-copy.sh similarity index 100% rename from tests/ca/nix-copy.sh rename to tests/functional/ca/nix-copy.sh diff --git a/tests/ca/nix-run.sh b/tests/functional/ca/nix-run.sh similarity index 100% rename from tests/ca/nix-run.sh rename to tests/functional/ca/nix-run.sh diff --git a/tests/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh similarity index 100% rename from tests/ca/nix-shell.sh rename to tests/functional/ca/nix-shell.sh diff --git a/tests/ca/nondeterministic.nix b/tests/functional/ca/nondeterministic.nix similarity index 100% rename from tests/ca/nondeterministic.nix rename to tests/functional/ca/nondeterministic.nix diff --git a/tests/ca/post-hook.sh b/tests/functional/ca/post-hook.sh similarity index 100% rename from tests/ca/post-hook.sh rename to tests/functional/ca/post-hook.sh diff --git a/tests/ca/racy.nix b/tests/functional/ca/racy.nix similarity index 100% rename from tests/ca/racy.nix rename to tests/functional/ca/racy.nix diff --git a/tests/ca/recursive.sh b/tests/functional/ca/recursive.sh similarity index 100% rename from tests/ca/recursive.sh rename to tests/functional/ca/recursive.sh diff --git a/tests/ca/repl.sh b/tests/functional/ca/repl.sh similarity index 100% rename from tests/ca/repl.sh rename to tests/functional/ca/repl.sh diff --git a/tests/ca/selfref-gc.sh b/tests/functional/ca/selfref-gc.sh similarity index 100% rename from tests/ca/selfref-gc.sh rename to tests/functional/ca/selfref-gc.sh diff --git a/tests/ca/signatures.sh b/tests/functional/ca/signatures.sh similarity index 100% rename from tests/ca/signatures.sh rename to tests/functional/ca/signatures.sh diff --git a/tests/ca/substitute.sh b/tests/functional/ca/substitute.sh similarity index 100% rename from tests/ca/substitute.sh rename to tests/functional/ca/substitute.sh diff --git a/tests/ca/why-depends.sh b/tests/functional/ca/why-depends.sh similarity index 100% rename from tests/ca/why-depends.sh rename to tests/functional/ca/why-depends.sh diff --git a/tests/case-hack.sh b/tests/functional/case-hack.sh similarity index 100% rename from tests/case-hack.sh rename to tests/functional/case-hack.sh diff --git a/tests/case.nar b/tests/functional/case.nar similarity index 100% rename from tests/case.nar rename to tests/functional/case.nar diff --git a/tests/check-refs.nix b/tests/functional/check-refs.nix similarity index 100% rename from tests/check-refs.nix rename to tests/functional/check-refs.nix diff --git a/tests/check-refs.sh b/tests/functional/check-refs.sh similarity index 100% rename from tests/check-refs.sh rename to tests/functional/check-refs.sh diff --git a/tests/check-reqs.nix b/tests/functional/check-reqs.nix similarity index 100% rename from tests/check-reqs.nix rename to tests/functional/check-reqs.nix diff --git a/tests/check-reqs.sh b/tests/functional/check-reqs.sh similarity index 100% rename from tests/check-reqs.sh rename to tests/functional/check-reqs.sh diff --git a/tests/check.nix b/tests/functional/check.nix similarity index 100% rename from tests/check.nix rename to tests/functional/check.nix diff --git a/tests/check.sh b/tests/functional/check.sh similarity index 100% rename from tests/check.sh rename to tests/functional/check.sh diff --git a/tests/common.sh b/tests/functional/common.sh similarity index 100% rename from tests/common.sh rename to tests/functional/common.sh diff --git a/tests/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in similarity index 99% rename from tests/common/vars-and-functions.sh.in rename to tests/functional/common/vars-and-functions.sh.in index 8f9ec4b1a..967d6be54 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -6,7 +6,7 @@ COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. diff --git a/tests/completions.sh b/tests/functional/completions.sh similarity index 100% rename from tests/completions.sh rename to tests/functional/completions.sh diff --git a/tests/compression-levels.sh b/tests/functional/compression-levels.sh similarity index 100% rename from tests/compression-levels.sh rename to tests/functional/compression-levels.sh diff --git a/tests/compute-levels.sh b/tests/functional/compute-levels.sh similarity index 100% rename from tests/compute-levels.sh rename to tests/functional/compute-levels.sh diff --git a/tests/config.nix.in b/tests/functional/config.nix.in similarity index 100% rename from tests/config.nix.in rename to tests/functional/config.nix.in diff --git a/tests/config.sh b/tests/functional/config.sh similarity index 100% rename from tests/config.sh rename to tests/functional/config.sh diff --git a/tests/config/nix-with-substituters.conf b/tests/functional/config/nix-with-substituters.conf similarity index 100% rename from tests/config/nix-with-substituters.conf rename to tests/functional/config/nix-with-substituters.conf diff --git a/tests/db-migration.sh b/tests/functional/db-migration.sh similarity index 100% rename from tests/db-migration.sh rename to tests/functional/db-migration.sh diff --git a/tests/dependencies.builder0.sh b/tests/functional/dependencies.builder0.sh similarity index 100% rename from tests/dependencies.builder0.sh rename to tests/functional/dependencies.builder0.sh diff --git a/tests/dependencies.nix b/tests/functional/dependencies.nix similarity index 100% rename from tests/dependencies.nix rename to tests/functional/dependencies.nix diff --git a/tests/dependencies.sh b/tests/functional/dependencies.sh similarity index 100% rename from tests/dependencies.sh rename to tests/functional/dependencies.sh diff --git a/tests/derivation-json.sh b/tests/functional/derivation-json.sh similarity index 100% rename from tests/derivation-json.sh rename to tests/functional/derivation-json.sh diff --git a/tests/dummy b/tests/functional/dummy similarity index 100% rename from tests/dummy rename to tests/functional/dummy diff --git a/tests/dump-db.sh b/tests/functional/dump-db.sh similarity index 100% rename from tests/dump-db.sh rename to tests/functional/dump-db.sh diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/functional/dyn-drv/build-built-drv.sh similarity index 100% rename from tests/dyn-drv/build-built-drv.sh rename to tests/functional/dyn-drv/build-built-drv.sh diff --git a/tests/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh similarity index 100% rename from tests/dyn-drv/common.sh rename to tests/functional/dyn-drv/common.sh diff --git a/tests/dyn-drv/config.nix.in b/tests/functional/dyn-drv/config.nix.in similarity index 100% rename from tests/dyn-drv/config.nix.in rename to tests/functional/dyn-drv/config.nix.in diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/functional/dyn-drv/dep-built-drv.sh similarity index 100% rename from tests/dyn-drv/dep-built-drv.sh rename to tests/functional/dyn-drv/dep-built-drv.sh diff --git a/tests/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh similarity index 100% rename from tests/dyn-drv/eval-outputOf.sh rename to tests/functional/dyn-drv/eval-outputOf.sh diff --git a/tests/dyn-drv/local.mk b/tests/functional/dyn-drv/local.mk similarity index 87% rename from tests/dyn-drv/local.mk rename to tests/functional/dyn-drv/local.mk index 6b435499b..c87534944 100644 --- a/tests/dyn-drv/local.mk +++ b/tests/functional/dyn-drv/local.mk @@ -12,4 +12,4 @@ clean-files += \ $(d)/config.nix test-deps += \ - tests/dyn-drv/config.nix + tests/functional/dyn-drv/config.nix diff --git a/tests/dyn-drv/old-daemon-error-hack.nix b/tests/functional/dyn-drv/old-daemon-error-hack.nix similarity index 100% rename from tests/dyn-drv/old-daemon-error-hack.nix rename to tests/functional/dyn-drv/old-daemon-error-hack.nix diff --git a/tests/dyn-drv/old-daemon-error-hack.sh b/tests/functional/dyn-drv/old-daemon-error-hack.sh similarity index 100% rename from tests/dyn-drv/old-daemon-error-hack.sh rename to tests/functional/dyn-drv/old-daemon-error-hack.sh diff --git a/tests/dyn-drv/recursive-mod-json.nix b/tests/functional/dyn-drv/recursive-mod-json.nix similarity index 100% rename from tests/dyn-drv/recursive-mod-json.nix rename to tests/functional/dyn-drv/recursive-mod-json.nix diff --git a/tests/dyn-drv/recursive-mod-json.sh b/tests/functional/dyn-drv/recursive-mod-json.sh similarity index 100% rename from tests/dyn-drv/recursive-mod-json.sh rename to tests/functional/dyn-drv/recursive-mod-json.sh diff --git a/tests/dyn-drv/text-hashed-output.nix b/tests/functional/dyn-drv/text-hashed-output.nix similarity index 100% rename from tests/dyn-drv/text-hashed-output.nix rename to tests/functional/dyn-drv/text-hashed-output.nix diff --git a/tests/dyn-drv/text-hashed-output.sh b/tests/functional/dyn-drv/text-hashed-output.sh similarity index 100% rename from tests/dyn-drv/text-hashed-output.sh rename to tests/functional/dyn-drv/text-hashed-output.sh diff --git a/tests/eval-store.sh b/tests/functional/eval-store.sh similarity index 100% rename from tests/eval-store.sh rename to tests/functional/eval-store.sh diff --git a/tests/eval.nix b/tests/functional/eval.nix similarity index 100% rename from tests/eval.nix rename to tests/functional/eval.nix diff --git a/tests/eval.sh b/tests/functional/eval.sh similarity index 100% rename from tests/eval.sh rename to tests/functional/eval.sh diff --git a/tests/experimental-features.sh b/tests/functional/experimental-features.sh similarity index 100% rename from tests/experimental-features.sh rename to tests/functional/experimental-features.sh diff --git a/tests/export-graph.nix b/tests/functional/export-graph.nix similarity index 100% rename from tests/export-graph.nix rename to tests/functional/export-graph.nix diff --git a/tests/export-graph.sh b/tests/functional/export-graph.sh similarity index 100% rename from tests/export-graph.sh rename to tests/functional/export-graph.sh diff --git a/tests/export.sh b/tests/functional/export.sh similarity index 100% rename from tests/export.sh rename to tests/functional/export.sh diff --git a/tests/failing.nix b/tests/functional/failing.nix similarity index 100% rename from tests/failing.nix rename to tests/functional/failing.nix diff --git a/tests/fetchClosure.sh b/tests/functional/fetchClosure.sh similarity index 100% rename from tests/fetchClosure.sh rename to tests/functional/fetchClosure.sh diff --git a/tests/fetchGit.sh b/tests/functional/fetchGit.sh similarity index 100% rename from tests/fetchGit.sh rename to tests/functional/fetchGit.sh diff --git a/tests/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh similarity index 100% rename from tests/fetchGitRefs.sh rename to tests/functional/fetchGitRefs.sh diff --git a/tests/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh similarity index 100% rename from tests/fetchGitSubmodules.sh rename to tests/functional/fetchGitSubmodules.sh diff --git a/tests/fetchMercurial.sh b/tests/functional/fetchMercurial.sh similarity index 100% rename from tests/fetchMercurial.sh rename to tests/functional/fetchMercurial.sh diff --git a/tests/fetchPath.sh b/tests/functional/fetchPath.sh similarity index 100% rename from tests/fetchPath.sh rename to tests/functional/fetchPath.sh diff --git a/tests/fetchTree-file.sh b/tests/functional/fetchTree-file.sh similarity index 100% rename from tests/fetchTree-file.sh rename to tests/functional/fetchTree-file.sh diff --git a/tests/fetchurl.sh b/tests/functional/fetchurl.sh similarity index 100% rename from tests/fetchurl.sh rename to tests/functional/fetchurl.sh diff --git a/tests/filter-source.nix b/tests/functional/filter-source.nix similarity index 100% rename from tests/filter-source.nix rename to tests/functional/filter-source.nix diff --git a/tests/filter-source.sh b/tests/functional/filter-source.sh similarity index 100% rename from tests/filter-source.sh rename to tests/functional/filter-source.sh diff --git a/tests/fixed.builder1.sh b/tests/functional/fixed.builder1.sh similarity index 100% rename from tests/fixed.builder1.sh rename to tests/functional/fixed.builder1.sh diff --git a/tests/fixed.builder2.sh b/tests/functional/fixed.builder2.sh similarity index 100% rename from tests/fixed.builder2.sh rename to tests/functional/fixed.builder2.sh diff --git a/tests/fixed.nix b/tests/functional/fixed.nix similarity index 100% rename from tests/fixed.nix rename to tests/functional/fixed.nix diff --git a/tests/fixed.sh b/tests/functional/fixed.sh similarity index 100% rename from tests/fixed.sh rename to tests/functional/fixed.sh diff --git a/tests/flakes/absolute-attr-paths.sh b/tests/functional/flakes/absolute-attr-paths.sh similarity index 100% rename from tests/flakes/absolute-attr-paths.sh rename to tests/functional/flakes/absolute-attr-paths.sh diff --git a/tests/flakes/absolute-paths.sh b/tests/functional/flakes/absolute-paths.sh similarity index 100% rename from tests/flakes/absolute-paths.sh rename to tests/functional/flakes/absolute-paths.sh diff --git a/tests/flakes/build-paths.sh b/tests/functional/flakes/build-paths.sh similarity index 100% rename from tests/flakes/build-paths.sh rename to tests/functional/flakes/build-paths.sh diff --git a/tests/flakes/bundle.sh b/tests/functional/flakes/bundle.sh similarity index 100% rename from tests/flakes/bundle.sh rename to tests/functional/flakes/bundle.sh diff --git a/tests/flakes/check.sh b/tests/functional/flakes/check.sh similarity index 100% rename from tests/flakes/check.sh rename to tests/functional/flakes/check.sh diff --git a/tests/flakes/circular.sh b/tests/functional/flakes/circular.sh similarity index 100% rename from tests/flakes/circular.sh rename to tests/functional/flakes/circular.sh diff --git a/tests/flakes/common.sh b/tests/functional/flakes/common.sh similarity index 100% rename from tests/flakes/common.sh rename to tests/functional/flakes/common.sh diff --git a/tests/flakes/config.sh b/tests/functional/flakes/config.sh similarity index 100% rename from tests/flakes/config.sh rename to tests/functional/flakes/config.sh diff --git a/tests/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh similarity index 100% rename from tests/flakes/flake-in-submodule.sh rename to tests/functional/flakes/flake-in-submodule.sh diff --git a/tests/flakes/flakes.sh b/tests/functional/flakes/flakes.sh similarity index 100% rename from tests/flakes/flakes.sh rename to tests/functional/flakes/flakes.sh diff --git a/tests/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh similarity index 100% rename from tests/flakes/follow-paths.sh rename to tests/functional/flakes/follow-paths.sh diff --git a/tests/flakes/init.sh b/tests/functional/flakes/init.sh similarity index 100% rename from tests/flakes/init.sh rename to tests/functional/flakes/init.sh diff --git a/tests/flakes/inputs.sh b/tests/functional/flakes/inputs.sh similarity index 100% rename from tests/flakes/inputs.sh rename to tests/functional/flakes/inputs.sh diff --git a/tests/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh similarity index 100% rename from tests/flakes/mercurial.sh rename to tests/functional/flakes/mercurial.sh diff --git a/tests/flakes/run.sh b/tests/functional/flakes/run.sh similarity index 100% rename from tests/flakes/run.sh rename to tests/functional/flakes/run.sh diff --git a/tests/flakes/search-root.sh b/tests/functional/flakes/search-root.sh similarity index 100% rename from tests/flakes/search-root.sh rename to tests/functional/flakes/search-root.sh diff --git a/tests/flakes/show.sh b/tests/functional/flakes/show.sh similarity index 100% rename from tests/flakes/show.sh rename to tests/functional/flakes/show.sh diff --git a/tests/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh similarity index 100% rename from tests/flakes/unlocked-override.sh rename to tests/functional/flakes/unlocked-override.sh diff --git a/tests/fmt.sh b/tests/functional/fmt.sh similarity index 100% rename from tests/fmt.sh rename to tests/functional/fmt.sh diff --git a/tests/fmt.simple.sh b/tests/functional/fmt.simple.sh similarity index 100% rename from tests/fmt.simple.sh rename to tests/functional/fmt.simple.sh diff --git a/tests/function-trace.sh b/tests/functional/function-trace.sh similarity index 100% rename from tests/function-trace.sh rename to tests/functional/function-trace.sh diff --git a/tests/gc-auto.sh b/tests/functional/gc-auto.sh similarity index 100% rename from tests/gc-auto.sh rename to tests/functional/gc-auto.sh diff --git a/tests/gc-concurrent.builder.sh b/tests/functional/gc-concurrent.builder.sh similarity index 100% rename from tests/gc-concurrent.builder.sh rename to tests/functional/gc-concurrent.builder.sh diff --git a/tests/gc-concurrent.nix b/tests/functional/gc-concurrent.nix similarity index 100% rename from tests/gc-concurrent.nix rename to tests/functional/gc-concurrent.nix diff --git a/tests/gc-concurrent.sh b/tests/functional/gc-concurrent.sh similarity index 100% rename from tests/gc-concurrent.sh rename to tests/functional/gc-concurrent.sh diff --git a/tests/gc-concurrent2.builder.sh b/tests/functional/gc-concurrent2.builder.sh similarity index 100% rename from tests/gc-concurrent2.builder.sh rename to tests/functional/gc-concurrent2.builder.sh diff --git a/tests/gc-non-blocking.sh b/tests/functional/gc-non-blocking.sh similarity index 100% rename from tests/gc-non-blocking.sh rename to tests/functional/gc-non-blocking.sh diff --git a/tests/gc-runtime.nix b/tests/functional/gc-runtime.nix similarity index 100% rename from tests/gc-runtime.nix rename to tests/functional/gc-runtime.nix diff --git a/tests/gc-runtime.sh b/tests/functional/gc-runtime.sh similarity index 100% rename from tests/gc-runtime.sh rename to tests/functional/gc-runtime.sh diff --git a/tests/gc.sh b/tests/functional/gc.sh similarity index 100% rename from tests/gc.sh rename to tests/functional/gc.sh diff --git a/tests/hash-check.nix b/tests/functional/hash-check.nix similarity index 100% rename from tests/hash-check.nix rename to tests/functional/hash-check.nix diff --git a/tests/hash.sh b/tests/functional/hash.sh similarity index 100% rename from tests/hash.sh rename to tests/functional/hash.sh diff --git a/tests/hermetic.nix b/tests/functional/hermetic.nix similarity index 100% rename from tests/hermetic.nix rename to tests/functional/hermetic.nix diff --git a/tests/import-derivation.nix b/tests/functional/import-derivation.nix similarity index 100% rename from tests/import-derivation.nix rename to tests/functional/import-derivation.nix diff --git a/tests/import-derivation.sh b/tests/functional/import-derivation.sh similarity index 100% rename from tests/import-derivation.sh rename to tests/functional/import-derivation.sh diff --git a/tests/impure-derivations.nix b/tests/functional/impure-derivations.nix similarity index 100% rename from tests/impure-derivations.nix rename to tests/functional/impure-derivations.nix diff --git a/tests/impure-derivations.sh b/tests/functional/impure-derivations.sh similarity index 100% rename from tests/impure-derivations.sh rename to tests/functional/impure-derivations.sh diff --git a/tests/init.sh b/tests/functional/init.sh similarity index 100% rename from tests/init.sh rename to tests/functional/init.sh diff --git a/tests/install-darwin.sh b/tests/functional/install-darwin.sh similarity index 100% rename from tests/install-darwin.sh rename to tests/functional/install-darwin.sh diff --git a/tests/lang-test-infra.sh b/tests/functional/lang-test-infra.sh similarity index 100% rename from tests/lang-test-infra.sh rename to tests/functional/lang-test-infra.sh diff --git a/tests/lang.sh b/tests/functional/lang.sh similarity index 98% rename from tests/lang.sh rename to tests/functional/lang.sh index 75dbbc38e..f4760eced 100755 --- a/tests/lang.sh +++ b/tests/functional/lang.sh @@ -134,7 +134,7 @@ else echo '' echo 'You can rerun this test with:' echo '' - echo ' _NIX_TEST_ACCEPT=1 make tests/lang.sh.test' + echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' echo '' echo 'to regenerate the files containing the expected output,' echo 'and then view the git diff to decide whether a change is' diff --git a/tests/lang/binary-data b/tests/functional/lang/binary-data similarity index 100% rename from tests/lang/binary-data rename to tests/functional/lang/binary-data diff --git a/tests/lang/data b/tests/functional/lang/data similarity index 100% rename from tests/lang/data rename to tests/functional/lang/data diff --git a/tests/lang/dir1/a.nix b/tests/functional/lang/dir1/a.nix similarity index 100% rename from tests/lang/dir1/a.nix rename to tests/functional/lang/dir1/a.nix diff --git a/tests/lang/dir2/a.nix b/tests/functional/lang/dir2/a.nix similarity index 100% rename from tests/lang/dir2/a.nix rename to tests/functional/lang/dir2/a.nix diff --git a/tests/lang/dir2/b.nix b/tests/functional/lang/dir2/b.nix similarity index 100% rename from tests/lang/dir2/b.nix rename to tests/functional/lang/dir2/b.nix diff --git a/tests/lang/dir3/a.nix b/tests/functional/lang/dir3/a.nix similarity index 100% rename from tests/lang/dir3/a.nix rename to tests/functional/lang/dir3/a.nix diff --git a/tests/lang/dir3/b.nix b/tests/functional/lang/dir3/b.nix similarity index 100% rename from tests/lang/dir3/b.nix rename to tests/functional/lang/dir3/b.nix diff --git a/tests/lang/dir3/c.nix b/tests/functional/lang/dir3/c.nix similarity index 100% rename from tests/lang/dir3/c.nix rename to tests/functional/lang/dir3/c.nix diff --git a/tests/lang/dir4/a.nix b/tests/functional/lang/dir4/a.nix similarity index 100% rename from tests/lang/dir4/a.nix rename to tests/functional/lang/dir4/a.nix diff --git a/tests/lang/dir4/c.nix b/tests/functional/lang/dir4/c.nix similarity index 100% rename from tests/lang/dir4/c.nix rename to tests/functional/lang/dir4/c.nix diff --git a/tests/lang/empty.exp b/tests/functional/lang/empty.exp similarity index 100% rename from tests/lang/empty.exp rename to tests/functional/lang/empty.exp diff --git a/tests/lang/eval-fail-abort.err.exp b/tests/functional/lang/eval-fail-abort.err.exp similarity index 100% rename from tests/lang/eval-fail-abort.err.exp rename to tests/functional/lang/eval-fail-abort.err.exp diff --git a/tests/lang/eval-fail-abort.nix b/tests/functional/lang/eval-fail-abort.nix similarity index 100% rename from tests/lang/eval-fail-abort.nix rename to tests/functional/lang/eval-fail-abort.nix diff --git a/tests/lang/eval-fail-antiquoted-path.err.exp b/tests/functional/lang/eval-fail-antiquoted-path.err.exp similarity index 100% rename from tests/lang/eval-fail-antiquoted-path.err.exp rename to tests/functional/lang/eval-fail-antiquoted-path.err.exp diff --git a/tests/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp similarity index 100% rename from tests/lang/eval-fail-assert.err.exp rename to tests/functional/lang/eval-fail-assert.err.exp diff --git a/tests/lang/eval-fail-assert.nix b/tests/functional/lang/eval-fail-assert.nix similarity index 100% rename from tests/lang/eval-fail-assert.nix rename to tests/functional/lang/eval-fail-assert.nix diff --git a/tests/lang/eval-fail-bad-antiquote-1.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-1.err.exp rename to tests/functional/lang/eval-fail-bad-antiquote-1.err.exp diff --git a/tests/lang/eval-fail-bad-antiquote-2.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-2.err.exp rename to tests/functional/lang/eval-fail-bad-antiquote-2.err.exp diff --git a/tests/lang/eval-fail-bad-antiquote-3.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-3.err.exp rename to tests/functional/lang/eval-fail-bad-antiquote-3.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-1.err.exp rename to tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-1.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-1.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-2.err.exp rename to tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-2.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-2.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-3.err.exp rename to tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-3.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-3.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp similarity index 100% rename from tests/lang/eval-fail-blackhole.err.exp rename to tests/functional/lang/eval-fail-blackhole.err.exp diff --git a/tests/lang/eval-fail-blackhole.nix b/tests/functional/lang/eval-fail-blackhole.nix similarity index 100% rename from tests/lang/eval-fail-blackhole.nix rename to tests/functional/lang/eval-fail-blackhole.nix diff --git a/tests/lang/eval-fail-deepseq.err.exp b/tests/functional/lang/eval-fail-deepseq.err.exp similarity index 100% rename from tests/lang/eval-fail-deepseq.err.exp rename to tests/functional/lang/eval-fail-deepseq.err.exp diff --git a/tests/lang/eval-fail-deepseq.nix b/tests/functional/lang/eval-fail-deepseq.nix similarity index 100% rename from tests/lang/eval-fail-deepseq.nix rename to tests/functional/lang/eval-fail-deepseq.nix diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp similarity index 100% rename from tests/lang/eval-fail-dup-dynamic-attrs.err.exp rename to tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.nix b/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-fail-dup-dynamic-attrs.nix rename to tests/functional/lang/eval-fail-dup-dynamic-attrs.nix diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp similarity index 100% rename from tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp rename to tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.nix b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix similarity index 100% rename from tests/lang/eval-fail-foldlStrict-strict-op-application.nix rename to tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp similarity index 100% rename from tests/lang/eval-fail-fromTOML-timestamps.err.exp rename to tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp diff --git a/tests/lang/eval-fail-fromTOML-timestamps.nix b/tests/functional/lang/eval-fail-fromTOML-timestamps.nix similarity index 100% rename from tests/lang/eval-fail-fromTOML-timestamps.nix rename to tests/functional/lang/eval-fail-fromTOML-timestamps.nix diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp similarity index 100% rename from tests/lang/eval-fail-hashfile-missing.err.exp rename to tests/functional/lang/eval-fail-hashfile-missing.err.exp diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/functional/lang/eval-fail-hashfile-missing.nix similarity index 100% rename from tests/lang/eval-fail-hashfile-missing.nix rename to tests/functional/lang/eval-fail-hashfile-missing.nix diff --git a/tests/lang/eval-fail-list.err.exp b/tests/functional/lang/eval-fail-list.err.exp similarity index 100% rename from tests/lang/eval-fail-list.err.exp rename to tests/functional/lang/eval-fail-list.err.exp diff --git a/tests/lang/eval-fail-list.nix b/tests/functional/lang/eval-fail-list.nix similarity index 100% rename from tests/lang/eval-fail-list.nix rename to tests/functional/lang/eval-fail-list.nix diff --git a/tests/lang/eval-fail-missing-arg.err.exp b/tests/functional/lang/eval-fail-missing-arg.err.exp similarity index 100% rename from tests/lang/eval-fail-missing-arg.err.exp rename to tests/functional/lang/eval-fail-missing-arg.err.exp diff --git a/tests/lang/eval-fail-missing-arg.nix b/tests/functional/lang/eval-fail-missing-arg.nix similarity index 100% rename from tests/lang/eval-fail-missing-arg.nix rename to tests/functional/lang/eval-fail-missing-arg.nix diff --git a/tests/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp similarity index 100% rename from tests/lang/eval-fail-nonexist-path.err.exp rename to tests/functional/lang/eval-fail-nonexist-path.err.exp diff --git a/tests/lang/eval-fail-nonexist-path.nix b/tests/functional/lang/eval-fail-nonexist-path.nix similarity index 100% rename from tests/lang/eval-fail-nonexist-path.nix rename to tests/functional/lang/eval-fail-nonexist-path.nix diff --git a/tests/lang/eval-fail-path-slash.err.exp b/tests/functional/lang/eval-fail-path-slash.err.exp similarity index 100% rename from tests/lang/eval-fail-path-slash.err.exp rename to tests/functional/lang/eval-fail-path-slash.err.exp diff --git a/tests/lang/eval-fail-path-slash.nix b/tests/functional/lang/eval-fail-path-slash.nix similarity index 100% rename from tests/lang/eval-fail-path-slash.nix rename to tests/functional/lang/eval-fail-path-slash.nix diff --git a/tests/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp similarity index 100% rename from tests/lang/eval-fail-recursion.err.exp rename to tests/functional/lang/eval-fail-recursion.err.exp diff --git a/tests/lang/eval-fail-recursion.nix b/tests/functional/lang/eval-fail-recursion.nix similarity index 100% rename from tests/lang/eval-fail-recursion.nix rename to tests/functional/lang/eval-fail-recursion.nix diff --git a/tests/lang/eval-fail-remove.err.exp b/tests/functional/lang/eval-fail-remove.err.exp similarity index 100% rename from tests/lang/eval-fail-remove.err.exp rename to tests/functional/lang/eval-fail-remove.err.exp diff --git a/tests/lang/eval-fail-remove.nix b/tests/functional/lang/eval-fail-remove.nix similarity index 100% rename from tests/lang/eval-fail-remove.nix rename to tests/functional/lang/eval-fail-remove.nix diff --git a/tests/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp similarity index 100% rename from tests/lang/eval-fail-scope-5.err.exp rename to tests/functional/lang/eval-fail-scope-5.err.exp diff --git a/tests/lang/eval-fail-scope-5.nix b/tests/functional/lang/eval-fail-scope-5.nix similarity index 100% rename from tests/lang/eval-fail-scope-5.nix rename to tests/functional/lang/eval-fail-scope-5.nix diff --git a/tests/lang/eval-fail-seq.err.exp b/tests/functional/lang/eval-fail-seq.err.exp similarity index 100% rename from tests/lang/eval-fail-seq.err.exp rename to tests/functional/lang/eval-fail-seq.err.exp diff --git a/tests/lang/eval-fail-seq.nix b/tests/functional/lang/eval-fail-seq.nix similarity index 100% rename from tests/lang/eval-fail-seq.nix rename to tests/functional/lang/eval-fail-seq.nix diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/functional/lang/eval-fail-set-override.err.exp similarity index 100% rename from tests/lang/eval-fail-set-override.err.exp rename to tests/functional/lang/eval-fail-set-override.err.exp diff --git a/tests/lang/eval-fail-set-override.nix b/tests/functional/lang/eval-fail-set-override.nix similarity index 100% rename from tests/lang/eval-fail-set-override.nix rename to tests/functional/lang/eval-fail-set-override.nix diff --git a/tests/lang/eval-fail-set.err.exp b/tests/functional/lang/eval-fail-set.err.exp similarity index 100% rename from tests/lang/eval-fail-set.err.exp rename to tests/functional/lang/eval-fail-set.err.exp diff --git a/tests/lang/eval-fail-set.nix b/tests/functional/lang/eval-fail-set.nix similarity index 100% rename from tests/lang/eval-fail-set.nix rename to tests/functional/lang/eval-fail-set.nix diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/functional/lang/eval-fail-substring.err.exp similarity index 100% rename from tests/lang/eval-fail-substring.err.exp rename to tests/functional/lang/eval-fail-substring.err.exp diff --git a/tests/lang/eval-fail-substring.nix b/tests/functional/lang/eval-fail-substring.nix similarity index 100% rename from tests/lang/eval-fail-substring.nix rename to tests/functional/lang/eval-fail-substring.nix diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/functional/lang/eval-fail-to-path.err.exp similarity index 100% rename from tests/lang/eval-fail-to-path.err.exp rename to tests/functional/lang/eval-fail-to-path.err.exp diff --git a/tests/lang/eval-fail-to-path.nix b/tests/functional/lang/eval-fail-to-path.nix similarity index 100% rename from tests/lang/eval-fail-to-path.nix rename to tests/functional/lang/eval-fail-to-path.nix diff --git a/tests/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp similarity index 100% rename from tests/lang/eval-fail-toJSON.err.exp rename to tests/functional/lang/eval-fail-toJSON.err.exp diff --git a/tests/lang/eval-fail-toJSON.nix b/tests/functional/lang/eval-fail-toJSON.nix similarity index 100% rename from tests/lang/eval-fail-toJSON.nix rename to tests/functional/lang/eval-fail-toJSON.nix diff --git a/tests/lang/eval-fail-undeclared-arg.err.exp b/tests/functional/lang/eval-fail-undeclared-arg.err.exp similarity index 100% rename from tests/lang/eval-fail-undeclared-arg.err.exp rename to tests/functional/lang/eval-fail-undeclared-arg.err.exp diff --git a/tests/lang/eval-fail-undeclared-arg.nix b/tests/functional/lang/eval-fail-undeclared-arg.nix similarity index 100% rename from tests/lang/eval-fail-undeclared-arg.nix rename to tests/functional/lang/eval-fail-undeclared-arg.nix diff --git a/tests/lang/eval-okay-any-all.exp b/tests/functional/lang/eval-okay-any-all.exp similarity index 100% rename from tests/lang/eval-okay-any-all.exp rename to tests/functional/lang/eval-okay-any-all.exp diff --git a/tests/lang/eval-okay-any-all.nix b/tests/functional/lang/eval-okay-any-all.nix similarity index 100% rename from tests/lang/eval-okay-any-all.nix rename to tests/functional/lang/eval-okay-any-all.nix diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/functional/lang/eval-okay-arithmetic.exp similarity index 100% rename from tests/lang/eval-okay-arithmetic.exp rename to tests/functional/lang/eval-okay-arithmetic.exp diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/functional/lang/eval-okay-arithmetic.nix similarity index 100% rename from tests/lang/eval-okay-arithmetic.nix rename to tests/functional/lang/eval-okay-arithmetic.nix diff --git a/tests/lang/eval-okay-attrnames.exp b/tests/functional/lang/eval-okay-attrnames.exp similarity index 100% rename from tests/lang/eval-okay-attrnames.exp rename to tests/functional/lang/eval-okay-attrnames.exp diff --git a/tests/lang/eval-okay-attrnames.nix b/tests/functional/lang/eval-okay-attrnames.nix similarity index 100% rename from tests/lang/eval-okay-attrnames.nix rename to tests/functional/lang/eval-okay-attrnames.nix diff --git a/tests/lang/eval-okay-attrs.exp b/tests/functional/lang/eval-okay-attrs.exp similarity index 100% rename from tests/lang/eval-okay-attrs.exp rename to tests/functional/lang/eval-okay-attrs.exp diff --git a/tests/lang/eval-okay-attrs.nix b/tests/functional/lang/eval-okay-attrs.nix similarity index 100% rename from tests/lang/eval-okay-attrs.nix rename to tests/functional/lang/eval-okay-attrs.nix diff --git a/tests/lang/eval-okay-attrs2.exp b/tests/functional/lang/eval-okay-attrs2.exp similarity index 100% rename from tests/lang/eval-okay-attrs2.exp rename to tests/functional/lang/eval-okay-attrs2.exp diff --git a/tests/lang/eval-okay-attrs2.nix b/tests/functional/lang/eval-okay-attrs2.nix similarity index 100% rename from tests/lang/eval-okay-attrs2.nix rename to tests/functional/lang/eval-okay-attrs2.nix diff --git a/tests/lang/eval-okay-attrs3.exp b/tests/functional/lang/eval-okay-attrs3.exp similarity index 100% rename from tests/lang/eval-okay-attrs3.exp rename to tests/functional/lang/eval-okay-attrs3.exp diff --git a/tests/lang/eval-okay-attrs3.nix b/tests/functional/lang/eval-okay-attrs3.nix similarity index 100% rename from tests/lang/eval-okay-attrs3.nix rename to tests/functional/lang/eval-okay-attrs3.nix diff --git a/tests/lang/eval-okay-attrs4.exp b/tests/functional/lang/eval-okay-attrs4.exp similarity index 100% rename from tests/lang/eval-okay-attrs4.exp rename to tests/functional/lang/eval-okay-attrs4.exp diff --git a/tests/lang/eval-okay-attrs4.nix b/tests/functional/lang/eval-okay-attrs4.nix similarity index 100% rename from tests/lang/eval-okay-attrs4.nix rename to tests/functional/lang/eval-okay-attrs4.nix diff --git a/tests/lang/eval-okay-attrs5.exp b/tests/functional/lang/eval-okay-attrs5.exp similarity index 100% rename from tests/lang/eval-okay-attrs5.exp rename to tests/functional/lang/eval-okay-attrs5.exp diff --git a/tests/lang/eval-okay-attrs5.nix b/tests/functional/lang/eval-okay-attrs5.nix similarity index 100% rename from tests/lang/eval-okay-attrs5.nix rename to tests/functional/lang/eval-okay-attrs5.nix diff --git a/tests/lang/eval-okay-attrs6.exp b/tests/functional/lang/eval-okay-attrs6.exp similarity index 100% rename from tests/lang/eval-okay-attrs6.exp rename to tests/functional/lang/eval-okay-attrs6.exp diff --git a/tests/lang/eval-okay-attrs6.nix b/tests/functional/lang/eval-okay-attrs6.nix similarity index 100% rename from tests/lang/eval-okay-attrs6.nix rename to tests/functional/lang/eval-okay-attrs6.nix diff --git a/tests/lang/eval-okay-autoargs.exp b/tests/functional/lang/eval-okay-autoargs.exp similarity index 100% rename from tests/lang/eval-okay-autoargs.exp rename to tests/functional/lang/eval-okay-autoargs.exp diff --git a/tests/lang/eval-okay-autoargs.flags b/tests/functional/lang/eval-okay-autoargs.flags similarity index 100% rename from tests/lang/eval-okay-autoargs.flags rename to tests/functional/lang/eval-okay-autoargs.flags diff --git a/tests/lang/eval-okay-autoargs.nix b/tests/functional/lang/eval-okay-autoargs.nix similarity index 100% rename from tests/lang/eval-okay-autoargs.nix rename to tests/functional/lang/eval-okay-autoargs.nix diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/functional/lang/eval-okay-backslash-newline-1.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.exp rename to tests/functional/lang/eval-okay-backslash-newline-1.exp diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/functional/lang/eval-okay-backslash-newline-1.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.nix rename to tests/functional/lang/eval-okay-backslash-newline-1.nix diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/functional/lang/eval-okay-backslash-newline-2.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.exp rename to tests/functional/lang/eval-okay-backslash-newline-2.exp diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/functional/lang/eval-okay-backslash-newline-2.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.nix rename to tests/functional/lang/eval-okay-backslash-newline-2.nix diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/functional/lang/eval-okay-builtins-add.exp similarity index 100% rename from tests/lang/eval-okay-builtins-add.exp rename to tests/functional/lang/eval-okay-builtins-add.exp diff --git a/tests/lang/eval-okay-builtins-add.nix b/tests/functional/lang/eval-okay-builtins-add.nix similarity index 100% rename from tests/lang/eval-okay-builtins-add.nix rename to tests/functional/lang/eval-okay-builtins-add.nix diff --git a/tests/lang/eval-okay-builtins.exp b/tests/functional/lang/eval-okay-builtins.exp similarity index 100% rename from tests/lang/eval-okay-builtins.exp rename to tests/functional/lang/eval-okay-builtins.exp diff --git a/tests/lang/eval-okay-builtins.nix b/tests/functional/lang/eval-okay-builtins.nix similarity index 100% rename from tests/lang/eval-okay-builtins.nix rename to tests/functional/lang/eval-okay-builtins.nix diff --git a/tests/lang/eval-okay-callable-attrs.exp b/tests/functional/lang/eval-okay-callable-attrs.exp similarity index 100% rename from tests/lang/eval-okay-callable-attrs.exp rename to tests/functional/lang/eval-okay-callable-attrs.exp diff --git a/tests/lang/eval-okay-callable-attrs.nix b/tests/functional/lang/eval-okay-callable-attrs.nix similarity index 100% rename from tests/lang/eval-okay-callable-attrs.nix rename to tests/functional/lang/eval-okay-callable-attrs.nix diff --git a/tests/lang/eval-okay-catattrs.exp b/tests/functional/lang/eval-okay-catattrs.exp similarity index 100% rename from tests/lang/eval-okay-catattrs.exp rename to tests/functional/lang/eval-okay-catattrs.exp diff --git a/tests/lang/eval-okay-catattrs.nix b/tests/functional/lang/eval-okay-catattrs.nix similarity index 100% rename from tests/lang/eval-okay-catattrs.nix rename to tests/functional/lang/eval-okay-catattrs.nix diff --git a/tests/lang/eval-okay-closure.exp b/tests/functional/lang/eval-okay-closure.exp similarity index 100% rename from tests/lang/eval-okay-closure.exp rename to tests/functional/lang/eval-okay-closure.exp diff --git a/tests/lang/eval-okay-closure.exp.xml b/tests/functional/lang/eval-okay-closure.exp.xml similarity index 100% rename from tests/lang/eval-okay-closure.exp.xml rename to tests/functional/lang/eval-okay-closure.exp.xml diff --git a/tests/lang/eval-okay-closure.nix b/tests/functional/lang/eval-okay-closure.nix similarity index 100% rename from tests/lang/eval-okay-closure.nix rename to tests/functional/lang/eval-okay-closure.nix diff --git a/tests/lang/eval-okay-comments.exp b/tests/functional/lang/eval-okay-comments.exp similarity index 100% rename from tests/lang/eval-okay-comments.exp rename to tests/functional/lang/eval-okay-comments.exp diff --git a/tests/lang/eval-okay-comments.nix b/tests/functional/lang/eval-okay-comments.nix similarity index 100% rename from tests/lang/eval-okay-comments.nix rename to tests/functional/lang/eval-okay-comments.nix diff --git a/tests/lang/eval-okay-concat.exp b/tests/functional/lang/eval-okay-concat.exp similarity index 100% rename from tests/lang/eval-okay-concat.exp rename to tests/functional/lang/eval-okay-concat.exp diff --git a/tests/lang/eval-okay-concat.nix b/tests/functional/lang/eval-okay-concat.nix similarity index 100% rename from tests/lang/eval-okay-concat.nix rename to tests/functional/lang/eval-okay-concat.nix diff --git a/tests/lang/eval-okay-concatmap.exp b/tests/functional/lang/eval-okay-concatmap.exp similarity index 100% rename from tests/lang/eval-okay-concatmap.exp rename to tests/functional/lang/eval-okay-concatmap.exp diff --git a/tests/lang/eval-okay-concatmap.nix b/tests/functional/lang/eval-okay-concatmap.nix similarity index 100% rename from tests/lang/eval-okay-concatmap.nix rename to tests/functional/lang/eval-okay-concatmap.nix diff --git a/tests/lang/eval-okay-concatstringssep.exp b/tests/functional/lang/eval-okay-concatstringssep.exp similarity index 100% rename from tests/lang/eval-okay-concatstringssep.exp rename to tests/functional/lang/eval-okay-concatstringssep.exp diff --git a/tests/lang/eval-okay-concatstringssep.nix b/tests/functional/lang/eval-okay-concatstringssep.nix similarity index 100% rename from tests/lang/eval-okay-concatstringssep.nix rename to tests/functional/lang/eval-okay-concatstringssep.nix diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/functional/lang/eval-okay-context-introspection.exp similarity index 100% rename from tests/lang/eval-okay-context-introspection.exp rename to tests/functional/lang/eval-okay-context-introspection.exp diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/functional/lang/eval-okay-context-introspection.nix similarity index 100% rename from tests/lang/eval-okay-context-introspection.nix rename to tests/functional/lang/eval-okay-context-introspection.nix diff --git a/tests/lang/eval-okay-context.exp b/tests/functional/lang/eval-okay-context.exp similarity index 100% rename from tests/lang/eval-okay-context.exp rename to tests/functional/lang/eval-okay-context.exp diff --git a/tests/lang/eval-okay-context.nix b/tests/functional/lang/eval-okay-context.nix similarity index 100% rename from tests/lang/eval-okay-context.nix rename to tests/functional/lang/eval-okay-context.nix diff --git a/tests/lang/eval-okay-curpos.exp b/tests/functional/lang/eval-okay-curpos.exp similarity index 100% rename from tests/lang/eval-okay-curpos.exp rename to tests/functional/lang/eval-okay-curpos.exp diff --git a/tests/lang/eval-okay-curpos.nix b/tests/functional/lang/eval-okay-curpos.nix similarity index 100% rename from tests/lang/eval-okay-curpos.nix rename to tests/functional/lang/eval-okay-curpos.nix diff --git a/tests/lang/eval-okay-deepseq.exp b/tests/functional/lang/eval-okay-deepseq.exp similarity index 100% rename from tests/lang/eval-okay-deepseq.exp rename to tests/functional/lang/eval-okay-deepseq.exp diff --git a/tests/lang/eval-okay-deepseq.nix b/tests/functional/lang/eval-okay-deepseq.nix similarity index 100% rename from tests/lang/eval-okay-deepseq.nix rename to tests/functional/lang/eval-okay-deepseq.nix diff --git a/tests/lang/eval-okay-delayed-with-inherit.exp b/tests/functional/lang/eval-okay-delayed-with-inherit.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.exp rename to tests/functional/lang/eval-okay-delayed-with-inherit.exp diff --git a/tests/lang/eval-okay-delayed-with-inherit.nix b/tests/functional/lang/eval-okay-delayed-with-inherit.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.nix rename to tests/functional/lang/eval-okay-delayed-with-inherit.nix diff --git a/tests/lang/eval-okay-delayed-with.exp b/tests/functional/lang/eval-okay-delayed-with.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with.exp rename to tests/functional/lang/eval-okay-delayed-with.exp diff --git a/tests/lang/eval-okay-delayed-with.nix b/tests/functional/lang/eval-okay-delayed-with.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with.nix rename to tests/functional/lang/eval-okay-delayed-with.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-2.exp b/tests/functional/lang/eval-okay-dynamic-attrs-2.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-2.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-2.nix b/tests/functional/lang/eval-okay-dynamic-attrs-2.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-2.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.exp b/tests/functional/lang/eval-okay-dynamic-attrs-bare.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.nix b/tests/functional/lang/eval-okay-dynamic-attrs-bare.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.nix diff --git a/tests/lang/eval-okay-dynamic-attrs.exp b/tests/functional/lang/eval-okay-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-dynamic-attrs.nix b/tests/functional/lang/eval-okay-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-elem.exp b/tests/functional/lang/eval-okay-elem.exp similarity index 100% rename from tests/lang/eval-okay-elem.exp rename to tests/functional/lang/eval-okay-elem.exp diff --git a/tests/lang/eval-okay-elem.nix b/tests/functional/lang/eval-okay-elem.nix similarity index 100% rename from tests/lang/eval-okay-elem.nix rename to tests/functional/lang/eval-okay-elem.nix diff --git a/tests/lang/eval-okay-empty-args.exp b/tests/functional/lang/eval-okay-empty-args.exp similarity index 100% rename from tests/lang/eval-okay-empty-args.exp rename to tests/functional/lang/eval-okay-empty-args.exp diff --git a/tests/lang/eval-okay-empty-args.nix b/tests/functional/lang/eval-okay-empty-args.nix similarity index 100% rename from tests/lang/eval-okay-empty-args.nix rename to tests/functional/lang/eval-okay-empty-args.nix diff --git a/tests/lang/eval-okay-eq-derivations.exp b/tests/functional/lang/eval-okay-eq-derivations.exp similarity index 100% rename from tests/lang/eval-okay-eq-derivations.exp rename to tests/functional/lang/eval-okay-eq-derivations.exp diff --git a/tests/lang/eval-okay-eq-derivations.nix b/tests/functional/lang/eval-okay-eq-derivations.nix similarity index 100% rename from tests/lang/eval-okay-eq-derivations.nix rename to tests/functional/lang/eval-okay-eq-derivations.nix diff --git a/tests/lang/eval-okay-eq.exp b/tests/functional/lang/eval-okay-eq.exp similarity index 100% rename from tests/lang/eval-okay-eq.exp rename to tests/functional/lang/eval-okay-eq.exp diff --git a/tests/lang/eval-okay-eq.nix b/tests/functional/lang/eval-okay-eq.nix similarity index 100% rename from tests/lang/eval-okay-eq.nix rename to tests/functional/lang/eval-okay-eq.nix diff --git a/tests/lang/eval-okay-filter.exp b/tests/functional/lang/eval-okay-filter.exp similarity index 100% rename from tests/lang/eval-okay-filter.exp rename to tests/functional/lang/eval-okay-filter.exp diff --git a/tests/lang/eval-okay-filter.nix b/tests/functional/lang/eval-okay-filter.nix similarity index 100% rename from tests/lang/eval-okay-filter.nix rename to tests/functional/lang/eval-okay-filter.nix diff --git a/tests/lang/eval-okay-flake-ref-to-string.exp b/tests/functional/lang/eval-okay-flake-ref-to-string.exp similarity index 100% rename from tests/lang/eval-okay-flake-ref-to-string.exp rename to tests/functional/lang/eval-okay-flake-ref-to-string.exp diff --git a/tests/lang/eval-okay-flake-ref-to-string.nix b/tests/functional/lang/eval-okay-flake-ref-to-string.nix similarity index 100% rename from tests/lang/eval-okay-flake-ref-to-string.nix rename to tests/functional/lang/eval-okay-flake-ref-to-string.nix diff --git a/tests/lang/eval-okay-flatten.exp b/tests/functional/lang/eval-okay-flatten.exp similarity index 100% rename from tests/lang/eval-okay-flatten.exp rename to tests/functional/lang/eval-okay-flatten.exp diff --git a/tests/lang/eval-okay-flatten.nix b/tests/functional/lang/eval-okay-flatten.nix similarity index 100% rename from tests/lang/eval-okay-flatten.nix rename to tests/functional/lang/eval-okay-flatten.nix diff --git a/tests/lang/eval-okay-float.exp b/tests/functional/lang/eval-okay-float.exp similarity index 100% rename from tests/lang/eval-okay-float.exp rename to tests/functional/lang/eval-okay-float.exp diff --git a/tests/lang/eval-okay-float.nix b/tests/functional/lang/eval-okay-float.nix similarity index 100% rename from tests/lang/eval-okay-float.nix rename to tests/functional/lang/eval-okay-float.nix diff --git a/tests/lang/eval-okay-floor-ceil.exp b/tests/functional/lang/eval-okay-floor-ceil.exp similarity index 100% rename from tests/lang/eval-okay-floor-ceil.exp rename to tests/functional/lang/eval-okay-floor-ceil.exp diff --git a/tests/lang/eval-okay-floor-ceil.nix b/tests/functional/lang/eval-okay-floor-ceil.nix similarity index 100% rename from tests/lang/eval-okay-floor-ceil.nix rename to tests/functional/lang/eval-okay-floor-ceil.nix diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-elements.exp rename to tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-elements.nix rename to tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp rename to tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix rename to tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix diff --git a/tests/lang/eval-okay-foldlStrict.exp b/tests/functional/lang/eval-okay-foldlStrict.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict.exp rename to tests/functional/lang/eval-okay-foldlStrict.exp diff --git a/tests/lang/eval-okay-foldlStrict.nix b/tests/functional/lang/eval-okay-foldlStrict.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict.nix rename to tests/functional/lang/eval-okay-foldlStrict.nix diff --git a/tests/lang/eval-okay-fromTOML-timestamps.exp b/tests/functional/lang/eval-okay-fromTOML-timestamps.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.exp rename to tests/functional/lang/eval-okay-fromTOML-timestamps.exp diff --git a/tests/lang/eval-okay-fromTOML-timestamps.flags b/tests/functional/lang/eval-okay-fromTOML-timestamps.flags similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.flags rename to tests/functional/lang/eval-okay-fromTOML-timestamps.flags diff --git a/tests/lang/eval-okay-fromTOML-timestamps.nix b/tests/functional/lang/eval-okay-fromTOML-timestamps.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.nix rename to tests/functional/lang/eval-okay-fromTOML-timestamps.nix diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/functional/lang/eval-okay-fromTOML.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML.exp rename to tests/functional/lang/eval-okay-fromTOML.exp diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/functional/lang/eval-okay-fromTOML.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML.nix rename to tests/functional/lang/eval-okay-fromTOML.nix diff --git a/tests/lang/eval-okay-fromjson-escapes.exp b/tests/functional/lang/eval-okay-fromjson-escapes.exp similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.exp rename to tests/functional/lang/eval-okay-fromjson-escapes.exp diff --git a/tests/lang/eval-okay-fromjson-escapes.nix b/tests/functional/lang/eval-okay-fromjson-escapes.nix similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.nix rename to tests/functional/lang/eval-okay-fromjson-escapes.nix diff --git a/tests/lang/eval-okay-fromjson.exp b/tests/functional/lang/eval-okay-fromjson.exp similarity index 100% rename from tests/lang/eval-okay-fromjson.exp rename to tests/functional/lang/eval-okay-fromjson.exp diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/functional/lang/eval-okay-fromjson.nix similarity index 100% rename from tests/lang/eval-okay-fromjson.nix rename to tests/functional/lang/eval-okay-fromjson.nix diff --git a/tests/lang/eval-okay-functionargs.exp b/tests/functional/lang/eval-okay-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-functionargs.exp rename to tests/functional/lang/eval-okay-functionargs.exp diff --git a/tests/lang/eval-okay-functionargs.exp.xml b/tests/functional/lang/eval-okay-functionargs.exp.xml similarity index 100% rename from tests/lang/eval-okay-functionargs.exp.xml rename to tests/functional/lang/eval-okay-functionargs.exp.xml diff --git a/tests/lang/eval-okay-functionargs.nix b/tests/functional/lang/eval-okay-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-functionargs.nix rename to tests/functional/lang/eval-okay-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/functional/lang/eval-okay-getattrpos-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.exp rename to tests/functional/lang/eval-okay-getattrpos-functionargs.exp diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/functional/lang/eval-okay-getattrpos-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.nix rename to tests/functional/lang/eval-okay-getattrpos-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/functional/lang/eval-okay-getattrpos-undefined.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.exp rename to tests/functional/lang/eval-okay-getattrpos-undefined.exp diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/functional/lang/eval-okay-getattrpos-undefined.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.nix rename to tests/functional/lang/eval-okay-getattrpos-undefined.nix diff --git a/tests/lang/eval-okay-getattrpos.exp b/tests/functional/lang/eval-okay-getattrpos.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos.exp rename to tests/functional/lang/eval-okay-getattrpos.exp diff --git a/tests/lang/eval-okay-getattrpos.nix b/tests/functional/lang/eval-okay-getattrpos.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos.nix rename to tests/functional/lang/eval-okay-getattrpos.nix diff --git a/tests/lang/eval-okay-getenv.exp b/tests/functional/lang/eval-okay-getenv.exp similarity index 100% rename from tests/lang/eval-okay-getenv.exp rename to tests/functional/lang/eval-okay-getenv.exp diff --git a/tests/lang/eval-okay-getenv.nix b/tests/functional/lang/eval-okay-getenv.nix similarity index 100% rename from tests/lang/eval-okay-getenv.nix rename to tests/functional/lang/eval-okay-getenv.nix diff --git a/tests/lang/eval-okay-groupBy.exp b/tests/functional/lang/eval-okay-groupBy.exp similarity index 100% rename from tests/lang/eval-okay-groupBy.exp rename to tests/functional/lang/eval-okay-groupBy.exp diff --git a/tests/lang/eval-okay-groupBy.nix b/tests/functional/lang/eval-okay-groupBy.nix similarity index 100% rename from tests/lang/eval-okay-groupBy.nix rename to tests/functional/lang/eval-okay-groupBy.nix diff --git a/tests/lang/eval-okay-hash.exp b/tests/functional/lang/eval-okay-hash.exp similarity index 100% rename from tests/lang/eval-okay-hash.exp rename to tests/functional/lang/eval-okay-hash.exp diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/functional/lang/eval-okay-hashfile.exp similarity index 100% rename from tests/lang/eval-okay-hashfile.exp rename to tests/functional/lang/eval-okay-hashfile.exp diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/functional/lang/eval-okay-hashfile.nix similarity index 100% rename from tests/lang/eval-okay-hashfile.nix rename to tests/functional/lang/eval-okay-hashfile.nix diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/functional/lang/eval-okay-hashstring.exp similarity index 100% rename from tests/lang/eval-okay-hashstring.exp rename to tests/functional/lang/eval-okay-hashstring.exp diff --git a/tests/lang/eval-okay-hashstring.nix b/tests/functional/lang/eval-okay-hashstring.nix similarity index 100% rename from tests/lang/eval-okay-hashstring.nix rename to tests/functional/lang/eval-okay-hashstring.nix diff --git a/tests/lang/eval-okay-if.exp b/tests/functional/lang/eval-okay-if.exp similarity index 100% rename from tests/lang/eval-okay-if.exp rename to tests/functional/lang/eval-okay-if.exp diff --git a/tests/lang/eval-okay-if.nix b/tests/functional/lang/eval-okay-if.nix similarity index 100% rename from tests/lang/eval-okay-if.nix rename to tests/functional/lang/eval-okay-if.nix diff --git a/tests/lang/eval-okay-import.exp b/tests/functional/lang/eval-okay-import.exp similarity index 100% rename from tests/lang/eval-okay-import.exp rename to tests/functional/lang/eval-okay-import.exp diff --git a/tests/lang/eval-okay-import.nix b/tests/functional/lang/eval-okay-import.nix similarity index 100% rename from tests/lang/eval-okay-import.nix rename to tests/functional/lang/eval-okay-import.nix diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/functional/lang/eval-okay-ind-string.exp similarity index 100% rename from tests/lang/eval-okay-ind-string.exp rename to tests/functional/lang/eval-okay-ind-string.exp diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/functional/lang/eval-okay-ind-string.nix similarity index 100% rename from tests/lang/eval-okay-ind-string.nix rename to tests/functional/lang/eval-okay-ind-string.nix diff --git a/tests/lang/eval-okay-intersectAttrs.exp b/tests/functional/lang/eval-okay-intersectAttrs.exp similarity index 100% rename from tests/lang/eval-okay-intersectAttrs.exp rename to tests/functional/lang/eval-okay-intersectAttrs.exp diff --git a/tests/lang/eval-okay-intersectAttrs.nix b/tests/functional/lang/eval-okay-intersectAttrs.nix similarity index 100% rename from tests/lang/eval-okay-intersectAttrs.nix rename to tests/functional/lang/eval-okay-intersectAttrs.nix diff --git a/tests/lang/eval-okay-let.exp b/tests/functional/lang/eval-okay-let.exp similarity index 100% rename from tests/lang/eval-okay-let.exp rename to tests/functional/lang/eval-okay-let.exp diff --git a/tests/lang/eval-okay-let.nix b/tests/functional/lang/eval-okay-let.nix similarity index 100% rename from tests/lang/eval-okay-let.nix rename to tests/functional/lang/eval-okay-let.nix diff --git a/tests/lang/eval-okay-list.exp b/tests/functional/lang/eval-okay-list.exp similarity index 100% rename from tests/lang/eval-okay-list.exp rename to tests/functional/lang/eval-okay-list.exp diff --git a/tests/lang/eval-okay-list.nix b/tests/functional/lang/eval-okay-list.nix similarity index 100% rename from tests/lang/eval-okay-list.nix rename to tests/functional/lang/eval-okay-list.nix diff --git a/tests/lang/eval-okay-listtoattrs.exp b/tests/functional/lang/eval-okay-listtoattrs.exp similarity index 100% rename from tests/lang/eval-okay-listtoattrs.exp rename to tests/functional/lang/eval-okay-listtoattrs.exp diff --git a/tests/lang/eval-okay-listtoattrs.nix b/tests/functional/lang/eval-okay-listtoattrs.nix similarity index 100% rename from tests/lang/eval-okay-listtoattrs.nix rename to tests/functional/lang/eval-okay-listtoattrs.nix diff --git a/tests/lang/eval-okay-logic.exp b/tests/functional/lang/eval-okay-logic.exp similarity index 100% rename from tests/lang/eval-okay-logic.exp rename to tests/functional/lang/eval-okay-logic.exp diff --git a/tests/lang/eval-okay-logic.nix b/tests/functional/lang/eval-okay-logic.nix similarity index 100% rename from tests/lang/eval-okay-logic.nix rename to tests/functional/lang/eval-okay-logic.nix diff --git a/tests/lang/eval-okay-map.exp b/tests/functional/lang/eval-okay-map.exp similarity index 100% rename from tests/lang/eval-okay-map.exp rename to tests/functional/lang/eval-okay-map.exp diff --git a/tests/lang/eval-okay-map.nix b/tests/functional/lang/eval-okay-map.nix similarity index 100% rename from tests/lang/eval-okay-map.nix rename to tests/functional/lang/eval-okay-map.nix diff --git a/tests/lang/eval-okay-mapattrs.exp b/tests/functional/lang/eval-okay-mapattrs.exp similarity index 100% rename from tests/lang/eval-okay-mapattrs.exp rename to tests/functional/lang/eval-okay-mapattrs.exp diff --git a/tests/lang/eval-okay-mapattrs.nix b/tests/functional/lang/eval-okay-mapattrs.nix similarity index 100% rename from tests/lang/eval-okay-mapattrs.nix rename to tests/functional/lang/eval-okay-mapattrs.nix diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.exp b/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-merge-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-merge-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.nix b/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-merge-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-merge-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/functional/lang/eval-okay-nested-with.exp similarity index 100% rename from tests/lang/eval-okay-nested-with.exp rename to tests/functional/lang/eval-okay-nested-with.exp diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/functional/lang/eval-okay-nested-with.nix similarity index 100% rename from tests/lang/eval-okay-nested-with.nix rename to tests/functional/lang/eval-okay-nested-with.nix diff --git a/tests/lang/eval-okay-new-let.exp b/tests/functional/lang/eval-okay-new-let.exp similarity index 100% rename from tests/lang/eval-okay-new-let.exp rename to tests/functional/lang/eval-okay-new-let.exp diff --git a/tests/lang/eval-okay-new-let.nix b/tests/functional/lang/eval-okay-new-let.nix similarity index 100% rename from tests/lang/eval-okay-new-let.nix rename to tests/functional/lang/eval-okay-new-let.nix diff --git a/tests/lang/eval-okay-null-dynamic-attrs.exp b/tests/functional/lang/eval-okay-null-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-null-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-null-dynamic-attrs.nix b/tests/functional/lang/eval-okay-null-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-null-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-overrides.exp b/tests/functional/lang/eval-okay-overrides.exp similarity index 100% rename from tests/lang/eval-okay-overrides.exp rename to tests/functional/lang/eval-okay-overrides.exp diff --git a/tests/lang/eval-okay-overrides.nix b/tests/functional/lang/eval-okay-overrides.nix similarity index 100% rename from tests/lang/eval-okay-overrides.nix rename to tests/functional/lang/eval-okay-overrides.nix diff --git a/tests/lang/eval-okay-parse-flake-ref.exp b/tests/functional/lang/eval-okay-parse-flake-ref.exp similarity index 100% rename from tests/lang/eval-okay-parse-flake-ref.exp rename to tests/functional/lang/eval-okay-parse-flake-ref.exp diff --git a/tests/lang/eval-okay-parse-flake-ref.nix b/tests/functional/lang/eval-okay-parse-flake-ref.nix similarity index 100% rename from tests/lang/eval-okay-parse-flake-ref.nix rename to tests/functional/lang/eval-okay-parse-flake-ref.nix diff --git a/tests/lang/eval-okay-partition.exp b/tests/functional/lang/eval-okay-partition.exp similarity index 100% rename from tests/lang/eval-okay-partition.exp rename to tests/functional/lang/eval-okay-partition.exp diff --git a/tests/lang/eval-okay-partition.nix b/tests/functional/lang/eval-okay-partition.nix similarity index 100% rename from tests/lang/eval-okay-partition.nix rename to tests/functional/lang/eval-okay-partition.nix diff --git a/tests/lang/eval-okay-path-string-interpolation.exp b/tests/functional/lang/eval-okay-path-string-interpolation.exp similarity index 100% rename from tests/lang/eval-okay-path-string-interpolation.exp rename to tests/functional/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-string-interpolation.nix b/tests/functional/lang/eval-okay-path-string-interpolation.nix similarity index 100% rename from tests/lang/eval-okay-path-string-interpolation.nix rename to tests/functional/lang/eval-okay-path-string-interpolation.nix diff --git a/tests/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp similarity index 100% rename from tests/lang/eval-okay-path.exp rename to tests/functional/lang/eval-okay-path.exp diff --git a/tests/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix similarity index 100% rename from tests/lang/eval-okay-path.nix rename to tests/functional/lang/eval-okay-path.nix diff --git a/tests/lang/eval-okay-pathexists.exp b/tests/functional/lang/eval-okay-pathexists.exp similarity index 100% rename from tests/lang/eval-okay-pathexists.exp rename to tests/functional/lang/eval-okay-pathexists.exp diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix similarity index 100% rename from tests/lang/eval-okay-pathexists.nix rename to tests/functional/lang/eval-okay-pathexists.nix diff --git a/tests/lang/eval-okay-patterns.exp b/tests/functional/lang/eval-okay-patterns.exp similarity index 100% rename from tests/lang/eval-okay-patterns.exp rename to tests/functional/lang/eval-okay-patterns.exp diff --git a/tests/lang/eval-okay-patterns.nix b/tests/functional/lang/eval-okay-patterns.nix similarity index 100% rename from tests/lang/eval-okay-patterns.nix rename to tests/functional/lang/eval-okay-patterns.nix diff --git a/tests/lang/eval-okay-print.err.exp b/tests/functional/lang/eval-okay-print.err.exp similarity index 100% rename from tests/lang/eval-okay-print.err.exp rename to tests/functional/lang/eval-okay-print.err.exp diff --git a/tests/lang/eval-okay-print.exp b/tests/functional/lang/eval-okay-print.exp similarity index 100% rename from tests/lang/eval-okay-print.exp rename to tests/functional/lang/eval-okay-print.exp diff --git a/tests/lang/eval-okay-print.nix b/tests/functional/lang/eval-okay-print.nix similarity index 100% rename from tests/lang/eval-okay-print.nix rename to tests/functional/lang/eval-okay-print.nix diff --git a/tests/lang/eval-okay-readDir.exp b/tests/functional/lang/eval-okay-readDir.exp similarity index 100% rename from tests/lang/eval-okay-readDir.exp rename to tests/functional/lang/eval-okay-readDir.exp diff --git a/tests/lang/eval-okay-readDir.nix b/tests/functional/lang/eval-okay-readDir.nix similarity index 100% rename from tests/lang/eval-okay-readDir.nix rename to tests/functional/lang/eval-okay-readDir.nix diff --git a/tests/lang/eval-okay-readFileType.exp b/tests/functional/lang/eval-okay-readFileType.exp similarity index 100% rename from tests/lang/eval-okay-readFileType.exp rename to tests/functional/lang/eval-okay-readFileType.exp diff --git a/tests/lang/eval-okay-readFileType.nix b/tests/functional/lang/eval-okay-readFileType.nix similarity index 100% rename from tests/lang/eval-okay-readFileType.nix rename to tests/functional/lang/eval-okay-readFileType.nix diff --git a/tests/lang/eval-okay-readfile.exp b/tests/functional/lang/eval-okay-readfile.exp similarity index 100% rename from tests/lang/eval-okay-readfile.exp rename to tests/functional/lang/eval-okay-readfile.exp diff --git a/tests/lang/eval-okay-readfile.nix b/tests/functional/lang/eval-okay-readfile.nix similarity index 100% rename from tests/lang/eval-okay-readfile.nix rename to tests/functional/lang/eval-okay-readfile.nix diff --git a/tests/lang/eval-okay-redefine-builtin.exp b/tests/functional/lang/eval-okay-redefine-builtin.exp similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.exp rename to tests/functional/lang/eval-okay-redefine-builtin.exp diff --git a/tests/lang/eval-okay-redefine-builtin.nix b/tests/functional/lang/eval-okay-redefine-builtin.nix similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.nix rename to tests/functional/lang/eval-okay-redefine-builtin.nix diff --git a/tests/lang/eval-okay-regex-match.exp b/tests/functional/lang/eval-okay-regex-match.exp similarity index 100% rename from tests/lang/eval-okay-regex-match.exp rename to tests/functional/lang/eval-okay-regex-match.exp diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/functional/lang/eval-okay-regex-match.nix similarity index 100% rename from tests/lang/eval-okay-regex-match.nix rename to tests/functional/lang/eval-okay-regex-match.nix diff --git a/tests/lang/eval-okay-regex-split.exp b/tests/functional/lang/eval-okay-regex-split.exp similarity index 100% rename from tests/lang/eval-okay-regex-split.exp rename to tests/functional/lang/eval-okay-regex-split.exp diff --git a/tests/lang/eval-okay-regex-split.nix b/tests/functional/lang/eval-okay-regex-split.nix similarity index 100% rename from tests/lang/eval-okay-regex-split.nix rename to tests/functional/lang/eval-okay-regex-split.nix diff --git a/tests/lang/eval-okay-regression-20220122.exp b/tests/functional/lang/eval-okay-regression-20220122.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220122.exp rename to tests/functional/lang/eval-okay-regression-20220122.exp diff --git a/tests/lang/eval-okay-regression-20220122.nix b/tests/functional/lang/eval-okay-regression-20220122.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220122.nix rename to tests/functional/lang/eval-okay-regression-20220122.nix diff --git a/tests/lang/eval-okay-regression-20220125.exp b/tests/functional/lang/eval-okay-regression-20220125.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220125.exp rename to tests/functional/lang/eval-okay-regression-20220125.exp diff --git a/tests/lang/eval-okay-regression-20220125.nix b/tests/functional/lang/eval-okay-regression-20220125.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220125.nix rename to tests/functional/lang/eval-okay-regression-20220125.nix diff --git a/tests/lang/eval-okay-remove.exp b/tests/functional/lang/eval-okay-remove.exp similarity index 100% rename from tests/lang/eval-okay-remove.exp rename to tests/functional/lang/eval-okay-remove.exp diff --git a/tests/lang/eval-okay-remove.nix b/tests/functional/lang/eval-okay-remove.nix similarity index 100% rename from tests/lang/eval-okay-remove.nix rename to tests/functional/lang/eval-okay-remove.nix diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/functional/lang/eval-okay-replacestrings.exp similarity index 100% rename from tests/lang/eval-okay-replacestrings.exp rename to tests/functional/lang/eval-okay-replacestrings.exp diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/functional/lang/eval-okay-replacestrings.nix similarity index 100% rename from tests/lang/eval-okay-replacestrings.nix rename to tests/functional/lang/eval-okay-replacestrings.nix diff --git a/tests/lang/eval-okay-scope-1.exp b/tests/functional/lang/eval-okay-scope-1.exp similarity index 100% rename from tests/lang/eval-okay-scope-1.exp rename to tests/functional/lang/eval-okay-scope-1.exp diff --git a/tests/lang/eval-okay-scope-1.nix b/tests/functional/lang/eval-okay-scope-1.nix similarity index 100% rename from tests/lang/eval-okay-scope-1.nix rename to tests/functional/lang/eval-okay-scope-1.nix diff --git a/tests/lang/eval-okay-scope-2.exp b/tests/functional/lang/eval-okay-scope-2.exp similarity index 100% rename from tests/lang/eval-okay-scope-2.exp rename to tests/functional/lang/eval-okay-scope-2.exp diff --git a/tests/lang/eval-okay-scope-2.nix b/tests/functional/lang/eval-okay-scope-2.nix similarity index 100% rename from tests/lang/eval-okay-scope-2.nix rename to tests/functional/lang/eval-okay-scope-2.nix diff --git a/tests/lang/eval-okay-scope-3.exp b/tests/functional/lang/eval-okay-scope-3.exp similarity index 100% rename from tests/lang/eval-okay-scope-3.exp rename to tests/functional/lang/eval-okay-scope-3.exp diff --git a/tests/lang/eval-okay-scope-3.nix b/tests/functional/lang/eval-okay-scope-3.nix similarity index 100% rename from tests/lang/eval-okay-scope-3.nix rename to tests/functional/lang/eval-okay-scope-3.nix diff --git a/tests/lang/eval-okay-scope-4.exp b/tests/functional/lang/eval-okay-scope-4.exp similarity index 100% rename from tests/lang/eval-okay-scope-4.exp rename to tests/functional/lang/eval-okay-scope-4.exp diff --git a/tests/lang/eval-okay-scope-4.nix b/tests/functional/lang/eval-okay-scope-4.nix similarity index 100% rename from tests/lang/eval-okay-scope-4.nix rename to tests/functional/lang/eval-okay-scope-4.nix diff --git a/tests/lang/eval-okay-scope-6.exp b/tests/functional/lang/eval-okay-scope-6.exp similarity index 100% rename from tests/lang/eval-okay-scope-6.exp rename to tests/functional/lang/eval-okay-scope-6.exp diff --git a/tests/lang/eval-okay-scope-6.nix b/tests/functional/lang/eval-okay-scope-6.nix similarity index 100% rename from tests/lang/eval-okay-scope-6.nix rename to tests/functional/lang/eval-okay-scope-6.nix diff --git a/tests/lang/eval-okay-scope-7.exp b/tests/functional/lang/eval-okay-scope-7.exp similarity index 100% rename from tests/lang/eval-okay-scope-7.exp rename to tests/functional/lang/eval-okay-scope-7.exp diff --git a/tests/lang/eval-okay-scope-7.nix b/tests/functional/lang/eval-okay-scope-7.nix similarity index 100% rename from tests/lang/eval-okay-scope-7.nix rename to tests/functional/lang/eval-okay-scope-7.nix diff --git a/tests/lang/eval-okay-search-path.exp b/tests/functional/lang/eval-okay-search-path.exp similarity index 100% rename from tests/lang/eval-okay-search-path.exp rename to tests/functional/lang/eval-okay-search-path.exp diff --git a/tests/lang/eval-okay-search-path.flags b/tests/functional/lang/eval-okay-search-path.flags similarity index 100% rename from tests/lang/eval-okay-search-path.flags rename to tests/functional/lang/eval-okay-search-path.flags diff --git a/tests/lang/eval-okay-search-path.nix b/tests/functional/lang/eval-okay-search-path.nix similarity index 100% rename from tests/lang/eval-okay-search-path.nix rename to tests/functional/lang/eval-okay-search-path.nix diff --git a/tests/lang/eval-okay-seq.exp b/tests/functional/lang/eval-okay-seq.exp similarity index 100% rename from tests/lang/eval-okay-seq.exp rename to tests/functional/lang/eval-okay-seq.exp diff --git a/tests/lang/eval-okay-seq.nix b/tests/functional/lang/eval-okay-seq.nix similarity index 100% rename from tests/lang/eval-okay-seq.nix rename to tests/functional/lang/eval-okay-seq.nix diff --git a/tests/lang/eval-okay-sort.exp b/tests/functional/lang/eval-okay-sort.exp similarity index 100% rename from tests/lang/eval-okay-sort.exp rename to tests/functional/lang/eval-okay-sort.exp diff --git a/tests/lang/eval-okay-sort.nix b/tests/functional/lang/eval-okay-sort.nix similarity index 100% rename from tests/lang/eval-okay-sort.nix rename to tests/functional/lang/eval-okay-sort.nix diff --git a/tests/lang/eval-okay-splitversion.exp b/tests/functional/lang/eval-okay-splitversion.exp similarity index 100% rename from tests/lang/eval-okay-splitversion.exp rename to tests/functional/lang/eval-okay-splitversion.exp diff --git a/tests/lang/eval-okay-splitversion.nix b/tests/functional/lang/eval-okay-splitversion.nix similarity index 100% rename from tests/lang/eval-okay-splitversion.nix rename to tests/functional/lang/eval-okay-splitversion.nix diff --git a/tests/lang/eval-okay-string.exp b/tests/functional/lang/eval-okay-string.exp similarity index 100% rename from tests/lang/eval-okay-string.exp rename to tests/functional/lang/eval-okay-string.exp diff --git a/tests/lang/eval-okay-string.nix b/tests/functional/lang/eval-okay-string.nix similarity index 100% rename from tests/lang/eval-okay-string.nix rename to tests/functional/lang/eval-okay-string.nix diff --git a/tests/lang/eval-okay-strings-as-attrs-names.exp b/tests/functional/lang/eval-okay-strings-as-attrs-names.exp similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.exp rename to tests/functional/lang/eval-okay-strings-as-attrs-names.exp diff --git a/tests/lang/eval-okay-strings-as-attrs-names.nix b/tests/functional/lang/eval-okay-strings-as-attrs-names.nix similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.nix rename to tests/functional/lang/eval-okay-strings-as-attrs-names.nix diff --git a/tests/lang/eval-okay-substring.exp b/tests/functional/lang/eval-okay-substring.exp similarity index 100% rename from tests/lang/eval-okay-substring.exp rename to tests/functional/lang/eval-okay-substring.exp diff --git a/tests/lang/eval-okay-substring.nix b/tests/functional/lang/eval-okay-substring.nix similarity index 100% rename from tests/lang/eval-okay-substring.nix rename to tests/functional/lang/eval-okay-substring.nix diff --git a/tests/lang/eval-okay-tail-call-1.exp-disabled b/tests/functional/lang/eval-okay-tail-call-1.exp-disabled similarity index 100% rename from tests/lang/eval-okay-tail-call-1.exp-disabled rename to tests/functional/lang/eval-okay-tail-call-1.exp-disabled diff --git a/tests/lang/eval-okay-tail-call-1.nix b/tests/functional/lang/eval-okay-tail-call-1.nix similarity index 100% rename from tests/lang/eval-okay-tail-call-1.nix rename to tests/functional/lang/eval-okay-tail-call-1.nix diff --git a/tests/lang/eval-okay-tojson.exp b/tests/functional/lang/eval-okay-tojson.exp similarity index 100% rename from tests/lang/eval-okay-tojson.exp rename to tests/functional/lang/eval-okay-tojson.exp diff --git a/tests/lang/eval-okay-tojson.nix b/tests/functional/lang/eval-okay-tojson.nix similarity index 100% rename from tests/lang/eval-okay-tojson.nix rename to tests/functional/lang/eval-okay-tojson.nix diff --git a/tests/lang/eval-okay-toxml.exp b/tests/functional/lang/eval-okay-toxml.exp similarity index 100% rename from tests/lang/eval-okay-toxml.exp rename to tests/functional/lang/eval-okay-toxml.exp diff --git a/tests/lang/eval-okay-toxml.nix b/tests/functional/lang/eval-okay-toxml.nix similarity index 100% rename from tests/lang/eval-okay-toxml.nix rename to tests/functional/lang/eval-okay-toxml.nix diff --git a/tests/lang/eval-okay-toxml2.exp b/tests/functional/lang/eval-okay-toxml2.exp similarity index 100% rename from tests/lang/eval-okay-toxml2.exp rename to tests/functional/lang/eval-okay-toxml2.exp diff --git a/tests/lang/eval-okay-toxml2.nix b/tests/functional/lang/eval-okay-toxml2.nix similarity index 100% rename from tests/lang/eval-okay-toxml2.nix rename to tests/functional/lang/eval-okay-toxml2.nix diff --git a/tests/lang/eval-okay-tryeval.exp b/tests/functional/lang/eval-okay-tryeval.exp similarity index 100% rename from tests/lang/eval-okay-tryeval.exp rename to tests/functional/lang/eval-okay-tryeval.exp diff --git a/tests/lang/eval-okay-tryeval.nix b/tests/functional/lang/eval-okay-tryeval.nix similarity index 100% rename from tests/lang/eval-okay-tryeval.nix rename to tests/functional/lang/eval-okay-tryeval.nix diff --git a/tests/lang/eval-okay-types.exp b/tests/functional/lang/eval-okay-types.exp similarity index 100% rename from tests/lang/eval-okay-types.exp rename to tests/functional/lang/eval-okay-types.exp diff --git a/tests/lang/eval-okay-types.nix b/tests/functional/lang/eval-okay-types.nix similarity index 100% rename from tests/lang/eval-okay-types.nix rename to tests/functional/lang/eval-okay-types.nix diff --git a/tests/lang/eval-okay-versions.exp b/tests/functional/lang/eval-okay-versions.exp similarity index 100% rename from tests/lang/eval-okay-versions.exp rename to tests/functional/lang/eval-okay-versions.exp diff --git a/tests/lang/eval-okay-versions.nix b/tests/functional/lang/eval-okay-versions.nix similarity index 100% rename from tests/lang/eval-okay-versions.nix rename to tests/functional/lang/eval-okay-versions.nix diff --git a/tests/lang/eval-okay-with.exp b/tests/functional/lang/eval-okay-with.exp similarity index 100% rename from tests/lang/eval-okay-with.exp rename to tests/functional/lang/eval-okay-with.exp diff --git a/tests/lang/eval-okay-with.nix b/tests/functional/lang/eval-okay-with.nix similarity index 100% rename from tests/lang/eval-okay-with.nix rename to tests/functional/lang/eval-okay-with.nix diff --git a/tests/lang/eval-okay-xml.exp.xml b/tests/functional/lang/eval-okay-xml.exp.xml similarity index 100% rename from tests/lang/eval-okay-xml.exp.xml rename to tests/functional/lang/eval-okay-xml.exp.xml diff --git a/tests/lang/eval-okay-xml.nix b/tests/functional/lang/eval-okay-xml.nix similarity index 100% rename from tests/lang/eval-okay-xml.nix rename to tests/functional/lang/eval-okay-xml.nix diff --git a/tests/lang/eval-okay-zipAttrsWith.exp b/tests/functional/lang/eval-okay-zipAttrsWith.exp similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.exp rename to tests/functional/lang/eval-okay-zipAttrsWith.exp diff --git a/tests/lang/eval-okay-zipAttrsWith.nix b/tests/functional/lang/eval-okay-zipAttrsWith.nix similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.nix rename to tests/functional/lang/eval-okay-zipAttrsWith.nix diff --git a/tests/lang/framework.sh b/tests/functional/lang/framework.sh similarity index 100% rename from tests/lang/framework.sh rename to tests/functional/lang/framework.sh diff --git a/tests/lang/imported.nix b/tests/functional/lang/imported.nix similarity index 100% rename from tests/lang/imported.nix rename to tests/functional/lang/imported.nix diff --git a/tests/lang/imported2.nix b/tests/functional/lang/imported2.nix similarity index 100% rename from tests/lang/imported2.nix rename to tests/functional/lang/imported2.nix diff --git a/tests/lang/lib.nix b/tests/functional/lang/lib.nix similarity index 100% rename from tests/lang/lib.nix rename to tests/functional/lang/lib.nix diff --git a/tests/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-1.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-1.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-1.nix b/tests/functional/lang/parse-fail-dup-attrs-1.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-1.nix rename to tests/functional/lang/parse-fail-dup-attrs-1.nix diff --git a/tests/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-2.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-2.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-2.nix b/tests/functional/lang/parse-fail-dup-attrs-2.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-2.nix rename to tests/functional/lang/parse-fail-dup-attrs-2.nix diff --git a/tests/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-3.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-3.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-3.nix b/tests/functional/lang/parse-fail-dup-attrs-3.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-3.nix rename to tests/functional/lang/parse-fail-dup-attrs-3.nix diff --git a/tests/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-4.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-4.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-4.nix b/tests/functional/lang/parse-fail-dup-attrs-4.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-4.nix rename to tests/functional/lang/parse-fail-dup-attrs-4.nix diff --git a/tests/lang/parse-fail-dup-attrs-6.err.exp b/tests/functional/lang/parse-fail-dup-attrs-6.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-6.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-6.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-7.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-7.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-7.nix b/tests/functional/lang/parse-fail-dup-attrs-7.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-7.nix rename to tests/functional/lang/parse-fail-dup-attrs-7.nix diff --git a/tests/lang/parse-fail-dup-formals.err.exp b/tests/functional/lang/parse-fail-dup-formals.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-formals.err.exp rename to tests/functional/lang/parse-fail-dup-formals.err.exp diff --git a/tests/lang/parse-fail-dup-formals.nix b/tests/functional/lang/parse-fail-dup-formals.nix similarity index 100% rename from tests/lang/parse-fail-dup-formals.nix rename to tests/functional/lang/parse-fail-dup-formals.nix diff --git a/tests/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp similarity index 100% rename from tests/lang/parse-fail-eof-in-string.err.exp rename to tests/functional/lang/parse-fail-eof-in-string.err.exp diff --git a/tests/lang/parse-fail-eof-in-string.nix b/tests/functional/lang/parse-fail-eof-in-string.nix similarity index 100% rename from tests/lang/parse-fail-eof-in-string.nix rename to tests/functional/lang/parse-fail-eof-in-string.nix diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs1.err.exp rename to tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs1.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs1.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs1.nix diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs2.err.exp rename to tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs2.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs2.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs2.nix diff --git a/tests/lang/parse-fail-patterns-1.err.exp b/tests/functional/lang/parse-fail-patterns-1.err.exp similarity index 100% rename from tests/lang/parse-fail-patterns-1.err.exp rename to tests/functional/lang/parse-fail-patterns-1.err.exp diff --git a/tests/lang/parse-fail-patterns-1.nix b/tests/functional/lang/parse-fail-patterns-1.nix similarity index 100% rename from tests/lang/parse-fail-patterns-1.nix rename to tests/functional/lang/parse-fail-patterns-1.nix diff --git a/tests/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp similarity index 100% rename from tests/lang/parse-fail-regression-20060610.err.exp rename to tests/functional/lang/parse-fail-regression-20060610.err.exp diff --git a/tests/lang/parse-fail-regression-20060610.nix b/tests/functional/lang/parse-fail-regression-20060610.nix similarity index 100% rename from tests/lang/parse-fail-regression-20060610.nix rename to tests/functional/lang/parse-fail-regression-20060610.nix diff --git a/tests/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp similarity index 100% rename from tests/lang/parse-fail-undef-var-2.err.exp rename to tests/functional/lang/parse-fail-undef-var-2.err.exp diff --git a/tests/lang/parse-fail-undef-var-2.nix b/tests/functional/lang/parse-fail-undef-var-2.nix similarity index 100% rename from tests/lang/parse-fail-undef-var-2.nix rename to tests/functional/lang/parse-fail-undef-var-2.nix diff --git a/tests/lang/parse-fail-undef-var.err.exp b/tests/functional/lang/parse-fail-undef-var.err.exp similarity index 100% rename from tests/lang/parse-fail-undef-var.err.exp rename to tests/functional/lang/parse-fail-undef-var.err.exp diff --git a/tests/lang/parse-fail-undef-var.nix b/tests/functional/lang/parse-fail-undef-var.nix similarity index 100% rename from tests/lang/parse-fail-undef-var.nix rename to tests/functional/lang/parse-fail-undef-var.nix diff --git a/tests/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp similarity index 100% rename from tests/lang/parse-fail-utf8.err.exp rename to tests/functional/lang/parse-fail-utf8.err.exp diff --git a/tests/lang/parse-fail-utf8.nix b/tests/functional/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-utf8.nix rename to tests/functional/lang/parse-fail-utf8.nix diff --git a/tests/lang/parse-okay-1.exp b/tests/functional/lang/parse-okay-1.exp similarity index 100% rename from tests/lang/parse-okay-1.exp rename to tests/functional/lang/parse-okay-1.exp diff --git a/tests/lang/parse-okay-1.nix b/tests/functional/lang/parse-okay-1.nix similarity index 100% rename from tests/lang/parse-okay-1.nix rename to tests/functional/lang/parse-okay-1.nix diff --git a/tests/lang/parse-okay-crlf.exp b/tests/functional/lang/parse-okay-crlf.exp similarity index 100% rename from tests/lang/parse-okay-crlf.exp rename to tests/functional/lang/parse-okay-crlf.exp diff --git a/tests/lang/parse-okay-crlf.nix b/tests/functional/lang/parse-okay-crlf.nix similarity index 100% rename from tests/lang/parse-okay-crlf.nix rename to tests/functional/lang/parse-okay-crlf.nix diff --git a/tests/lang/parse-okay-dup-attrs-5.exp b/tests/functional/lang/parse-okay-dup-attrs-5.exp similarity index 100% rename from tests/lang/parse-okay-dup-attrs-5.exp rename to tests/functional/lang/parse-okay-dup-attrs-5.exp diff --git a/tests/lang/parse-okay-dup-attrs-5.nix b/tests/functional/lang/parse-okay-dup-attrs-5.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-5.nix rename to tests/functional/lang/parse-okay-dup-attrs-5.nix diff --git a/tests/lang/parse-okay-dup-attrs-6.exp b/tests/functional/lang/parse-okay-dup-attrs-6.exp similarity index 100% rename from tests/lang/parse-okay-dup-attrs-6.exp rename to tests/functional/lang/parse-okay-dup-attrs-6.exp diff --git a/tests/lang/parse-okay-dup-attrs-6.nix b/tests/functional/lang/parse-okay-dup-attrs-6.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-6.nix rename to tests/functional/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-1.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-1.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-2.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-2.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-3.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-3.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix diff --git a/tests/lang/parse-okay-regression-20041027.exp b/tests/functional/lang/parse-okay-regression-20041027.exp similarity index 100% rename from tests/lang/parse-okay-regression-20041027.exp rename to tests/functional/lang/parse-okay-regression-20041027.exp diff --git a/tests/lang/parse-okay-regression-20041027.nix b/tests/functional/lang/parse-okay-regression-20041027.nix similarity index 100% rename from tests/lang/parse-okay-regression-20041027.nix rename to tests/functional/lang/parse-okay-regression-20041027.nix diff --git a/tests/lang/parse-okay-regression-751.exp b/tests/functional/lang/parse-okay-regression-751.exp similarity index 100% rename from tests/lang/parse-okay-regression-751.exp rename to tests/functional/lang/parse-okay-regression-751.exp diff --git a/tests/lang/parse-okay-regression-751.nix b/tests/functional/lang/parse-okay-regression-751.nix similarity index 100% rename from tests/lang/parse-okay-regression-751.nix rename to tests/functional/lang/parse-okay-regression-751.nix diff --git a/tests/lang/parse-okay-subversion.exp b/tests/functional/lang/parse-okay-subversion.exp similarity index 100% rename from tests/lang/parse-okay-subversion.exp rename to tests/functional/lang/parse-okay-subversion.exp diff --git a/tests/lang/parse-okay-subversion.nix b/tests/functional/lang/parse-okay-subversion.nix similarity index 100% rename from tests/lang/parse-okay-subversion.nix rename to tests/functional/lang/parse-okay-subversion.nix diff --git a/tests/lang/parse-okay-url.exp b/tests/functional/lang/parse-okay-url.exp similarity index 100% rename from tests/lang/parse-okay-url.exp rename to tests/functional/lang/parse-okay-url.exp diff --git a/tests/lang/parse-okay-url.nix b/tests/functional/lang/parse-okay-url.nix similarity index 100% rename from tests/lang/parse-okay-url.nix rename to tests/functional/lang/parse-okay-url.nix diff --git a/tests/lang/readDir/bar b/tests/functional/lang/readDir/bar similarity index 100% rename from tests/lang/readDir/bar rename to tests/functional/lang/readDir/bar diff --git a/tests/lang/readDir/foo/git-hates-directories b/tests/functional/lang/readDir/foo/git-hates-directories similarity index 100% rename from tests/lang/readDir/foo/git-hates-directories rename to tests/functional/lang/readDir/foo/git-hates-directories diff --git a/tests/lang/readDir/ldir b/tests/functional/lang/readDir/ldir similarity index 100% rename from tests/lang/readDir/ldir rename to tests/functional/lang/readDir/ldir diff --git a/tests/lang/readDir/linked b/tests/functional/lang/readDir/linked similarity index 100% rename from tests/lang/readDir/linked rename to tests/functional/lang/readDir/linked diff --git a/tests/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh similarity index 100% rename from tests/legacy-ssh-store.sh rename to tests/functional/legacy-ssh-store.sh diff --git a/tests/linux-sandbox-cert-test.nix b/tests/functional/linux-sandbox-cert-test.nix similarity index 100% rename from tests/linux-sandbox-cert-test.nix rename to tests/functional/linux-sandbox-cert-test.nix diff --git a/tests/linux-sandbox.sh b/tests/functional/linux-sandbox.sh similarity index 100% rename from tests/linux-sandbox.sh rename to tests/functional/linux-sandbox.sh diff --git a/tests/local-store.sh b/tests/functional/local-store.sh similarity index 100% rename from tests/local-store.sh rename to tests/functional/local-store.sh diff --git a/tests/local.mk b/tests/functional/local.mk similarity index 90% rename from tests/local.mk rename to tests/functional/local.mk index ac418dc0d..4d29f60a1 100644 --- a/tests/local.mk +++ b/tests/functional/local.mk @@ -139,15 +139,16 @@ ifeq ($(HAVE_LOCAL_NIX_BUILD), 1) endif endif -tests/test-libstoreconsumer.sh.test: tests/test-libstoreconsumer/test-libstoreconsumer -tests/plugins.sh: tests/plugins/libplugintest.$(SO_EXT) +$(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ + $(d)/test-libstoreconsumer/test-libstoreconsumer +$(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ + $(d)/plugins/libplugintest.$(SO_EXT) install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) -clean-files += \ +test-clean-files := \ $(d)/common/vars-and-functions.sh \ $(d)/config.nix -test-deps += \ - tests/common/vars-and-functions.sh \ - tests/config.nix +clean-files += $(test-clean-files) +test-deps += $(test-clean-files) diff --git a/tests/logging.sh b/tests/functional/logging.sh similarity index 100% rename from tests/logging.sh rename to tests/functional/logging.sh diff --git a/tests/misc.sh b/tests/functional/misc.sh similarity index 100% rename from tests/misc.sh rename to tests/functional/misc.sh diff --git a/tests/multiple-outputs.nix b/tests/functional/multiple-outputs.nix similarity index 100% rename from tests/multiple-outputs.nix rename to tests/functional/multiple-outputs.nix diff --git a/tests/multiple-outputs.sh b/tests/functional/multiple-outputs.sh similarity index 100% rename from tests/multiple-outputs.sh rename to tests/functional/multiple-outputs.sh diff --git a/tests/nar-access.nix b/tests/functional/nar-access.nix similarity index 100% rename from tests/nar-access.nix rename to tests/functional/nar-access.nix diff --git a/tests/nar-access.sh b/tests/functional/nar-access.sh similarity index 100% rename from tests/nar-access.sh rename to tests/functional/nar-access.sh diff --git a/tests/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh similarity index 75% rename from tests/nested-sandboxing.sh rename to tests/functional/nested-sandboxing.sh index d9fa788aa..61fe043c6 100644 --- a/tests/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -1,5 +1,5 @@ source common.sh -# This test is run by `tests/nested-sandboxing/runner.nix` in an extra layer of sandboxing. +# This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. [[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" requireSandboxSupport diff --git a/tests/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh similarity index 100% rename from tests/nested-sandboxing/command.sh rename to tests/functional/nested-sandboxing/command.sh diff --git a/tests/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix similarity index 100% rename from tests/nested-sandboxing/runner.nix rename to tests/functional/nested-sandboxing/runner.nix diff --git a/tests/nix-build-examples.nix b/tests/functional/nix-build-examples.nix similarity index 100% rename from tests/nix-build-examples.nix rename to tests/functional/nix-build-examples.nix diff --git a/tests/nix-build.sh b/tests/functional/nix-build.sh similarity index 100% rename from tests/nix-build.sh rename to tests/functional/nix-build.sh diff --git a/tests/nix-channel.sh b/tests/functional/nix-channel.sh similarity index 100% rename from tests/nix-channel.sh rename to tests/functional/nix-channel.sh diff --git a/tests/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh similarity index 100% rename from tests/nix-collect-garbage-d.sh rename to tests/functional/nix-collect-garbage-d.sh diff --git a/tests/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh similarity index 100% rename from tests/nix-copy-ssh-ng.sh rename to tests/functional/nix-copy-ssh-ng.sh diff --git a/tests/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh similarity index 100% rename from tests/nix-copy-ssh.sh rename to tests/functional/nix-copy-ssh.sh diff --git a/tests/nix-daemon-untrusting.sh b/tests/functional/nix-daemon-untrusting.sh similarity index 100% rename from tests/nix-daemon-untrusting.sh rename to tests/functional/nix-daemon-untrusting.sh diff --git a/tests/nix-profile.sh b/tests/functional/nix-profile.sh similarity index 100% rename from tests/nix-profile.sh rename to tests/functional/nix-profile.sh diff --git a/tests/nix-shell.sh b/tests/functional/nix-shell.sh similarity index 100% rename from tests/nix-shell.sh rename to tests/functional/nix-shell.sh diff --git a/tests/nix_path.sh b/tests/functional/nix_path.sh similarity index 100% rename from tests/nix_path.sh rename to tests/functional/nix_path.sh diff --git a/tests/optimise-store.sh b/tests/functional/optimise-store.sh similarity index 100% rename from tests/optimise-store.sh rename to tests/functional/optimise-store.sh diff --git a/tests/output-normalization.sh b/tests/functional/output-normalization.sh similarity index 100% rename from tests/output-normalization.sh rename to tests/functional/output-normalization.sh diff --git a/tests/parallel.builder.sh b/tests/functional/parallel.builder.sh similarity index 100% rename from tests/parallel.builder.sh rename to tests/functional/parallel.builder.sh diff --git a/tests/parallel.nix b/tests/functional/parallel.nix similarity index 100% rename from tests/parallel.nix rename to tests/functional/parallel.nix diff --git a/tests/parallel.sh b/tests/functional/parallel.sh similarity index 100% rename from tests/parallel.sh rename to tests/functional/parallel.sh diff --git a/tests/pass-as-file.sh b/tests/functional/pass-as-file.sh similarity index 100% rename from tests/pass-as-file.sh rename to tests/functional/pass-as-file.sh diff --git a/tests/path-from-hash-part.sh b/tests/functional/path-from-hash-part.sh similarity index 100% rename from tests/path-from-hash-part.sh rename to tests/functional/path-from-hash-part.sh diff --git a/tests/path.nix b/tests/functional/path.nix similarity index 100% rename from tests/path.nix rename to tests/functional/path.nix diff --git a/tests/placeholders.sh b/tests/functional/placeholders.sh similarity index 100% rename from tests/placeholders.sh rename to tests/functional/placeholders.sh diff --git a/tests/plugins.sh b/tests/functional/plugins.sh similarity index 100% rename from tests/plugins.sh rename to tests/functional/plugins.sh diff --git a/tests/plugins/local.mk b/tests/functional/plugins/local.mk similarity index 100% rename from tests/plugins/local.mk rename to tests/functional/plugins/local.mk diff --git a/tests/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc similarity index 100% rename from tests/plugins/plugintest.cc rename to tests/functional/plugins/plugintest.cc diff --git a/tests/post-hook.sh b/tests/functional/post-hook.sh similarity index 100% rename from tests/post-hook.sh rename to tests/functional/post-hook.sh diff --git a/tests/pure-eval.nix b/tests/functional/pure-eval.nix similarity index 100% rename from tests/pure-eval.nix rename to tests/functional/pure-eval.nix diff --git a/tests/pure-eval.sh b/tests/functional/pure-eval.sh similarity index 100% rename from tests/pure-eval.sh rename to tests/functional/pure-eval.sh diff --git a/tests/push-to-store-old.sh b/tests/functional/push-to-store-old.sh similarity index 100% rename from tests/push-to-store-old.sh rename to tests/functional/push-to-store-old.sh diff --git a/tests/push-to-store.sh b/tests/functional/push-to-store.sh similarity index 100% rename from tests/push-to-store.sh rename to tests/functional/push-to-store.sh diff --git a/tests/read-only-store.sh b/tests/functional/read-only-store.sh similarity index 100% rename from tests/read-only-store.sh rename to tests/functional/read-only-store.sh diff --git a/tests/readfile-context.nix b/tests/functional/readfile-context.nix similarity index 100% rename from tests/readfile-context.nix rename to tests/functional/readfile-context.nix diff --git a/tests/readfile-context.sh b/tests/functional/readfile-context.sh similarity index 100% rename from tests/readfile-context.sh rename to tests/functional/readfile-context.sh diff --git a/tests/recursive.nix b/tests/functional/recursive.nix similarity index 100% rename from tests/recursive.nix rename to tests/functional/recursive.nix diff --git a/tests/recursive.sh b/tests/functional/recursive.sh similarity index 100% rename from tests/recursive.sh rename to tests/functional/recursive.sh diff --git a/tests/referrers.sh b/tests/functional/referrers.sh similarity index 100% rename from tests/referrers.sh rename to tests/functional/referrers.sh diff --git a/tests/remote-store.sh b/tests/functional/remote-store.sh similarity index 100% rename from tests/remote-store.sh rename to tests/functional/remote-store.sh diff --git a/tests/repair.sh b/tests/functional/repair.sh similarity index 100% rename from tests/repair.sh rename to tests/functional/repair.sh diff --git a/tests/repl.sh b/tests/functional/repl.sh similarity index 100% rename from tests/repl.sh rename to tests/functional/repl.sh diff --git a/tests/restricted.nix b/tests/functional/restricted.nix similarity index 100% rename from tests/restricted.nix rename to tests/functional/restricted.nix diff --git a/tests/restricted.sh b/tests/functional/restricted.sh similarity index 95% rename from tests/restricted.sh rename to tests/functional/restricted.sh index 17f310a4b..197ae7a10 100644 --- a/tests/restricted.sh +++ b/tests/functional/restricted.sh @@ -9,10 +9,10 @@ nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') -nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=.. +nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src +(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') +nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. diff --git a/tests/search.nix b/tests/functional/search.nix similarity index 100% rename from tests/search.nix rename to tests/functional/search.nix diff --git a/tests/search.sh b/tests/functional/search.sh similarity index 100% rename from tests/search.sh rename to tests/functional/search.sh diff --git a/tests/secure-drv-outputs.nix b/tests/functional/secure-drv-outputs.nix similarity index 100% rename from tests/secure-drv-outputs.nix rename to tests/functional/secure-drv-outputs.nix diff --git a/tests/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh similarity index 100% rename from tests/secure-drv-outputs.sh rename to tests/functional/secure-drv-outputs.sh diff --git a/tests/selfref-gc.sh b/tests/functional/selfref-gc.sh similarity index 100% rename from tests/selfref-gc.sh rename to tests/functional/selfref-gc.sh diff --git a/tests/shell-hello.nix b/tests/functional/shell-hello.nix similarity index 100% rename from tests/shell-hello.nix rename to tests/functional/shell-hello.nix diff --git a/tests/shell.nix b/tests/functional/shell.nix similarity index 100% rename from tests/shell.nix rename to tests/functional/shell.nix diff --git a/tests/shell.sh b/tests/functional/shell.sh similarity index 100% rename from tests/shell.sh rename to tests/functional/shell.sh diff --git a/tests/shell.shebang.rb b/tests/functional/shell.shebang.rb similarity index 100% rename from tests/shell.shebang.rb rename to tests/functional/shell.shebang.rb diff --git a/tests/shell.shebang.sh b/tests/functional/shell.shebang.sh similarity index 100% rename from tests/shell.shebang.sh rename to tests/functional/shell.shebang.sh diff --git a/tests/signing.sh b/tests/functional/signing.sh similarity index 100% rename from tests/signing.sh rename to tests/functional/signing.sh diff --git a/tests/simple-failing.nix b/tests/functional/simple-failing.nix similarity index 100% rename from tests/simple-failing.nix rename to tests/functional/simple-failing.nix diff --git a/tests/simple.builder.sh b/tests/functional/simple.builder.sh similarity index 100% rename from tests/simple.builder.sh rename to tests/functional/simple.builder.sh diff --git a/tests/simple.nix b/tests/functional/simple.nix similarity index 100% rename from tests/simple.nix rename to tests/functional/simple.nix diff --git a/tests/simple.sh b/tests/functional/simple.sh similarity index 100% rename from tests/simple.sh rename to tests/functional/simple.sh diff --git a/tests/ssh-relay.sh b/tests/functional/ssh-relay.sh similarity index 100% rename from tests/ssh-relay.sh rename to tests/functional/ssh-relay.sh diff --git a/tests/store-ping.sh b/tests/functional/store-ping.sh similarity index 100% rename from tests/store-ping.sh rename to tests/functional/store-ping.sh diff --git a/tests/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix similarity index 100% rename from tests/structured-attrs-shell.nix rename to tests/functional/structured-attrs-shell.nix diff --git a/tests/structured-attrs.nix b/tests/functional/structured-attrs.nix similarity index 100% rename from tests/structured-attrs.nix rename to tests/functional/structured-attrs.nix diff --git a/tests/structured-attrs.sh b/tests/functional/structured-attrs.sh similarity index 100% rename from tests/structured-attrs.sh rename to tests/functional/structured-attrs.sh diff --git a/tests/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh similarity index 100% rename from tests/substitute-with-invalid-ca.sh rename to tests/functional/substitute-with-invalid-ca.sh diff --git a/tests/suggestions.sh b/tests/functional/suggestions.sh similarity index 100% rename from tests/suggestions.sh rename to tests/functional/suggestions.sh diff --git a/tests/supplementary-groups.sh b/tests/functional/supplementary-groups.sh similarity index 100% rename from tests/supplementary-groups.sh rename to tests/functional/supplementary-groups.sh diff --git a/tests/tarball.sh b/tests/functional/tarball.sh similarity index 100% rename from tests/tarball.sh rename to tests/functional/tarball.sh diff --git a/tests/test-infra.sh b/tests/functional/test-infra.sh similarity index 100% rename from tests/test-infra.sh rename to tests/functional/test-infra.sh diff --git a/tests/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh similarity index 100% rename from tests/test-libstoreconsumer.sh rename to tests/functional/test-libstoreconsumer.sh diff --git a/tests/test-libstoreconsumer/README.md b/tests/functional/test-libstoreconsumer/README.md similarity index 100% rename from tests/test-libstoreconsumer/README.md rename to tests/functional/test-libstoreconsumer/README.md diff --git a/tests/test-libstoreconsumer/local.mk b/tests/functional/test-libstoreconsumer/local.mk similarity index 100% rename from tests/test-libstoreconsumer/local.mk rename to tests/functional/test-libstoreconsumer/local.mk diff --git a/tests/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc similarity index 100% rename from tests/test-libstoreconsumer/main.cc rename to tests/functional/test-libstoreconsumer/main.cc diff --git a/tests/timeout.nix b/tests/functional/timeout.nix similarity index 100% rename from tests/timeout.nix rename to tests/functional/timeout.nix diff --git a/tests/timeout.sh b/tests/functional/timeout.sh similarity index 100% rename from tests/timeout.sh rename to tests/functional/timeout.sh diff --git a/tests/toString-path.sh b/tests/functional/toString-path.sh similarity index 100% rename from tests/toString-path.sh rename to tests/functional/toString-path.sh diff --git a/tests/undefined-variable.nix b/tests/functional/undefined-variable.nix similarity index 100% rename from tests/undefined-variable.nix rename to tests/functional/undefined-variable.nix diff --git a/tests/user-envs-migration.sh b/tests/functional/user-envs-migration.sh similarity index 100% rename from tests/user-envs-migration.sh rename to tests/functional/user-envs-migration.sh diff --git a/tests/user-envs.builder.sh b/tests/functional/user-envs.builder.sh similarity index 100% rename from tests/user-envs.builder.sh rename to tests/functional/user-envs.builder.sh diff --git a/tests/user-envs.nix b/tests/functional/user-envs.nix similarity index 100% rename from tests/user-envs.nix rename to tests/functional/user-envs.nix diff --git a/tests/user-envs.sh b/tests/functional/user-envs.sh similarity index 100% rename from tests/user-envs.sh rename to tests/functional/user-envs.sh diff --git a/tests/why-depends.sh b/tests/functional/why-depends.sh similarity index 100% rename from tests/why-depends.sh rename to tests/functional/why-depends.sh diff --git a/tests/zstd.sh b/tests/functional/zstd.sh similarity index 100% rename from tests/zstd.sh rename to tests/functional/zstd.sh From 644ebaab5f96089bc3a8fd3873e8e7f2131a9074 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Oct 2023 10:53:01 -0400 Subject: [PATCH 114/402] Define NixOS tests in `tests/nixos/default.nix` rather than `flake.nix` I think the our `flake.nix` is currently too large and too scary looking. I think this matters --- if Nix cannot dog-food itself in a way that is elegant, why should other people have confidence that their own code can be elegant and easy to maintain? We could do this at many points in time, but I think around now, when we are thinking about stabilizing parts of Flakes, is an especially good time. This is a first step to make the `flake.nix` smaller, and make individual components responsible for their own packaging. I hope we can do this many more follow-ups like it, until the top-level `flake.nix` is very small and just coordinates between other things. --- flake.nix | 74 ++++++++++++----------------------------- tests/nixos/default.nix | 41 +++++++++++++++++++++++ 2 files changed, 62 insertions(+), 53 deletions(-) create mode 100644 tests/nixos/default.nix diff --git a/flake.nix b/flake.nix index c331b2651..bec62fb95 100644 --- a/flake.nix +++ b/flake.nix @@ -509,18 +509,6 @@ }; }; - nixos-lib = import (nixpkgs + "/nixos/lib") { }; - - # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests - runNixOSTestFor = system: test: nixos-lib.runTest { - imports = [ test ]; - hostPkgs = nixpkgsFor.${system}.native; - defaults = { - nixpkgs.pkgs = nixpkgsFor.${system}.native; - }; - _module.args.nixpkgs = nixpkgs; - }; - in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. @@ -627,49 +615,29 @@ }; # System tests. - tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix; + tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + with nixpkgsFor.x86_64-linux.native; + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] + mkdir $out + ''; - tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; - - tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix; - - tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; - - tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; - - tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; - - tests.tarballFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/tarball-flakes.nix; - - tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; - - tests.setuid = lib.genAttrs - ["i686-linux" "x86_64-linux"] - (system: runNixOSTestFor system ./tests/nixos/setuid.nix); - - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - tests.evalNixpkgs = - with nixpkgsFor.x86_64-linux.native; - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - tests.nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/release.nix") + { pkgs = nixpkgsFor.${system}.native; + nixVersions = [ self.packages.${system}.nix ]; + } + ); + }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { pkgs = nixpkgsFor.x86_64-linux.native; diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix new file mode 100644 index 000000000..b391d7ef2 --- /dev/null +++ b/tests/nixos/default.nix @@ -0,0 +1,41 @@ +{ lib, nixpkgs, nixpkgsFor }: + +let + + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}.native; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}.native; + }; + _module.args.nixpkgs = nixpkgs; + }; + +in + +{ + authorization = runNixOSTestFor "x86_64-linux" ./authorization.nix; + + remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; + + nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; + + nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; + + nssPreload = runNixOSTestFor "x86_64-linux" ./nss-preload.nix; + + githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; + + sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./sourcehut-flakes.nix; + + tarballFlakes = runNixOSTestFor "x86_64-linux" ./tarball-flakes.nix; + + containers = runNixOSTestFor "x86_64-linux" ./containers/containers.nix; + + setuid = lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: runNixOSTestFor system ./setuid.nix); +} From 517c547dec086f6332292332571d05dd1cbb6f4b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 6 Oct 2023 23:34:08 +0200 Subject: [PATCH 115/402] remove duplicate redirects entry --- doc/manual/redirects.js | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index b2fad19bb..9d083a43d 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -336,7 +336,6 @@ const redirects = { "simple-values": "#primitives", "lists": "#list", "strings": "#string", - "lists": "#list", "attribute-sets": "#attribute-set", }, "installation/installing-binary.html": { From a67cee965a72f60da73713f47ee665a9ffe1873d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 02:27:06 +0200 Subject: [PATCH 116/402] expand on interpolated expressions --- .../src/language/string-interpolation.md | 120 ++++++++++++++++-- doc/manual/src/language/values.md | 12 +- src/libexpr/primops.cc | 4 +- 3 files changed, 116 insertions(+), 20 deletions(-) diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index ddc6b8230..85e02df37 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -1,19 +1,12 @@ # String interpolation -String interpolation is a language feature where a [string], [path], or [attribute name] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). +String interpolation is a language feature where a [string], [path], or [attribute name][attribute set] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). -Such a string is an *interpolated string*, and an expression inside is an *interpolated expression*. - -Interpolated expressions must evaluate to one of the following: - -- a [string] -- a [path] -- a [derivation] +Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). [string]: ./values.md#type-string [path]: ./values.md#type-path -[attribute name]: ./values.md#attribute-set -[derivation]: ../glossary.md#gloss-derivation +[attribute set]: ./values.md#attribute-set ## Examples @@ -80,3 +73,110 @@ let name = "foo"; in ``` { foo = "bar"; } + +# Interpolated expression + +An interpolated expression must evaluate to one of the following: + +- a [string] +- a [path] +- an [attribute set] that has a `__toString` attribute or an `outPath` attribute + + - `__toString` must be a function that takes the attribute set itself and returns a string + - `outPath` must be a string + + This includes [derivations](./derivations.md) or [flake inputs](@docroot@/command-ref/new-cli/nix3-flake.md#flake-inputs) (experimental). + +A string interpolates to itself. + +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). + +[store path]: ../glossary.md#gloss-store-path + +> **Example** +> +> ```console +> $ mkdir foo +> ``` +> +> Reference the empty directory in an interpolated expression: +> +> ```nix +> "${./foo}" +> ``` +> +> "/nix/store/2hhl2nz5v0khbn06ys82nrk99aa1xxdw-foo" + +A derivation interpolates to the [store path] of its first [output](./derivations.md#attr-outputs). + +> **Example** +> +> ```nix +> let +> pkgs = import {}; +> in +> "${pkgs.hello}" +> ``` +> +> "/nix/store/4xpfqf29z4m8vbhrqcz064wfmb46w5r7-hello-2.12.1" + +An attribute set interpolates to the return value of the function in the `__toString` applied to the attribute set itself. + +> **Example** +> +> ```nix +> let +> a = { +> value = 1; +> __toString = self: toString (self.value + 1); +> }; +> in +> "${a}" +> ``` +> +> "2" + +An attribute set also interpolates to the value of its `outPath` attribute. + +> **Example** +> +> ```nix +> let +> a = { outPath = "foo"; }; +> in +> "${a}" +> ``` +> +> "foo" + +If both `__toString` and `outPath` are present in an attribute set, `__toString` takes precedence. + +> **Example** +> +> ```nix +> let +> a = { __toString = _: "yes"; outPath = throw "no"; }; +> in +> "${a}" +> ``` +> +> "yes" + +If neither is present, an error is thrown. + +> **Example** +> +> ```nix +> let +> a = {}; +> in +> "${a}" +> ``` +> +> error: cannot coerce a set to a string +> +> at «string»:4:2: +> +> 3| in +> 4| "${a}" +> | ^ diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 2ae3e143a..fe0e2e4d5 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -112,18 +112,16 @@ environment variable `NIX_PATH` will be searched for the given file or directory name. - When an [interpolated string][string interpolation] evaluates to a path, the path is first copied into the Nix store and the resulting string is the [store path] of the newly created [store object]. - - [store path]: ../glossary.md#gloss-store-path - [store object]: ../glossary.md#gloss-store-object - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new store path, since Nix might not re-read the file contents. + Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - Paths themselves, except those in angle brackets (`< >`), support [string interpolation]. + [store path]: ../glossary.md#gloss-store-path + + Paths, except those in angle brackets (`< >`), support [string interpolation] and can be used in [interpolated expressions]. + [interpolated expressions]: ./string-interpolation.md#interpolated-expressions At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5de1b2828..1d04aeb2e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -260,9 +260,7 @@ static RegisterPrimOp primop_import({ .doc = R"( Load, parse and return the Nix expression in the file *path*. - The value *path* can be a path, a string, or an attribute set with an - `__toString` attribute or a `outPath` attribute (as derivations or flake - inputs typically have). + The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). If *path* is a directory, the file `default.nix` in that directory is loaded. From a7ba8c3f4a69118c4c936fd8fd1415c0446803ca Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 02:46:59 +0200 Subject: [PATCH 117/402] complete example on attribute name interpolation --- .../src/language/string-interpolation.md | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 85e02df37..2e650e348 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -63,16 +63,32 @@ you can instead write ### Attribute name -Attribute names can be created dynamically with string interpolation: + -```nix -let name = "foo"; in -{ - ${name} = "bar"; -} -``` +Attribute names can be interpolated strings. - { foo = "bar"; } +> **Example** +> +> ```nix +> let name = "foo"; in +> { ${name} = 123; } +> ``` +> +> { foo = 123; } + +Attributes can be selected with interpolated strings. + +> **Example** +> +> ```nix +> let name = "foo"; in +> { foo = 123; }.${name} +> ``` +> +> 123 # Interpolated expression From a86a3e5e59dd932d80431e9bbbb9ec3f883b2899 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 03:12:21 +0200 Subject: [PATCH 118/402] document that pure-eval also disables `builtins.nixPath` --- src/libexpr/eval-settings.hh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 19ba679be..6a5c448b4 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -40,7 +40,10 @@ struct EvalSettings : Config Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: - Restrict file system and network access to files specified by cryptographic hash - - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + - Disable impure constants: + - [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) )" }; From 2fe1ccf7973891cfbd921279a5f94879b2c7dce3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 03:20:20 +0200 Subject: [PATCH 119/402] describe the effect of `restrict-eval` in a more focused manner --- src/libexpr/eval-settings.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 19ba679be..2f534f496 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -29,10 +29,12 @@ struct EvalSettings : Config this, false, "restrict-eval", R"( If set to `true`, the Nix evaluator will not allow access to any - files outside of the Nix search path (as set via the `NIX_PATH` - environment variable or the `-I` option), or to URIs outside of - [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). - The default is `false`. + files outside of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath), + or to URIs outside of + [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). + + Also the default value for [`nix-path`](#conf-nix-path) is ignored, such that only explicitly set search path entries are taken into account. )"}; Setting pureEval{this, false, "pure-eval", From 630580162686dfd9a74d23acdee7b24ebd6b7c4c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 02:53:41 +0200 Subject: [PATCH 120/402] reword and reformat description of `builtins.import` --- src/libexpr/primops.cc | 101 ++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d04aeb2e..1ff2d5f0b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -258,62 +258,71 @@ static RegisterPrimOp primop_import({ .args = {"path"}, // TODO turn "normal path values" into link below .doc = R"( - Load, parse and return the Nix expression in the file *path*. - - The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). - - If *path* is a directory, the file `default.nix` in that directory - is loaded. - - Evaluation aborts if the file doesn’t exist or contains - an incorrect Nix expression. `import` implements Nix’s module - system: you can put any Nix expression (such as a set or a - function) in a separate file, and use it from Nix expressions in - other files. + Load, parse, and return the Nix expression in the file *path*. > **Note** > > Unlike some languages, `import` is a regular function in Nix. - > Paths using the angle bracket syntax (e.g., `import` *\*) - > are normal [path values](@docroot@/language/values.md#type-path). - A Nix expression loaded by `import` must not contain any *free - variables* (identifiers that are not defined in the Nix expression - itself and are not built-in). Therefore, it cannot refer to - variables that are in scope at the call site. For instance, if you - have a calling expression + The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). - ```nix - rec { - x = 123; - y = import ./foo.nix; - } - ``` + If *path* is a directory, the file `default.nix` in that directory is used if it exists. - then the following `foo.nix` will give an error: + > **Example** + > + > ```console + > $ echo 123 > default.nix + > ``` + > + > Import `default.nix` from the current directory. + > + > ```nix + > import ./. + > ``` + > + > 123 - ```nix - x + 456 - ``` + Evaluation aborts if the file doesn’t exist or contains an invalid Nix expression. - since `x` is not in scope in `foo.nix`. If you want `x` to be - available in `foo.nix`, you should pass it as a function argument: + A Nix expression loaded by `import` must not contain any *free variables*, that is, identifiers that are not defined in the Nix expression itself and are not built-in. + Therefore, it cannot refer to variables that are in scope at the call site. - ```nix - rec { - x = 123; - y = import ./foo.nix x; - } - ``` - - and - - ```nix - x: x + 456 - ``` - - (The function argument doesn’t have to be called `x` in `foo.nix`; - any name would work.) + > **Example** + > + > If you have a calling expression + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix; + > } + > ``` + > + > then the following `foo.nix` will give an error: + > + > ```nix + > # foo.nix + > x + 456 + > ``` + > + > since `x` is not in scope in `foo.nix`. + > If you want `x` to be available in `foo.nix`, pass it as a function argument: + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix x; + > } + > ``` + > + > and + > + > ```nix + > # foo.nix + > x: x + 456 + > ``` + > + > The function argument doesn’t have to be called `x` in `foo.nix`; any name would work. )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { From f00a5eb11b53a800850468f5e9e4658d05b419f9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 03:10:20 +0200 Subject: [PATCH 121/402] introduce lookup paths as a distinct language construct so far they did not really have a name, and were at best referred to as "angle bracket syntax". --- doc/manual/src/SUMMARY.md.in | 1 + .../src/language/constructs/lookup-path.md | 27 +++++++++++++++++++ doc/manual/src/language/values.md | 9 +++---- src/libexpr/primops.cc | 6 ++--- 4 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 doc/manual/src/language/constructs/lookup-path.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 2e631058f..e49e77cf5 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -30,6 +30,7 @@ - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) - [String interpolation](language/string-interpolation.md) + - [Lookup path](language/constructs/lookup-path.md) - [Operators](language/operators.md) - [Derivations](language/derivations.md) - [Advanced Attributes](language/advanced-attributes.md) diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md new file mode 100644 index 000000000..8ec835309 --- /dev/null +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -0,0 +1,27 @@ +# Lookup path + +> **Syntax** +> +> *lookup-path* = `<` *identifier* [ `/` *identifier* `]`... `>` + +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. + +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). + +See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs/nixos diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 2ae3e143a..4d8950e85 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -107,11 +107,6 @@ e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. - Paths can also be specified between angle brackets, e.g. - ``. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. - When an [interpolated string][string interpolation] evaluates to a path, the path is first copied into the Nix store and the resulting string is the [store path] of the newly created [store object]. [store path]: ../glossary.md#gloss-store-path @@ -123,13 +118,15 @@ For example, assume you used a file path in an interpolated string during a `nix repl` session. Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new store path, since Nix might not re-read the file contents. - Paths themselves, except those in angle brackets (`< >`), support [string interpolation]. + Paths support [string interpolation]. At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division operation. `./a.${foo}/b.${bar}` is a path. + [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. + - Boolean *Booleans* with values `true` and `false`. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5de1b2828..a8cb4ccfe 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1723,7 +1723,7 @@ static RegisterPrimOp primop_findFile(PrimOp { If the suffix is found inside that directory, then the entry is a match; the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - The syntax + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): ```nix @@ -4389,9 +4389,9 @@ void EvalState::createBaseEnv() addConstant("__nixPath", v, { .type = nList, .doc = R"( - The search path used to resolve angle bracket path lookups. + List of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - Angle bracket expressions can be + Lookup path expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): From 896a90520280f2645c32b9a0136ab281843942c3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 Oct 2023 10:07:46 +0200 Subject: [PATCH 122/402] AE -> BE; fix redirects --- doc/manual/redirects.js | 1 + doc/manual/src/contributing/testing.md | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 9d083a43d..d1b10109d 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -354,6 +354,7 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", + "characterization-testing": "#characterisation-testing-unit", } }; diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 1a6388e40..3d75ebe7b 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -62,11 +62,11 @@ The path to the `unit-test-data` directory is passed to the unit test executable You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. -### Characterization testing +### Characterisation testing { #characaterisation-testing-unit } -See [below](#characterization-testing-1) for a broader discussion of characterization testing. +See [functional characterisation testing](#characterisation-testing-functional) for a broader discussion of characterisation testing. -Like with the functional characterization, `_NIX_TEST_ACCEPT=1` is also used. +Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN @@ -77,8 +77,8 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN [ SKIPPED ] WorkerProtoTest.storePath_write ... ``` -will regenerate the "golden master" expected result for the `libnixstore` characterization tests. -The characterization tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. +will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. +The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. ## Functional tests @@ -195,9 +195,9 @@ To remove any traces of that: git clean -x --force tests ``` -### Characterization testing +### Characterisation testing { #characterisation-testing-functional } -Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +Occasionally, Nix utilizes a technique called [Characterisation Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. @@ -208,7 +208,7 @@ For example: ```bash _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test ``` -This convention is shared with the [characterization unit tests](#characterization-testing-1) too. +This convention is shared with the [characterisation unit tests](#characterisation-testing-unit) too. An interesting situation to document is the case when these tests are "overfitted". The language tests are, again, an example of this. @@ -221,7 +221,7 @@ Diagnostic outputs are indeed not a stable interface, but they still are importa By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. Regressions are caught, and improvements always show up in code review. -To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. ## Integration tests @@ -235,7 +235,7 @@ You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix- After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): +Creating a Cachix cache for your installer tests and adding its authorisation token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - `x86_64-linux` From 0246de1896df732739b0874c83b7d05becc83cf3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 Oct 2023 10:14:25 +0200 Subject: [PATCH 123/402] remove unnecessary indentation from markdown list this makes it a bit easier to work with, as some tooling doesn't work well with too much indentation. --- doc/manual/src/command-ref/env-common.md | 175 ++++++++++++----------- 1 file changed, 95 insertions(+), 80 deletions(-) diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index b4a9bb2a9..34e0dbfbd 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -2,109 +2,124 @@ Most Nix commands interpret the following environment variables: - - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL)\ - Indicator that tells if the current environment was set up by - `nix-shell`. It can have the values `pure` or `impure`. +- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) - - [`NIX_PATH`](#env-NIX_PATH)\ - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/values.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + Indicator that tells if the current environment was set up by + `nix-shell`. It can have the values `pure` or `impure`. - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: +- [`NIX_PATH`](#env-NIX_PATH) - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` + A colon-separated list of directories used to look up the location of Nix + expressions using [paths](@docroot@/language/values.md#type-path) + enclosed in angle brackets (i.e., ``), + e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the + [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). - If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: + If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: - error: file 'nixpkgs' was not found in the Nix search path + 1. `$HOME/.nix-defexpr/channels` + 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` + 3. `/nix/var/nix/profiles/per-user/root/channels` - - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE)\ - Normally, the Nix store directory (typically `/nix/store`) is not - allowed to contain any symlink components. This is to prevent - “impure” builds. Builders sometimes “canonicalise” paths by - resolving all symlink components. Thus, builds on different machines - (with `/nix/store` resolving to different locations) could yield - different results. This is generally not a problem, except when - builds are deployed to machines where `/nix/store` resolves - differently. If you are sure that you’re not going to do that, you - can set `NIX_IGNORE_SYMLINK_STORE` to `1`. + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + For example, attempting to use `` will produce: - Note that if you’re symlinking the Nix store so that you can put it - on another file system than the root file system, on Linux you’re - better off using `bind` mount points, e.g., + error: file 'nixpkgs' was not found in the Nix search path - ```console - $ mkdir /nix - $ mount -o bind /mnt/otherdisk/nix /nix - ``` +- [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) - Consult the mount 8 manual page for details. + Normally, the Nix store directory (typically `/nix/store`) is not + allowed to contain any symlink components. This is to prevent + “impure” builds. Builders sometimes “canonicalise” paths by + resolving all symlink components. Thus, builds on different machines + (with `/nix/store` resolving to different locations) could yield + different results. This is generally not a problem, except when + builds are deployed to machines where `/nix/store` resolves + differently. If you are sure that you’re not going to do that, you + can set `NIX_IGNORE_SYMLINK_STORE` to `1`. - - [`NIX_STORE_DIR`](#env-NIX_STORE_DIR)\ - Overrides the location of the Nix store (default `prefix/store`). + Note that if you’re symlinking the Nix store so that you can put it + on another file system than the root file system, on Linux you’re + better off using `bind` mount points, e.g., - - [`NIX_DATA_DIR`](#env-NIX_DATA_DIR)\ - Overrides the location of the Nix static data directory (default - `prefix/share`). + ```console + $ mkdir /nix + $ mount -o bind /mnt/otherdisk/nix /nix + ``` - - [`NIX_LOG_DIR`](#env-NIX_LOG_DIR)\ - Overrides the location of the Nix log directory (default - `prefix/var/log/nix`). + Consult the mount 8 manual page for details. - - [`NIX_STATE_DIR`](#env-NIX_STATE_DIR)\ - Overrides the location of the Nix state directory (default - `prefix/var/nix`). +- [`NIX_STORE_DIR`](#env-NIX_STORE_DIR) - - [`NIX_CONF_DIR`](#env-NIX_CONF_DIR)\ - Overrides the location of the system Nix configuration directory - (default `prefix/etc/nix`). + Overrides the location of the Nix store (default `prefix/store`). - - [`NIX_CONFIG`](#env-NIX_CONFIG)\ - Applies settings from Nix configuration from the environment. - The content is treated as if it was read from a Nix configuration file. - Settings are separated by the newline character. +- [`NIX_DATA_DIR`](#env-NIX_DATA_DIR) - - [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES)\ - Overrides the location of the Nix user configuration files to load from. + Overrides the location of the Nix static data directory (default + `prefix/share`). - The default are the locations according to the [XDG Base Directory Specification]. - See the [XDG Base Directories](#xdg-base-directories) sub-section for details. +- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) - The variable is treated as a list separated by the `:` token. + Overrides the location of the Nix log directory (default + `prefix/var/log/nix`). - - [`TMPDIR`](#env-TMPDIR)\ - Use the specified directory to store temporary files. In particular, - this includes temporary build directories; these can take up - substantial amounts of disk space. The default is `/tmp`. +- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) - - [`NIX_REMOTE`](#env-NIX_REMOTE)\ - This variable should be set to `daemon` if you want to use the Nix - daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](@docroot@/installation/multi-user.md). If the Nix - daemon's Unix socket is at some non-standard path, this variable - should be set to `unix://path/to/socket`. Otherwise, it should be - left unset. + Overrides the location of the Nix state directory (default + `prefix/var/nix`). - - [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS)\ - If set to `1`, Nix will print some evaluation statistics, such as - the number of values allocated. +- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) - - [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS)\ - If set to `1`, Nix will print how often functions were called during - Nix expression evaluation. This is useful for profiling your Nix - expressions. + Overrides the location of the system Nix configuration directory + (default `prefix/etc/nix`). - - [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE)\ - If Nix has been configured to use the Boehm garbage collector, this - variable sets the initial size of the heap in bytes. It defaults to - 384 MiB. Setting it to a low value reduces memory consumption, but - will increase runtime due to the overhead of garbage collection. +- [`NIX_CONFIG`](#env-NIX_CONFIG) + + Applies settings from Nix configuration from the environment. + The content is treated as if it was read from a Nix configuration file. + Settings are separated by the newline character. + +- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) + + Overrides the location of the Nix user configuration files to load from. + + The default are the locations according to the [XDG Base Directory Specification]. + See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + + The variable is treated as a list separated by the `:` token. + +- [`TMPDIR`](#env-TMPDIR) + + Use the specified directory to store temporary files. In particular, + this includes temporary build directories; these can take up + substantial amounts of disk space. The default is `/tmp`. + +- [`NIX_REMOTE`](#env-NIX_REMOTE) + + This variable should be set to `daemon` if you want to use the Nix + daemon to execute Nix operations. This is necessary in [multi-user + Nix installations](@docroot@/installation/multi-user.md). If the Nix + daemon's Unix socket is at some non-standard path, this variable + should be set to `unix://path/to/socket`. Otherwise, it should be + left unset. + +- [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS) + + If set to `1`, Nix will print some evaluation statistics, such as + the number of values allocated. + +- [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS) + + If set to `1`, Nix will print how often functions were called during + Nix expression evaluation. This is useful for profiling your Nix + expressions. + +- [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE) + + If Nix has been configured to use the Boehm garbage collector, this + variable sets the initial size of the heap in bytes. It defaults to + 384 MiB. Setting it to a low value reduces memory consumption, but + will increase runtime due to the overhead of garbage collection. ## XDG Base Directories From 47b3508665bb9dc6a8467841ce487c7426080f17 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Oct 2023 11:57:31 -0400 Subject: [PATCH 124/402] Use positive source filtering for the standalone functional tests job Additionally this skipping of the building is reimplemented to be a bit more robust and use the same idioms as the functionality for skipping the tests. In particular, it will now work even if the source files exist, so we can do this during development too. --- Makefile | 15 +++++++--- Makefile.config.in | 3 +- configure.ac | 11 +++++-- flake.nix | 60 +++++++++++++++++++++++++-------------- local.mk | 2 -- tests/functional/local.mk | 6 +--- 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 6658e3490..4f4ac0c6e 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +-include Makefile.config +clean-files += Makefile.config + +ifeq ($(ENABLE_BUILD), yes) makefiles = \ mk/precompiled-headers.mk \ local.mk \ @@ -18,15 +22,18 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ doc/internal-api/local.mk +endif --include Makefile.config - -ifeq ($(tests), yes) +ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ src/libutil/tests/local.mk \ src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk \ + src/libexpr/tests/local.mk +endif + +ifeq ($(ENABLE_TESTS), yes) +makefiles += \ tests/functional/local.mk \ tests/functional/ca/local.mk \ tests/functional/dyn-drv/local.mk \ diff --git a/Makefile.config.in b/Makefile.config.in index 707cfe0e3..19992fa20 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -46,5 +46,6 @@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ -tests = @tests@ +ENABLE_BUILD = @ENABLE_BUILD@ +ENABLE_TESTS = @ENABLE_TESTS@ internal_api_docs = @internal_api_docs@ diff --git a/configure.ac b/configure.ac index 6d78237f0..225baf6b5 100644 --- a/configure.ac +++ b/configure.ac @@ -152,12 +152,17 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LDFLAGS="-latomic $LDFLAGS" fi +# Running the functional tests without building Nix is useful for testing +# different pre-built versions of Nix against each other. +AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), + ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) +AC_SUBST(ENABLE_BUILD) # Building without tests is useful for bootstrapping with a smaller footprint # or running the tests in a separate derivation. Otherwise, we do compile and # run them. AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), - tests=$enableval, tests=yes) -AC_SUBST(tests) + ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) +AC_SUBST(ENABLE_TESTS) # Building without API docs is the default as Nix' C++ interfaces are internal and unstable. AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), @@ -289,7 +294,7 @@ if test "$gc" = yes; then fi -if test "$tests" = yes; then +if test "$ENABLE_TESTS" = yes; then # Look for gtest. PKG_CHECK_MODULES([GTEST], [gtest_main]) diff --git a/flake.nix b/flake.nix index bec62fb95..1fe26853f 100644 --- a/flake.nix +++ b/flake.nix @@ -59,29 +59,40 @@ # that would interfere with repo semantics. fileset.fileFilter (f: f.name != ".gitignore") ./.; + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + nixSrc = fileset.toSource { root = ./.; fileset = fileset.intersect baseFiles (fileset.unions [ - ./.version + configureFiles + topLevelBuildFiles ./boehmgc-coroutine-sp-fallback.diff - ./bootstrap.sh - ./configure.ac ./doc - ./local.mk - ./m4 - ./Makefile - ./Makefile.config.in ./misc - ./mk ./precompiled-headers.h ./src - ./tests/functional ./unit-test-data ./COPYING ./scripts/local.mk - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md + functionalTestFiles ]); }; @@ -252,7 +263,6 @@ testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; - HAVE_LOCAL_NIX_BUILD = false; name = "nix-tests" + optionalString @@ -261,7 +271,14 @@ "-${client.version}-against-${daemon.version}"; inherit version; - src = nixSrc; + src = fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + functionalTestFiles + ]); + }; VERSION_SUFFIX = versionSuffix; @@ -271,19 +288,20 @@ enableParallelBuilding = true; - configureFlags = testConfigureFlags; # otherwise configure fails + configureFlags = + testConfigureFlags # otherwise configure fails + ++ [ "--disable-build" ]; + dontBuild = true; doInstallCheck = true; - buildPhase = '' - # Remove the source files to make sure that we're not accidentally rebuilding Nix - rm src/**/*.cc - ''; - installPhase = '' mkdir -p $out ''; - installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES"; + installCheckPhase = '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + ''; }; binaryTarball = nix: pkgs: diff --git a/local.mk b/local.mk index 6951c179e..3f3abb9f0 100644 --- a/local.mk +++ b/local.mk @@ -1,5 +1,3 @@ -clean-files += Makefile.config - GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 4d29f60a1..6f6c94fe6 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -1,7 +1,3 @@ -# whether to run the tests that assume that we have a local build of -# Nix -HAVE_LOCAL_NIX_BUILD ?= 1 - nix_tests = \ test-infra.sh \ init.sh \ @@ -131,7 +127,7 @@ ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif -ifeq ($(HAVE_LOCAL_NIX_BUILD), 1) +ifeq ($(ENABLE_BUILD), yes) nix_tests += test-libstoreconsumer.sh ifeq ($(BUILD_SHARED_LIBS), 1) From 6654b4e3b4cf4b2181a0d14aa792d56eeab6219a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 9 Oct 2023 07:56:59 -0400 Subject: [PATCH 125/402] Use positive source filtering for the Perl bindings --- flake.nix | 10 +++++++++- perl/Makefile | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 1fe26853f..301e65545 100644 --- a/flake.nix +++ b/flake.nix @@ -477,7 +477,15 @@ passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { name = "nix-perl-${version}"; - src = self; + src = fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + ./perl + ./.version + ./m4 + ./mk + ]); + }; nativeBuildInputs = [ buildPackages.autoconf-archive diff --git a/perl/Makefile b/perl/Makefile index c2c95f255..832668dd1 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,12 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a -I ../src +GLOBAL_CXXFLAGS += -g -Wall -std=c++2a + +# A convenience for concurrent development of Nix and its Perl bindings. +# Not needed in a standalone build of the Perl bindings. +ifneq ("$(wildcard ../src)", "") + GLOBAL_CXXFLAGS += -I ../src +endif -include Makefile.config From 81d3a8542a6c919bb2302a877fd4443a6c07a2dc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 9 Oct 2023 10:21:18 -0400 Subject: [PATCH 126/402] doc: Slight reword of "interpolated expression" in paragraph I was sleepy and confused that "interpolated expression" was a new type of thing at first. This nudges the reader to understand that its just a regular expression, and these conditions are imposed by the interpolation operation. --- doc/manual/src/language/string-interpolation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 2e650e348..e999b287b 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -92,7 +92,7 @@ Attributes can be selected with interpolated strings. # Interpolated expression -An interpolated expression must evaluate to one of the following: +An expression that is interpolated must evaluate to one of the following: - a [string] - a [path] From c32084a12cb922f54829b329fa1527f865e5b8ca Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Oct 2023 16:25:53 +0200 Subject: [PATCH 127/402] printStats -> maybePrintStats --- src/libcmd/command.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/eval.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 4fc197956..a88ba8134 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -98,7 +98,7 @@ EvalCommand::EvalCommand() EvalCommand::~EvalCommand() { if (evalState) - evalState->printStats(); + evalState->maybePrintStats(); } ref EvalCommand::getEvalStore() diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 600444955..b18eb5266 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2491,7 +2491,7 @@ bool EvalState::fullGC() { #endif } -void EvalState::printStats() +void EvalState::maybePrintStats() { bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3bfa34ef6..c95558952 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -714,7 +714,7 @@ public: * Performs a full memory GC before printing the statistics, so that the * GC statistics are more accurate. */ - void printStats(); + void maybePrintStats(); /** * Print statistics, unconditionally, cheaply, without performing a GC first. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index e2189fc66..2895c5e3c 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -344,7 +344,7 @@ static void main_nix_build(int argc, char * * argv) } } - state->printStats(); + state->maybePrintStats(); auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e455a50fd..01742daa8 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1531,7 +1531,7 @@ static int main_nix_env(int argc, char * * argv) op(globals, std::move(opFlags), std::move(opArgs)); - globals.state->printStats(); + globals.state->maybePrintStats(); return 0; } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 446b27e66..d40196497 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -189,7 +189,7 @@ static int main_nix_instantiate(int argc, char * * argv) evalOnly, outputKind, xmlOutputSourceLocation, e); } - state->printStats(); + state->maybePrintStats(); return 0; } From 2f0b508c29f1c8eedc1f93c1a2dc31e8eb60de9a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 9 Oct 2023 12:52:58 -0400 Subject: [PATCH 128/402] Get rid of `bootstrap.sh` For people working on Nix with `nix develop`, it's better to just use `autoreconfPhase` and `configurePhase`, which is standard Nixpkgs / nix shell make from Nixpkgs practice --- it is good to emphasize the degree to which Nix is *just* a regular C++ project which can be worked on in the regular way. (For people running `nix-shell`, the story is similar, except `configurePhase` would use non-writable store paths, which matters for hte times we use output paths before `make install`, so I kept the existing `./configure ...` instruction.) For people building Nix without Nix (e.g. packaging it for another distro) they also don't need `bootstrap.sh`, and can just run `autoreconf -vfi` directly. (More likely, they have their own idioms to do this just as we have `autoreconfPhase`.) --- bootstrap.sh | 4 ---- doc/manual/src/contributing/hacking.md | 6 +++--- doc/manual/src/installation/building-source.md | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) delete mode 100755 bootstrap.sh diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index e3e259351..000000000 --- a/bootstrap.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -e -rm -f aclocal.m4 -mkdir -p config -exec autoreconf -vfi diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index db393942c..38c144fcc 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -42,8 +42,8 @@ $ nix develop .#native-clang11StdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ ./bootstrap.sh -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ autoreconfPhase +[nix-shell]$ configurePhase [nix-shell]$ make -j $NIX_BUILD_CORES ``` @@ -86,7 +86,7 @@ $ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ ./bootstrap.sh +[nix-shell]$ autoreconfPhase [nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out [nix-shell]$ make -j $NIX_BUILD_CORES ``` diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index ed1efffd8..7dad9805a 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -3,7 +3,7 @@ After cloning Nix's Git repository, issue the following commands: ```console -$ ./bootstrap.sh +$ autoreconf -vfi $ ./configure options... $ make $ make install From be81764320fc28131d23b85575076218eb7424c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 04:39:57 +0000 Subject: [PATCH 129/402] Factor out bits of the worker protocol to use elsewhere This introduces some shared infrastructure for our notion of protocols. We can then define multiple protocols in terms of that notion. We an also express how particular protocols depend on each other. For example, we can define a common protocol and a worker protocol, where the second depends on the first in terms of the data types it can read and write. The "serve" protocol can just use the common one for now, but will eventually need its own machinary just like the worker protocol for version-aware serialisers --- doc/internal-api/doxygen.cfg.in | 30 ++++ src/libstore/build/derivation-goal.cc | 10 +- src/libstore/common-protocol-impl.hh | 41 +++++ src/libstore/common-protocol.cc | 98 +++++++++++ src/libstore/common-protocol.hh | 106 ++++++++++++ src/libstore/derivations.cc | 12 +- src/libstore/export-import.cc | 12 +- src/libstore/legacy-ssh-store.cc | 44 ++--- .../length-prefixed-protocol-helper.hh | 162 ++++++++++++++++++ src/libstore/worker-protocol-impl.hh | 111 ++++-------- src/libstore/worker-protocol.cc | 81 +-------- src/libstore/worker-protocol.hh | 60 ++----- src/nix-store/nix-store.cc | 26 +-- 13 files changed, 542 insertions(+), 251 deletions(-) create mode 100644 src/libstore/common-protocol-impl.hh create mode 100644 src/libstore/common-protocol.cc create mode 100644 src/libstore/common-protocol.hh create mode 100644 src/libstore/length-prefixed-protocol-helper.hh diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 8f526536d..599be2470 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -54,6 +54,23 @@ INPUT = \ src/nix-env \ src/nix-store +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the # preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of @@ -61,3 +78,16 @@ INPUT = \ # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @RAPIDCHECK_HEADERS@ + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = \ + DECLARE_COMMON_SERIALISER \ + DECLARE_WORKER_SERIALISER \ + DECLARE_SERVE_SERIALISER \ + LENGTH_PREFIXED_PROTO_HELPER diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 83c0a3135..dc4d91079 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -8,8 +8,8 @@ #include "util.hh" #include "archive.hh" #include "compression.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts @@ -1185,11 +1185,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } - WorkerProto::WriteConn conn { hook->sink }; + CommonProto::WriteConn conn { hook->sink }; /* Tell the hook all the inputs that have to be copied to the remote system. */ - WorkerProto::write(worker.store, conn, inputPaths); + CommonProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1200,7 +1200,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - WorkerProto::write(worker.store, conn, missingOutputs); + CommonProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh new file mode 100644 index 000000000..079c182b8 --- /dev/null +++ b/src/libstore/common-protocol-impl.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "common-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + + +/* protocol-specific templates */ + +} diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc new file mode 100644 index 000000000..f906814bc --- /dev/null +++ b/src/libstore/common-protocol.cc @@ -0,0 +1,98 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "archive.hh" +#include "derivations.hh" + +#include + +namespace nix { + +/* protocol-agnostic definitions */ + +std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return readString(conn.from); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +{ + conn.to << str; +} + + +StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return store.parseStorePath(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +{ + conn.to << store.printStorePath(storePath); +} + + +ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +{ + conn.to << renderContentAddress(ca); +} + + +Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << realisation.toJSON().dump(); +} + + +DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + auto s = readString(conn.from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +{ + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parseOpt(readString(conn.from)); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +{ + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh new file mode 100644 index 000000000..f3f28972a --- /dev/null +++ b/src/libstore/common-protocol.hh @@ -0,0 +1,106 @@ +#pragma once +///@file + +#include "serialise.hh" + +namespace nix { + +class Store; +struct Source; + +// items being serialized +class StorePath; +struct ContentAddress; +struct DrvOutput; +struct Realisation; + + +/** + * Shared serializers between the worker protocol, serve protocol, and a + * few others. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct CommonProto +{ + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + }; + + template + struct Serialise; + + /** + * Wrapper function around `CommonProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, conn, t); + } +}; + +#define DECLARE_COMMON_SERIALISER(T) \ + struct CommonProto::Serialise< T > \ + { \ + static T read(const Store & store, CommonProto::ReadConn conn); \ + static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + } + +template<> +DECLARE_COMMON_SERIALISER(std::string); +template<> +DECLARE_COMMON_SERIALISER(StorePath); +template<> +DECLARE_COMMON_SERIALISER(ContentAddress); +template<> +DECLARE_COMMON_SERIALISER(DrvOutput); +template<> +DECLARE_COMMON_SERIALISER(Realisation); + +template +DECLARE_COMMON_SERIALISER(std::vector); +template +DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ + +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. + */ +template<> +DECLARE_COMMON_SERIALISER(std::optional); +template<> +DECLARE_COMMON_SERIALISER(std::optional); + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 67069c3c9..fc17e520c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,8 +4,8 @@ #include "globals.hh" #include "util.hh" #include "split.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "fs-accessor.hh" #include #include @@ -895,8 +895,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = in }); + drv.inputSrcs = CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -944,8 +944,8 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw); } - WorkerProto::write(store, - WorkerProto::WriteConn { .to = out }, + CommonProto::write(store, + CommonProto::WriteConn { .to = out }, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index e866aeb42..87b2f8741 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,8 +1,8 @@ #include "serialise.hh" #include "store-api.hh" #include "archive.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include @@ -46,8 +46,8 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - WorkerProto::write(*this, - WorkerProto::WriteConn { .to = teeSink }, + CommonProto::write(*this, + CommonProto::WriteConn { .to = teeSink }, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") @@ -76,8 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = WorkerProto::Serialise::read(*this, - WorkerProto::ReadConn { .from = source }); + auto references = CommonProto::Serialise::read(*this, + CommonProto::ReadConn { .from = source }); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 78b05031a..7bf4476d0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,8 +6,8 @@ #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -50,37 +50,37 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor bool good = true; /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `CommonProto::ReadConn`. This makes it easy to use the + * factored out common protocol serialisers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The common protocol connection types are unidirectional, unlike * this type. * - * @todo Use server protocol serializers, not worker protocol + * @todo Use server protocol serializers, not common protocol * serializers, once we have made that distiction. */ - operator WorkerProto::ReadConn () + operator CommonProto::ReadConn () { - return WorkerProto::ReadConn { + return CommonProto::ReadConn { .from = from, }; } /* - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `CommonProto::WriteConn`. This makes it easy to use the + * factored out common protocol searlizers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The common protocol connection types are unidirectional, unlike * this type. * - * @todo Use server protocol serializers, not worker protocol + * @todo Use server protocol serializers, not common protocol * serializers, once we have made that distiction. */ - operator WorkerProto::WriteConn () + operator CommonProto::WriteConn () { - return WorkerProto::WriteConn { + return CommonProto::WriteConn { .to = to, }; } @@ -183,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, *conn); + info->references = CommonProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +217,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + CommonProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + CommonProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -331,7 +331,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); + auto builtOutputs = CommonProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -409,10 +409,10 @@ public: conn->to << ServeProto::Command::QueryClosure << includeOutputs; - WorkerProto::write(*this, *conn, paths); + CommonProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : WorkerProto::Serialise::read(*this, *conn)) + for (auto & i : CommonProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -425,10 +425,10 @@ public: << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, *conn, paths); + CommonProto::write(*this, *conn, paths); conn->to.flush(); - return WorkerProto::Serialise::read(*this, *conn); + return CommonProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh new file mode 100644 index 000000000..4061b0cd6 --- /dev/null +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -0,0 +1,162 @@ +#pragma once +/** + * @file Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * Used by both the Worker and Serve protocols. + */ + +#include "types.hh" + +namespace nix { + +class Store; + +/** + * Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * @param T The type of the collection being serialised + * + * @param Inner This the most important parameter; this is the "inner" + * protocol. The user of this will substitute `MyProtocol` or similar + * when making a `MyProtocol::Serialiser>`. Note that the + * inside is allowed to call to call `Inner::Serialiser` on different + * types. This is especially important for `std::map` which doesn't have + * a single `T` but one `K` and one `V`. + */ +template +struct LengthPrefixedProtoHelper; + +/*! + * \typedef LengthPrefixedProtoHelper::S + * + * Read this as simply `using S = Inner::Serialise;`. + * + * It would be nice to use that directly, but C++ doesn't seem to allow + * it. The `typename` keyword needed to refer to `Inner` seems to greedy + * (low precedence), and then C++ complains that `Serialise` is not a + * type parameter but a real type. + * + * Making this `S` alias seems to be the only way to avoid these issues. + */ + +#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ + struct LengthPrefixedProtoHelper< Inner, T > \ + { \ + static T read(const Store & store, typename Inner::ReadConn conn); \ + static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + private: \ + template using S = typename Inner::template Serialise; \ + } + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); + +template +#define _X std::map +LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); +#undef _X + +template +std::vector +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::vector resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.push_back(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::set +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::set resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.insert(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::set & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::map +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::map resMap; + auto size = readNum(conn.from); + while (size--) { + auto k = S::read(store, conn); + auto v = S::read(store, conn); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::map & resMap) +{ + conn.to << resMap.size(); + for (auto & i : resMap) { + S::write(store, conn, i.first); + S::write(store, conn, i.second); + } +} + +template +std::tuple +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + return std::tuple { + S::read(store, conn)..., + }; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (S::write(store, conn, args), ...); + }, res); +} + +} diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index 4f797f95a..c043588d6 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -9,86 +9,51 @@ */ #include "worker-protocol.hh" +#include "length-prefixed-protocol-helper.hh" namespace nix { +/* protocol-agnostic templates */ + +#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ template -std::vector WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +struct WorkerProto::Serialise { - std::vector resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.push_back(WorkerProto::Serialise::read(store, conn)); + static T read(const Store & store, WorkerProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); + static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); } -} +}; -template -std::set WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::set resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.insert(WorkerProto::Serialise::read(store, conn)); - } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::set & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); - } -} - -template -std::map WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::map resMap; - auto size = readNum(conn.from); - while (size--) { - auto k = WorkerProto::Serialise::read(store, conn); - auto v = WorkerProto::Serialise::read(store, conn); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::map & resMap) -{ - conn.to << resMap.size(); - for (auto & i : resMap) { - WorkerProto::Serialise::write(store, conn, i.first); - WorkerProto::Serialise::write(store, conn, i.second); - } -} - -template -std::tuple WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - return std::tuple { - WorkerProto::Serialise::read(store, conn)..., - }; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple & res) -{ - std::apply([&](const Us &... args) { - (WorkerProto::Serialise::write(store, conn, args), ...); - }, res); -} +/* protocol-specific templates */ } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a23130743..415e66f16 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -12,27 +12,7 @@ namespace nix { -std::string WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return readString(conn.from); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str) -{ - conn.to << str; -} - - -StorePath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return store.parseStorePath(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath) -{ - conn.to << store.printStorePath(storePath); -} - +/* protocol-specific definitions */ std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { @@ -68,17 +48,6 @@ void WorkerProto::Serialise>::write(const Store & sto } -ContentAddress WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca) -{ - conn.to << renderContentAddress(ca); -} - - DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); @@ -91,32 +60,6 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -Realisation WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << realisation.toJSON().dump(); -} - - -DrvOutput WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - - KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); @@ -168,26 +111,4 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - auto s = readString(conn.from); - return s == "" ? std::optional {} : store.parseStorePath(s); -} - -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & storePathOpt) -{ - conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parseOpt(readString(conn.from)); -} - -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & caOpt) -{ - conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); -} - } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b7f42f24d..c84060103 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "serialise.hh" +#include "common-protocol.hh" namespace nix { @@ -28,11 +28,7 @@ class Store; struct Source; // items being serialised -class StorePath; -struct ContentAddress; struct DerivedPath; -struct DrvOutput; -struct Realisation; struct BuildResult; struct KeyedBuildResult; enum TrustedFlag : bool; @@ -193,60 +189,32 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) * be legal specialization syntax. See below for what that looks like in * practice. */ -#define MAKE_WORKER_PROTO(T) \ - struct WorkerProto::Serialise< T > { \ +#define DECLARE_WORKER_SERIALISER(T) \ + struct WorkerProto::Serialise< T > \ + { \ static T read(const Store & store, WorkerProto::ReadConn conn); \ static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> -MAKE_WORKER_PROTO(std::string); +DECLARE_WORKER_SERIALISER(DerivedPath); template<> -MAKE_WORKER_PROTO(StorePath); +DECLARE_WORKER_SERIALISER(BuildResult); template<> -MAKE_WORKER_PROTO(ContentAddress); +DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> -MAKE_WORKER_PROTO(DerivedPath); -template<> -MAKE_WORKER_PROTO(DrvOutput); -template<> -MAKE_WORKER_PROTO(Realisation); -template<> -MAKE_WORKER_PROTO(BuildResult); -template<> -MAKE_WORKER_PROTO(KeyedBuildResult); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::optional); template -MAKE_WORKER_PROTO(std::vector); +DECLARE_WORKER_SERIALISER(std::vector); template -MAKE_WORKER_PROTO(std::set); +DECLARE_WORKER_SERIALISER(std::set); template -MAKE_WORKER_PROTO(std::tuple); +DECLARE_WORKER_SERIALISER(std::tuple); +#define COMMA_ , template -#define X_ std::map -MAKE_WORKER_PROTO(X_); -#undef X_ - -/** - * These use the empty string for the null case, relying on the fact - * that the underlying types never serialise to the empty string. - * - * We do this instead of a generic std::optional instance because - * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - * the same reason, we don't have a std::variant instances (ordinal - * tags 0...n). - * - * We could the generic instances and then these as specializations for - * compatability, but that's proven a bit finnicky, and also makes the - * worker protocol harder to implement in other languages where such - * specializations may not be allowed. - */ -template<> -MAKE_WORKER_PROTO(std::optional); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::map); +#undef COMMA_ } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 96c3f7d7e..6fc22214a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,8 +11,8 @@ #include "serve-protocol.hh" #include "shared.hh" #include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -821,8 +821,8 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); - WorkerProto::ReadConn rconn { .from = in }; - WorkerProto::WriteConn wconn { .to = out }; + CommonProto::ReadConn rconn { .from = in }; + CommonProto::WriteConn wconn { .to = out }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = CommonProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -876,19 +876,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - WorkerProto::write(*store, wconn, store->queryValidPaths(paths)); + CommonProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = CommonProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, wconn, info->references); + CommonProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -916,7 +916,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::Serialise::read(*store, rconn), out); + store->exportPaths(CommonProto::Serialise::read(*store, rconn), out); break; } @@ -962,7 +962,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); + CommonProto::write(*store, wconn, builtOutputs); } break; @@ -971,9 +971,9 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::Serialise::read(*store, rconn), + store->computeFSClosure(CommonProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - WorkerProto::write(*store, wconn, closure); + CommonProto::write(*store, wconn, closure); break; } @@ -988,7 +988,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.references = CommonProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); From 4de54b21907e3fc41b9e6250396eb01d2b10c4db Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 4 Oct 2023 23:27:50 -0400 Subject: [PATCH 130/402] Unit test the "common protocol" too Copy the relevant tests to ensure the new interfaces added in the last commit are tested. Perhaps I should try to deduplicat these tests some more. However its not clear how to do that outside of a big ugly C++ macro. https://github.com/google/googletest/blob/main/docs/advanced.md has some stuff but it is cumbersome and I didn't figure it out yet. This is done in a separate commit in order to be sure that the first commit really didn't change any behavior; if we changed the implementation and the tests at once, it would be harder to tell whether or not some behavioral changes slipped in what is supposed to be a "pure refactor". Co-Authored-By: Valentin Gagarin --- src/libstore/tests/characterization.hh | 23 +++ src/libstore/tests/common-protocol.cc | 152 ++++++++++++++++++ src/libstore/tests/protocol.hh | 88 ++++++++++ src/libstore/tests/worker-protocol.cc | 90 ++--------- .../common-protocol/content-address.bin | Bin 0 -> 208 bytes .../libstore/common-protocol/drv-output.bin | Bin 0 -> 176 bytes .../optional-content-address.bin | Bin 0 -> 64 bytes .../common-protocol/optional-store-path.bin | Bin 0 -> 72 bytes .../libstore/common-protocol/realisation.bin | Bin 0 -> 520 bytes .../libstore/common-protocol/set.bin | Bin 0 -> 152 bytes .../libstore/common-protocol/store-path.bin | Bin 0 -> 120 bytes .../libstore/common-protocol/string.bin | Bin 0 -> 88 bytes .../libstore/common-protocol/vector.bin | Bin 0 -> 152 bytes 13 files changed, 280 insertions(+), 73 deletions(-) create mode 100644 src/libstore/tests/characterization.hh create mode 100644 src/libstore/tests/common-protocol.cc create mode 100644 src/libstore/tests/protocol.hh create mode 100644 unit-test-data/libstore/common-protocol/content-address.bin create mode 100644 unit-test-data/libstore/common-protocol/drv-output.bin create mode 100644 unit-test-data/libstore/common-protocol/optional-content-address.bin create mode 100644 unit-test-data/libstore/common-protocol/optional-store-path.bin create mode 100644 unit-test-data/libstore/common-protocol/realisation.bin create mode 100644 unit-test-data/libstore/common-protocol/set.bin create mode 100644 unit-test-data/libstore/common-protocol/store-path.bin create mode 100644 unit-test-data/libstore/common-protocol/string.bin create mode 100644 unit-test-data/libstore/common-protocol/vector.bin diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh new file mode 100644 index 000000000..5f366cb42 --- /dev/null +++ b/src/libstore/tests/characterization.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +namespace nix { + +/** + * The path to the `unit-test-data` directory. See the contributing + * guide in the manual for further details. + */ +static Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +} diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc new file mode 100644 index 000000000..de57211f0 --- /dev/null +++ b/src/libstore/tests/common-protocol.cc @@ -0,0 +1,152 @@ +#include + +#include +#include + +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "common-protocol"; + +using CommonProtoTest = ProtoTest; + +CHARACTERIZATION_TEST( + CommonProtoTest, + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh new file mode 100644 index 000000000..86a900757 --- /dev/null +++ b/src/libstore/tests/protocol.hh @@ -0,0 +1,88 @@ +#include +#include + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +template +class ProtoTest : public LibStoreTest +{ + /** + * Read this as simply `using S = Inner::Serialise;`. + * + * See `LengthPrefixedProtoHelper::S` for the same trick, and its + * rationale. + */ + template using S = typename Proto::template Serialise; + +public: + Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem + ".bin"; + } + + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + S::read( + *store, + typename Proto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + Proto::write( + *store, + typename Proto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VALUE) \ + TEST_F(FIXTURE, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(FIXTURE, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } + +} diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index fa7cbe121..59d7ff96d 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -7,85 +7,17 @@ #include "worker-protocol-impl.hh" #include "derived-path.hh" #include "build-result.hh" -#include "tests/libstore.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" namespace nix { -class WorkerProtoTest : public LibStoreTest -{ -public: - Path unitTestData = getEnv("_NIX_TEST_UNIT_DATA").value() + "/libstore/worker-protocol"; +const char workerProtoDir[] = "worker-protocol"; - bool testAccept() { - return getEnv("_NIX_TEST_ACCEPT") == "1"; - } - - Path goldenMaster(std::string_view testStem) { - return unitTestData + "/" + testStem + ".bin"; - } - - /** - * Golden test for `T` reading - */ - template - void readTest(PathView testStem, T value) - { - if (testAccept()) - { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; - } - else - { - auto expected = readFile(goldenMaster(testStem)); - - T got = ({ - StringSource from { expected }; - WorkerProto::Serialise::read( - *store, - WorkerProto::ReadConn { .from = from }); - }); - - ASSERT_EQ(got, value); - } - } - - /** - * Golden test for `T` write - */ - template - void writeTest(PathView testStem, const T & value) - { - auto file = goldenMaster(testStem); - - StringSink to; - WorkerProto::write( - *store, - WorkerProto::WriteConn { .to = to }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } - } -}; - -#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ - TEST_F(WorkerProtoTest, NAME ## _read) { \ - readTest(STEM, VALUE); \ - } \ - TEST_F(WorkerProtoTest, NAME ## _write) { \ - writeTest(STEM, VALUE); \ - } +using WorkerProtoTest = ProtoTest; CHARACTERIZATION_TEST( + WorkerProtoTest, string, "string", (std::tuple { @@ -97,6 +29,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, storePath, "store-path", (std::tuple { @@ -105,6 +38,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, contentAddress, "content-address", (std::tuple { @@ -123,6 +57,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, derivedPath, "derived-path", (std::tuple { @@ -138,6 +73,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, drvOutput, "drv-output", (std::tuple { @@ -152,6 +88,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, realisation, "realisation", (std::tuple { @@ -183,6 +120,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, buildResult, "build-result", ({ @@ -240,6 +178,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, keyedBuildResult, "keyed-build-result", ({ @@ -275,6 +214,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalTrustedFlag, "optional-trusted-flag", (std::tuple, std::optional, std::optional> { @@ -284,6 +224,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, vector, "vector", (std::tuple, std::vector, std::vector, std::vector>> { @@ -294,6 +235,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, set, "set", (std::tuple, std::set, std::set, std::set>> { @@ -304,6 +246,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalStorePath, "optional-store-path", (std::tuple, std::optional> { @@ -314,6 +257,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalContentAddress, "optional-content-address", (std::tuple, std::optional> { diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/unit-test-data/libstore/common-protocol/content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..8f14bcdb3e50cc72d87106ce159a6fd7b9f75713 GIT binary patch literal 208 zcmY+;K@P$o5QSmy;|7WYrYjRqQfMm$+LWPzDW_MfG4bucKks(>Y#V56lkFOiEzi24 zUMdC5VUCa86OJ&gL0BF^B{HBQEY%f|I<6v7#q+l_PBirI5N~-md%37WE|g33QTE2k zc~-WF&R}7Oxc@o)T?ojpD!+*5=xpO;%IoCKSV5FMh={>xePK|8<${>E)*huNHZ(pX literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/unit-test-data/libstore/common-protocol/drv-output.bin new file mode 100644 index 0000000000000000000000000000000000000000..800a45fd8757a15e8810066aed48be56a600b7d3 GIT binary patch literal 176 zcmY++xeWp_5I|8{*$(Wn=b{B@!gGlfp_LHTb8N(qe)KM%U7Sq@}q)U|6h9n2j1!MT?w+C~^gO WO9=?G&ojrYjy`x4ZufnEe#tjMpDPdm literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/unit-test-data/libstore/common-protocol/optional-content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..f8cfe65ba27fd78ec7787eed7ebec8dfdead2fba GIT binary patch literal 64 zcmZQzfB#ECTBU06lLKJOBUy literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/unit-test-data/libstore/common-protocol/realisation.bin new file mode 100644 index 0000000000000000000000000000000000000000..2176c6c4afd96b32fe372de6084ac7f4c7a11d49 GIT binary patch literal 520 zcmdUrO-{o=429t+opq7s&z_l_03#3x z?rW}!>Uc!$VBY>Sr2mUC`<2<)qY;)1KN Po44)luetw6d9AunL$;dR literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/set.bin b/unit-test-data/libstore/common-protocol/set.bin new file mode 100644 index 0000000000000000000000000000000000000000..ce11ede7fe7aa061a30b211a18cd7a336f879de8 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obCnXkvMPU52{CpHXOdBEdVDg4g4KThDln(&+h63;a literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/unit-test-data/libstore/common-protocol/store-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..3fc05f2981d3398ab30ec34125c903cdcefa8729 GIT binary patch literal 120 tcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F?srQlM;)-IsqP{BM|@q literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/string.bin b/unit-test-data/libstore/common-protocol/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..aa7b5a604745b735473ec34283177e69320776e4 GIT binary patch literal 88 zcmZQzfB+^aoskJ)@Id+H8JQ)i3Pp)YNtq=eAx^0H( Date: Tue, 10 Oct 2023 00:32:36 +0200 Subject: [PATCH 131/402] annotate admonitions showing syntax also fix typos --- .../src/language/constructs/lookup-path.md | 2 +- doc/manual/src/language/operators.md | 16 ++++++++++++++++ doc/manual/src/language/values.md | 6 +++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index 8ec835309..e87d2922b 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -2,7 +2,7 @@ > **Syntax** > -> *lookup-path* = `<` *identifier* [ `/` *identifier* `]`... `>` +> *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index f8382ae19..a22c71109 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -35,6 +35,8 @@ ## Attribute selection +> **Syntax** +> > *attrset* `.` *attrpath* \[ `or` *expr* \] Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. @@ -42,12 +44,16 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). +> **Syntax** +> > *attrpath* = *name* [ `.` *name* ]... [Attribute selection]: #attribute-selection ## Has attribute +> **Syntax** +> > *attrset* `?` *attrpath* Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. @@ -70,6 +76,8 @@ The `+` operator is overloaded to also work on strings and paths. ## String concatenation +> **Syntax** +> > *string* `+` *string* Concatenate two [string]s and merge their string contexts. @@ -78,6 +86,8 @@ Concatenate two [string]s and merge their string contexts. ## Path concatenation +> **Syntax** +> > *path* `+` *path* Concatenate two [path]s. @@ -87,6 +97,8 @@ The result is a path. ## Path and string concatenation +> **Syntax** +> > *path* + *string* Concatenate *[path]* with *[string]*. @@ -100,6 +112,8 @@ The result is a path. ## String and path concatenation +> **Syntax** +> > *string* + *path* Concatenate *[string]* with *[path]*. @@ -117,6 +131,8 @@ The result is a string. ## Update +> **Syntax** +> > *attrset1* // *attrset2* Update [attribute set] *attrset1* with names and values from *attrset2*. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 8856d33ae..0bb656746 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -162,13 +162,17 @@ An attribute set is a collection of name-value-pairs (called *attributes*) enclo An attribute name can be an identifier or a [string](#string). An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). +> **Syntax** +> > *name* = *identifier* | *string* \ > *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` Names and values are separated by an equal sign (`=`). Each value is an arbitrary expression terminated by a semicolon (`;`). -> *attrset* = `{` [ *name* `=` *expr* `;` `]`... `}` +> **Syntax** +> +> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` Attributes can appear in any order. An attribute name may only occur once. From 7642894a4e1c9916f92a234c8cbaef604a7b7d00 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 03:49:49 +0200 Subject: [PATCH 132/402] reword documentation on lookup path resolution --- src/libexpr/primops.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index bfcac7510..3bed7c5aa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1698,13 +1698,14 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_findFile(PrimOp { .name = "__findFile", - .args = {"search path", "lookup path"}, + .args = {"search-path", "lookup-path"}, .doc = R"( - Look up the given path with the given search path. + Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`. - `prefix` is a relative path. - `path` denotes a file system location; the exact syntax depends on the command line interface. + A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + The exact syntax depends on the command line interface. Examples of search path attribute sets: @@ -1722,13 +1723,12 @@ static RegisterPrimOp primop_findFile(PrimOp { } ``` - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match. + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match: - This is the process for each entry: - If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`. - Note that the `path` may need to be downloaded at this point to look inside. - If the suffix is found inside that directory, then the entry is a match; - the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + Note that the `path` may need to be downloaded at this point to look inside. + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): From f7b8f8aff63b1a4c7289c423d7877c95c0c6ae1f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 May 2023 14:11:08 -0400 Subject: [PATCH 133/402] Introduce separate Serve protocol serialisers To start, it is just a clone of the common protocol. But now that we have the separate protocol implementations, we can add versioning information without the versions of one protocol leaking into another. Using the infrastructure from the previous commit, we don't have to duplicate code for shared behavior. Motivation: No more perverse incentives. [0] did some awkward things because the serialisers did not store the version. I don't want anyone making changes to be pushed towards keeping the serialization logic with the core data types just because it's easier or the alternative is tedious. The actual versioning of the Worker and Serve protocol serialisers (Common remains unversioned as the underlying mini-protocols are not versioned) will happen in subsequent commits / PRs. [0]: fe1f34fa60ad79e339c38e58af071a44774663f7 --- src/libstore/legacy-ssh-store.cc | 45 +++--- src/libstore/serve-protocol-impl.hh | 59 +++++++ src/libstore/serve-protocol.cc | 15 ++ src/libstore/serve-protocol.hh | 87 ++++++++++ src/libstore/tests/serve-protocol.cc | 152 ++++++++++++++++++ src/nix-store/nix-store.cc | 25 ++- .../serve-protocol/content-address.bin | Bin 0 -> 208 bytes .../libstore/serve-protocol/drv-output.bin | Bin 0 -> 176 bytes .../optional-content-address.bin | Bin 0 -> 64 bytes .../serve-protocol/optional-store-path.bin | Bin 0 -> 72 bytes .../libstore/serve-protocol/realisation.bin | Bin 0 -> 520 bytes .../libstore/serve-protocol/set.bin | Bin 0 -> 152 bytes .../libstore/serve-protocol/store-path.bin | Bin 0 -> 120 bytes .../libstore/serve-protocol/string.bin | Bin 0 -> 88 bytes .../libstore/serve-protocol/vector.bin | Bin 0 -> 152 bytes 15 files changed, 344 insertions(+), 39 deletions(-) create mode 100644 src/libstore/serve-protocol-impl.hh create mode 100644 src/libstore/serve-protocol.cc create mode 100644 src/libstore/tests/serve-protocol.cc create mode 100644 unit-test-data/libstore/serve-protocol/content-address.bin create mode 100644 unit-test-data/libstore/serve-protocol/drv-output.bin create mode 100644 unit-test-data/libstore/serve-protocol/optional-content-address.bin create mode 100644 unit-test-data/libstore/serve-protocol/optional-store-path.bin create mode 100644 unit-test-data/libstore/serve-protocol/realisation.bin create mode 100644 unit-test-data/libstore/serve-protocol/set.bin create mode 100644 unit-test-data/libstore/serve-protocol/store-path.bin create mode 100644 unit-test-data/libstore/serve-protocol/string.bin create mode 100644 unit-test-data/libstore/serve-protocol/vector.bin diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7bf4476d0..703ded0b2 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,11 +3,10 @@ #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "common-protocol.hh" -#include "common-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -50,37 +49,31 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor bool good = true; /** - * Coercion to `CommonProto::ReadConn`. This makes it easy to use the - * factored out common protocol serialisers with a + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The common protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not common protocol - * serializers, once we have made that distiction. */ - operator CommonProto::ReadConn () + operator ServeProto::ReadConn () { - return CommonProto::ReadConn { + return ServeProto::ReadConn { .from = from, }; } /* - * Coercion to `CommonProto::WriteConn`. This makes it easy to use the - * factored out common protocol searlizers with a + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The common protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not common protocol - * serializers, once we have made that distiction. */ - operator CommonProto::WriteConn () + operator ServeProto::WriteConn () { - return CommonProto::WriteConn { + return ServeProto::WriteConn { .to = to, }; } @@ -183,7 +176,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = CommonProto::Serialise::read(*this, *conn); + info->references = ServeProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +210,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - CommonProto::write(*this, *conn, info.references); + ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +239,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - CommonProto::write(*this, *conn, info.references); + ServeProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -331,7 +324,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = CommonProto::Serialise::read(*this, *conn); + auto builtOutputs = ServeProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -409,10 +402,10 @@ public: conn->to << ServeProto::Command::QueryClosure << includeOutputs; - CommonProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : CommonProto::Serialise::read(*this, *conn)) + for (auto & i : ServeProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -425,10 +418,10 @@ public: << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - CommonProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - return CommonProto::Serialise::read(*this, *conn); + return ServeProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh new file mode 100644 index 000000000..a3ce81026 --- /dev/null +++ b/src/libstore/serve-protocol-impl.hh @@ -0,0 +1,59 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "serve-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +SERVE_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ +template +struct ServeProto::Serialise +{ + static T read(const Store & store, ServeProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); + } + static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); + } +}; + +/* protocol-specific templates */ + +} diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc new file mode 100644 index 000000000..16a62b5bc --- /dev/null +++ b/src/libstore/serve-protocol.cc @@ -0,0 +1,15 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "archive.hh" + +#include + +namespace nix { + +/* protocol-specific definitions */ + +} diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 7e43b3969..e2345d450 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "common-protocol.hh" + namespace nix { #define SERVE_MAGIC_1 0x390c9deb @@ -10,6 +12,11 @@ namespace nix { #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + +class Store; +struct Source; + + /** * The "serve protocol", used by ssh:// stores. * @@ -22,6 +29,57 @@ struct ServeProto * Enumeration of all the request types for the protocol. */ enum struct Command : uint64_t; + + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + * + * This currently is just a `Source &`, but more fields will be added + * later. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + * + * This currently is just a `Sink &`, but more fields will be added + * later. + */ + struct WriteConn { + Sink & to; + }; + + /** + * Data type for canonical pairs of serialisers for the serve protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // See `worker-protocol.hh` for a longer explanation. +#if 0 + { + static T read(const Store & store, ReadConn conn); + static void write(const Store & store, WriteConn conn, const T & t); + }; +#endif + + /** + * Wrapper function around `ServeProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + ServeProto::Serialise::write(store, conn, t); + } }; enum struct ServeProto::Command : uint64_t @@ -58,4 +116,33 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) return s << (uint64_t) op; } +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define DECLARE_SERVE_SERIALISER(T) \ + struct ServeProto::Serialise< T > \ + { \ + static T read(const Store & store, ServeProto::ReadConn conn); \ + static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + }; + +template +DECLARE_SERVE_SERIALISER(std::vector); +template +DECLARE_SERVE_SERIALISER(std::set); +template +DECLARE_SERVE_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_SERVE_SERIALISER(std::map); +#undef COMMA_ + } diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc new file mode 100644 index 000000000..5d7df56cf --- /dev/null +++ b/src/libstore/tests/serve-protocol.cc @@ -0,0 +1,152 @@ +#include + +#include +#include + +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "serve-protocol"; + +using ServeProtoTest = ProtoTest; + +CHARACTERIZATION_TEST( + ServeProtoTest, + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6fc22214a..9b6c80a75 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,10 +9,9 @@ #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "shared.hh" #include "util.hh" -#include "common-protocol.hh" -#include "common-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -821,8 +820,8 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); - CommonProto::ReadConn rconn { .from = in }; - CommonProto::WriteConn wconn { .to = out }; + ServeProto::ReadConn rconn { .from = in }; + ServeProto::WriteConn wconn { .to = out }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -867,7 +866,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = CommonProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -876,19 +875,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - CommonProto::write(*store, wconn, store->queryValidPaths(paths)); + ServeProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = CommonProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - CommonProto::write(*store, wconn, info->references); + ServeProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -916,7 +915,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(CommonProto::Serialise::read(*store, rconn), out); + store->exportPaths(ServeProto::Serialise::read(*store, rconn), out); break; } @@ -962,7 +961,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - CommonProto::write(*store, wconn, builtOutputs); + ServeProto::write(*store, wconn, builtOutputs); } break; @@ -971,9 +970,9 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(CommonProto::Serialise::read(*store, rconn), + store->computeFSClosure(ServeProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - CommonProto::write(*store, wconn, closure); + ServeProto::write(*store, wconn, closure); break; } @@ -988,7 +987,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = CommonProto::Serialise::read(*store, rconn); + info.references = ServeProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/unit-test-data/libstore/serve-protocol/content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..8f14bcdb3e50cc72d87106ce159a6fd7b9f75713 GIT binary patch literal 208 zcmY+;K@P$o5QSmy;|7WYrYjRqQfMm$+LWPzDW_MfG4bucKks(>Y#V56lkFOiEzi24 zUMdC5VUCa86OJ&gL0BF^B{HBQEY%f|I<6v7#q+l_PBirI5N~-md%37WE|g33QTE2k zc~-WF&R}7Oxc@o)T?ojpD!+*5=xpO;%IoCKSV5FMh={>xePK|8<${>E)*huNHZ(pX literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/unit-test-data/libstore/serve-protocol/drv-output.bin new file mode 100644 index 0000000000000000000000000000000000000000..800a45fd8757a15e8810066aed48be56a600b7d3 GIT binary patch literal 176 zcmY++xeWp_5I|8{*$(Wn=b{B@!gGlfp_LHTb8N(qe)KM%U7Sq@}q)U|6h9n2j1!MT?w+C~^gO WO9=?G&ojrYjy`x4ZufnEe#tjMpDPdm literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/unit-test-data/libstore/serve-protocol/optional-content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..f8cfe65ba27fd78ec7787eed7ebec8dfdead2fba GIT binary patch literal 64 zcmZQzfB#ECTBU06lLKJOBUy literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/unit-test-data/libstore/serve-protocol/realisation.bin new file mode 100644 index 0000000000000000000000000000000000000000..2176c6c4afd96b32fe372de6084ac7f4c7a11d49 GIT binary patch literal 520 zcmdUrO-{o=429t+opq7s&z_l_03#3x z?rW}!>Uc!$VBY>Sr2mUC`<2<)qY;)1KN Po44)luetw6d9AunL$;dR literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/unit-test-data/libstore/serve-protocol/set.bin new file mode 100644 index 0000000000000000000000000000000000000000..ce11ede7fe7aa061a30b211a18cd7a336f879de8 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obCnXkvMPU52{CpHXOdBEdVDg4g4KThDln(&+h63;a literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/unit-test-data/libstore/serve-protocol/store-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..3fc05f2981d3398ab30ec34125c903cdcefa8729 GIT binary patch literal 120 tcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F?srQlM;)-IsqP{BM|@q literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/unit-test-data/libstore/serve-protocol/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..aa7b5a604745b735473ec34283177e69320776e4 GIT binary patch literal 88 zcmZQzfB+^aoskJ)@Id+H8JQ)i3Pp)YNtq=eAx^0H( Date: Wed, 11 Oct 2023 19:58:42 +0800 Subject: [PATCH 134/402] Allow CLI to pass environment variables to FOD builder (#8830) Add a new experimental `impure-env` setting that is a key-value list of environment variables to inject into FOD derivations that specify the corresponding `impureEnvVars`. This allows clients to make use of this feature (without having to change the environment of the daemon itself) and might eventually deprecate the current behaviour (pick whatever is in the environment of the daemon) as it's more principled and might prevent information leakage. --- .../src/language/advanced-attributes.md | 7 ++++ src/libstore/build/local-derivation-goal.cc | 14 ++++++-- src/libstore/globals.hh | 20 +++++++++++ src/libutil/experimental-features.cc | 9 ++++- src/libutil/experimental-features.hh | 1 + tests/functional/impure-env.nix | 16 +++++++++ tests/functional/impure-env.sh | 33 +++++++++++++++++++ tests/functional/local.mk | 3 +- 8 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 tests/functional/impure-env.nix create mode 100644 tests/functional/impure-env.sh diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7d97b24b6..4e4372331 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -112,6 +112,13 @@ Derivations can declare some infrequently used optional attributes. > environmental variables come from the environment of the > `nix-build`. + If the [`configurable-impure-env` experimental + feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + is enabled, these environment variables can also be controlled + through the + [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) + configuration setting. + - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ These attributes declare that the derivation is a so-called *fixed-output derivation*, which means that a cryptographic hash of diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64b55ca6a..817b20b2f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1135,8 +1135,18 @@ void LocalDerivationGoal::initEnv() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (!derivationType->isSandboxed()) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); + auto & impureEnv = settings.impureEnv.get(); + if (!impureEnv.empty()) + experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); + + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) { + auto envVar = impureEnv.find(i); + if (envVar != impureEnv.end()) { + env[i] = envVar->second; + } else { + env[i] = getEnv(i).value_or(""); + } + } } /* Currently structured log messages piggyback on stderr, but we diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0569f43b9..bd793c3d6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "config.hh" #include "util.hh" +#include "experimental-features.hh" #include #include @@ -1052,6 +1053,25 @@ public: ``` )" }; + + Setting impureEnv {this, {}, "impure-env", + R"( + A list of items, each in the format of: + + - `name=value`: Set environment variable `name` to `value`. + + If the user is trusted (see `trusted-users` option), when building + a fixed-output derivation, environment variables set in this option + will be passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md##adv-attr-impureEnvVars). + + This option is useful for, e.g., setting `https_proxy` for + fixed-output derivations and in a multi-user Nix installation, or + setting private access tokens when fetching a private repository. + )", + {}, // aliases + true, // document default + Xp::ConfigurableImpureEnv + }; }; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 203455b63..74af9aae0 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -221,6 +221,13 @@ constexpr std::array xpFeatureDetails = {{ Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, + { + .tag = Xp::ConfigurableImpureEnv, + .name = "configurable-impure-env", + .description = R"( + Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. + )", + } }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index add592ae6..e02f8353e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -31,6 +31,7 @@ enum struct ExperimentalFeature DynamicDerivations, ParseTomlTimestamps, ReadOnlyLocalStore, + ConfigurableImpureEnv, }; /** diff --git a/tests/functional/impure-env.nix b/tests/functional/impure-env.nix new file mode 100644 index 000000000..2b0380ed7 --- /dev/null +++ b/tests/functional/impure-env.nix @@ -0,0 +1,16 @@ +{ var, value }: + +with import ./config.nix; + +mkDerivation { + name = "test"; + buildCommand = '' + echo ${var} = "''$${var}" + echo -n "''$${var}" > "$out" + ''; + + impureEnvVars = [ var ]; + + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" value; +} diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh new file mode 100644 index 000000000..d9e4a34a2 --- /dev/null +++ b/tests/functional/impure-env.sh @@ -0,0 +1,33 @@ +source common.sh + +# Needs the config option 'impure-env' to work +requireDaemonNewerThan "2.18.0pre20230816" + +enableFeatures "configurable-impure-env" +restartDaemon + +varTest() { + local var="$1"; shift + local value="$1"; shift + nix build --no-link -vL --argstr var "$var" --argstr value "$value" --impure "$@" --file impure-env.nix + clearStore +} + +clearStore +startDaemon + +varTest env_name value --impure-env env_name=value + +echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf" +set_in_config=daemon_value restartDaemon + +varTest set_in_config config_value +varTest set_in_config client_value --impure-env set_in_config=client_value + +sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf" + +env_name=daemon_value restartDaemon + +varTest env_name daemon_value --impure-env env_name=client_value + +killDaemon diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 6f6c94fe6..643351163 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -121,7 +121,8 @@ nix_tests = \ path-from-hash-part.sh \ toString-path.sh \ read-only-store.sh \ - nested-sandboxing.sh + nested-sandboxing.sh \ + impure-env.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh From b4b1a07f9771d4fd106323672d833a1cb17c7364 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Fri, 13 Oct 2023 06:48:35 +0530 Subject: [PATCH 135/402] store info alias created --- src/nix/ping-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index ec450e8e0..5f8a10e22 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -47,3 +47,4 @@ struct CmdPingStore : StoreCommand, MixJSON }; static auto rCmdPingStore = registerCommand2({"store", "ping"}); +static auto rCmdInfoStore = registerCommand2({"store", "info"}); From 5c65379b228cee2c37408e1810c6654da5c91b90 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Fri, 13 Oct 2023 07:16:05 +0530 Subject: [PATCH 136/402] info store alias added to store-ping --- src/nix/ping-store.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 5f8a10e22..247d9c6bb 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -46,5 +46,15 @@ struct CmdPingStore : StoreCommand, MixJSON } }; +struct CmdInfoStore : CmdPingStore +{ + void run(nix::ref store) override + { + warn("'nix store info' is a deprecated alias for 'nix store ping'"); + CmdPingStore::run(store); + } +}; + + static auto rCmdPingStore = registerCommand2({"store", "ping"}); -static auto rCmdInfoStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "info"}); From db0d94560b2c72b695b420031ccdb8d1cbfb9f2b Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 21:36:56 +0200 Subject: [PATCH 137/402] Document builtins.fetchTree Co-authored-by: Valentin Gagarin Supersedes #6740 --- src/libexpr/primops/fetchTree.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 124165896..af4036460 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -236,6 +236,10 @@ static RegisterPrimOp primop_fetchTree({ ```nix builtins.fetchTree "https://example.com/" ``` + + > **Note** + > + > This function requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. )", .fun = prim_fetchTree, .experimentalFeature = Xp::Flakes, From 856fe1353367836d7d9282ae52e91a47e6bd4683 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Sep 2023 16:52:28 +0200 Subject: [PATCH 138/402] fetchTree cleanup Two changes: * The (probably unintentional) hack to handle paths as tarballs has been removed. This is almost certainly not what users expect and is inconsistent with flakeref handling everywhere else. * The hack to support scp-style Git URLs has been moved to the Git fetcher, so it's now supported not just by fetchTree but by flake inputs. --- src/libexpr/primops/fetchTree.cc | 55 ++++++++++---------------------- src/libfetchers/git.cc | 4 ++- src/libutil/url.cc | 17 ++++++++++ src/libutil/url.hh | 5 +++ src/nix/flake.md | 6 ++++ tests/functional/fetchGit.sh | 2 ++ tests/nixos/github-flakes.nix | 4 +++ 7 files changed, 54 insertions(+), 39 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index af4036460..920839058 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -25,7 +25,6 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -71,36 +70,10 @@ void emitTreeAttrs( v.mkAttrs(attrs); } -std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") -{ - state.checkURI(uri); - if (uri.find("://") == std::string::npos) { - const auto p = ParsedURL { - .scheme = defaultScheme, - .authority = "", - .path = uri - }; - return p.to_string(); - } else { - return uri; - } -} - -std::string fixURIForGit(std::string uri, EvalState & state) -{ - /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes - * them by removing the `:` and assuming a scheme of `ssh://` - * */ - static std::regex scp_uri("([^/]*)@(.*):(.*)"); - if (uri[0] != '/' && std::regex_match(uri, scp_uri)) - return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); - else - return fixURI(uri, state); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; + bool isFetchGit = false; }; static void fetchTree( @@ -108,11 +81,12 @@ static void fetchTree( const PosIdx pos, Value * * args, Value & v, - std::optional type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; NixStringContext context; + std::optional type; + if (params.isFetchGit) type = "git"; state.forceValue(*args[0], pos); @@ -142,10 +116,8 @@ static void fetchTree( if (attr.value->type() == nPath || attr.value->type() == nString) { auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); attrs.emplace(state.symbols[attr.name], - state.symbols[attr.name] == "url" - ? type == "git" - ? fixURIForGit(s, state) - : fixURI(s, state) + params.isFetchGit && state.symbols[attr.name] == "url" + ? fixGitURL(s) : s); } else if (attr.value->type() == nBool) @@ -170,13 +142,13 @@ static void fetchTree( "while evaluating the first argument passed to the fetcher", false, false).toOwned(); - if (type == "git") { + if (params.isFetchGit) { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", fixURIForGit(url, state)); + attrs.emplace("url", fixGitURL(url)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - input = fetchers::Input::fromURL(fixURI(url, state)); + input = fetchers::Input::fromURL(url); } } @@ -186,6 +158,8 @@ static void fetchTree( if (evalSettings.pureEval && !input.isLocked()) state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); + state.checkURI(input.toURLString()); + auto [tree, input2] = input.fetch(state.store); state.allowPath(tree.storePath); @@ -195,7 +169,7 @@ static void fetchTree( static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); + fetchTree(state, pos, args, v, { }); } static RegisterPrimOp primop_fetchTree({ @@ -392,7 +366,12 @@ static RegisterPrimOp primop_fetchTarball({ static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); + fetchTree(state, pos, args, v, + FetchTreeParams { + .emptyRevFallback = true, + .allowNameArgument = true, + .isFetchGit = true + }); } static RegisterPrimOp primop_fetchGit({ diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f8d89ab2f..7e9f34790 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -293,7 +293,6 @@ struct GitInputScheme : InputScheme if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") throw Error("unsupported Git input attribute '%s'", name); - parseURL(getStrAttr(attrs, "url")); maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "allRefs"); @@ -305,6 +304,9 @@ struct GitInputScheme : InputScheme Input input; input.attrs = attrs; + auto url = fixGitURL(getStrAttr(attrs, "url")); + parseURL(url); + input.attrs["url"] = url; return input; } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 2970f8d2d..9b438e6cd 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -158,4 +158,21 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme) }; } +std::string fixGitURL(const std::string & url) +{ + std::regex scpRegex("([^/]*)@(.*):(.*)"); + if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) + return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); + else { + if (url.find("://") == std::string::npos) { + return (ParsedURL { + .scheme = "file", + .authority = "", + .path = url + }).to_string(); + } else + return url; + } +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d2413ec0e..26c2dcc28 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -45,4 +45,9 @@ struct ParsedUrlScheme { ParsedUrlScheme parseUrlScheme(std::string_view scheme); +/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + them by removing the `:` and assuming a scheme of `ssh://`. Also + changes absolute paths into file:// URLs. */ +std::string fixGitURL(const std::string & url); + } diff --git a/src/nix/flake.md b/src/nix/flake.md index 939269c6a..f08648417 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -182,6 +182,12 @@ Currently the `type` attribute can be one of the following: git(+http|+https|+ssh|+git|+file|):(//)?(\?)? ``` + or + + ``` + @: + ``` + The `ref` attribute defaults to resolving the `HEAD` reference. The `rev` attribute must denote a commit that exists in the branch diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 418b4f63f..fc89f2040 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -35,6 +35,8 @@ unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") [[ $path0 = $path0_ ]] +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 6de702d17..62ae8871b 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -186,6 +186,10 @@ in client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + # Test fetchTree on a github URL. + hash = client.succeed(f"nix eval --raw --expr '(fetchTree {info['url']}).narHash'") + assert hash == info['locked']['narHash'] + # Shut down the web server. The flake should be cached on the client. github.succeed("systemctl stop httpd.service") From 4ce7a53a9ce4eb223f594fcbf627f968f5df02fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Sep 2023 17:08:26 +0200 Subject: [PATCH 139/402] Update fetchTree docs --- src/libexpr/primops/fetchTree.cc | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 920839058..1e2cee77b 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -177,12 +177,12 @@ static RegisterPrimOp primop_fetchTree({ .args = {"input"}, .doc = R"( Fetch a source tree or a plain file using one of the supported backends. - *input* can be an attribute set representation of [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) or a URL. - The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is allowed. + *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. + The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. Here are some examples of how to use `fetchTree`: - - Fetch a GitHub repository: + - Fetch a GitHub repository using the attribute set representation: ```nix builtins.fetchTree { @@ -193,7 +193,7 @@ static RegisterPrimOp primop_fetchTree({ } ``` - This evaluates to attribute set: + This evaluates to the following attribute set: ``` { @@ -205,15 +205,12 @@ static RegisterPrimOp primop_fetchTree({ shortRev = "ae2e6b3"; } ``` - - Fetch a single file from a URL: - ```nix - builtins.fetchTree "https://example.com/" + - Fetch the same GitHub repository using the URL-like syntax: + + ``` + builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" ``` - - > **Note** - > - > This function requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. )", .fun = prim_fetchTree, .experimentalFeature = Xp::Flakes, From 8eb4f735dc230fb8135b7e513cfc51eb66d937a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Sep 2023 10:48:27 +0200 Subject: [PATCH 140/402] fetchTree: Only use the registry if flakes are enabled --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 1e2cee77b..3d7a85047 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -152,7 +152,7 @@ static void fetchTree( } } - if (!evalSettings.pureEval && !input.isDirect()) + if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; if (evalSettings.pureEval && !input.isLocked()) From 4112dd1fc93c9ff03a5a4e8be773c45ebefbbd1f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Sep 2023 17:08:04 +0200 Subject: [PATCH 141/402] Mark fetchTree as stable --- doc/manual/src/release-notes/rl-next.md | 4 +++- src/libexpr/primops/fetchTree.cc | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 418c1187c..c905f445f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -6,4 +6,6 @@ - The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. -- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. \ No newline at end of file +- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- `builtins.fetchTree` is now marked as stable. diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3d7a85047..0335e5f36 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -213,7 +213,6 @@ static RegisterPrimOp primop_fetchTree({ ``` )", .fun = prim_fetchTree, - .experimentalFeature = Xp::Flakes, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, From a23cc147cb93fe9138b9a2f8283ad3ebbee43088 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 13 Oct 2023 11:00:55 -0400 Subject: [PATCH 142/402] Factor out Perl bindings Nix package Progress breaking up `flake.nix` by introducing separate `default.nix` files which make sense on their own. (This one is a regular `callPackage`-able package.) --- flake.nix | 43 ++++------------------------------------ perl/default.nix | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 39 deletions(-) create mode 100644 perl/default.nix diff --git a/flake.nix b/flake.nix index 301e65545..2263ea572 100644 --- a/flake.nix +++ b/flake.nix @@ -474,45 +474,10 @@ hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { - name = "nix-perl-${version}"; - - src = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - ./perl - ./.version - ./m4 - ./mk - ]); - }; - - nativeBuildInputs = - [ buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - pkgs.perl - boost - ] - ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium - ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - }); + passthru.perl-bindings = final.callPackage ./perl { + inherit fileset; + stdenv = currentStdenv; + }; meta.platforms = lib.platforms.unix; }); diff --git a/perl/default.nix b/perl/default.nix new file mode 100644 index 000000000..4687976a1 --- /dev/null +++ b/perl/default.nix @@ -0,0 +1,51 @@ +{ lib, fileset +, stdenv +, perl, perlPackages +, autoconf-archive, autoreconfHook, pkg-config +, nix, curl, bzip2, xz, boost, libsodium, darwin +}: + +perl.pkgs.toPerlModule (stdenv.mkDerivation { + name = "nix-perl-${nix.version}"; + + src = fileset.toSource { + root = ../.; + fileset = fileset.unions [ + ../.version + ../m4 + ../mk + ./MANIFEST + ./Makefile + ./Makefile.config.in + ./configure.ac + ./lib + ./local.mk + ]; + }; + + nativeBuildInputs = + [ autoconf-archive + autoreconfHook + pkg-config + ]; + + buildInputs = + [ nix + curl + bzip2 + xz + perl + boost + ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + + configureFlags = [ + "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" + "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" + ]; + + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; +}) From e5ce53f3db93ddbc9cca6d51d5fc0e2fd7335517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=A9clairevoyant?= <848000+eclairevoyant@users.noreply.github.com> Date: Sun, 15 Oct 2023 15:08:07 -0400 Subject: [PATCH 143/402] explicitly set meta.mainProgram --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 301e65545..05bb8fad4 100644 --- a/flake.nix +++ b/flake.nix @@ -515,6 +515,7 @@ }); meta.platforms = lib.platforms.unix; + meta.mainProgram = "nix"; }); lowdown-nix = with final; currentStdenv.mkDerivation rec { From c27d2f8da9643a0a87e36ae992b29ea00a50e0d9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 23:37:55 -0400 Subject: [PATCH 144/402] Add two more completions tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks @ncfavier for catching these regressions in my PR. Co-Authored-By: Naïm Favier --- tests/functional/completions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh index 19dc61098..7c1e4b287 100644 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -48,6 +48,8 @@ EOF [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]] ## Out of order [[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] From 483d99c622414504bb1b6a565f30ca6f21284315 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 10:42:15 -0400 Subject: [PATCH 145/402] Add API docs to some args-related functionality --- src/libcmd/command.hh | 78 ++++++++++++++++++++++++++++++++++++------- src/libutil/args.hh | 59 ++++++++++++++++++++++++++++++-- 2 files changed, 123 insertions(+), 14 deletions(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 96236b987..5c4569001 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -34,21 +34,28 @@ struct NixMultiCommand : virtual MultiCommand, virtual Command // For the overloaded run methods #pragma GCC diagnostic ignored "-Woverloaded-virtual" -/* A command that requires a Nix store. */ +/** + * A command that requires a \ref Store "Nix store". + */ struct StoreCommand : virtual Command { StoreCommand(); void run() override; ref getStore(); virtual ref createStore(); + /** + * Main entry point, with a `Store` provided + */ virtual void run(ref) = 0; private: std::shared_ptr _store; }; -/* A command that copies something between `--from` and `--to` - stores. */ +/** + * A command that copies something between `--from` and `--to` \ref + * Store stores. + */ struct CopyCommand : virtual StoreCommand { std::string srcUri, dstUri; @@ -60,6 +67,9 @@ struct CopyCommand : virtual StoreCommand ref getDstStore(); }; +/** + * A command that needs to evaluate Nix language expressions. + */ struct EvalCommand : virtual StoreCommand, MixEvalArgs { bool startReplOnEvalErrors = false; @@ -79,6 +89,10 @@ private: std::shared_ptr evalState; }; +/** + * A mixin class for commands that process flakes, adding a few standard + * flake-related options/flags. + */ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; @@ -87,6 +101,14 @@ struct MixFlakeOptions : virtual Args, EvalCommand MixFlakeOptions(); + /** + * The completion for some of these flags depends on the flake(s) in + * question. + * + * This method should be implemented to gather all flakerefs the + * command is operating with (presumably specified via some other + * arguments) so that the completions for these flags can use them. + */ virtual std::vector getFlakesForCompletion() { return {}; } @@ -112,15 +134,29 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions virtual Strings getDefaultFlakeAttrPathPrefixes(); + /** + * Complete an installable from the given prefix. + */ void completeInstallable(std::string_view prefix); }; +/** + * A mixin class for commands that need a read-only flag. + * + * What exactly is "read-only" is unspecified, but it will usually be + * the \ref Store "Nix store". + */ struct MixReadOnlyOption : virtual Args { MixReadOnlyOption(); }; -/* Like InstallablesCommand but the installables are not loaded */ +/** + * Like InstallablesCommand but the installables are not loaded. + * + * This is needed by `CmdRepl` which wants to load (and reload) the + * installables itself. + */ struct RawInstallablesCommand : virtual Args, SourceExprCommand { RawInstallablesCommand(); @@ -129,7 +165,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand void run(ref store) override; - // FIXME make const after CmdRepl's override is fixed up + // FIXME make const after `CmdRepl`'s override is fixed up virtual void applyDefaultInstallables(std::vector & rawInstallables); bool readFromStdIn = false; @@ -140,8 +176,11 @@ private: std::vector rawInstallables; }; -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ + +/** + * A command that operates on a list of "installables", which can be + * store paths, attribute paths, Nix expressions, etc. + */ struct InstallablesCommand : RawInstallablesCommand { virtual void run(ref store, Installables && installables) = 0; @@ -149,7 +188,9 @@ struct InstallablesCommand : RawInstallablesCommand void run(ref store, std::vector && rawInstallables) override; }; -/* A command that operates on exactly one "installable" */ +/** + * A command that operates on exactly one "installable". + */ struct InstallableCommand : virtual Args, SourceExprCommand { InstallableCommand(); @@ -175,7 +216,12 @@ struct MixOperateOnOptions : virtual Args MixOperateOnOptions(); }; -/* A command that operates on zero or more store paths. */ +/** + * A command that operates on zero or more extant store paths. + * + * If the argument the user passes is a some sort of recipe for a path + * not yet built, it must be built first. + */ struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -207,7 +253,9 @@ struct StorePathsCommand : public BuiltPathsCommand void run(ref store, BuiltPaths && paths) override; }; -/* A command that operates on exactly one store path. */ +/** + * A command that operates on exactly one store path. + */ struct StorePathCommand : public StorePathsCommand { virtual void run(ref store, const StorePath & storePath) = 0; @@ -215,7 +263,9 @@ struct StorePathCommand : public StorePathsCommand void run(ref store, StorePaths && storePaths) override; }; -/* A helper class for registering commands globally. */ +/** + * A helper class for registering \ref Command commands globally. + */ struct RegisterCommand { typedef std::map, std::function()>> Commands; @@ -271,7 +321,11 @@ struct MixEnvironment : virtual Args { MixEnvironment(); - /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + /*** + * Modify global environ based on `ignoreEnvironment`, `keep`, and + * `unset`. It's expected that exec will be called before this class + * goes out of scope, otherwise `environ` will become invalid. + */ void setEnviron(); }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 1e5bac650..6457cceed 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -39,8 +39,21 @@ public: protected: + /** + * The largest `size_t` is used to indicate the "any" arity, for + * handlers/flags/arguments that accept an arbitrary number of + * arguments. + */ static const size_t ArityAny = std::numeric_limits::max(); + /** + * Arguments (flags/options and positional) have a "handler" which is + * caused when the argument is parsed. The handler has an arbitrary side + * effect, including possible affect further command-line parsing. + * + * There are many constructors in order to support many shorthand + * initializations, and this is used a lot. + */ struct Handler { std::function)> fun; @@ -110,7 +123,12 @@ protected: { } }; - /* Options. */ + /** + * Description of flags / options + * + * These are arguments like `-s` or `--long` that can (mostly) + * appear in any order. + */ struct Flag { typedef std::shared_ptr ptr; @@ -130,12 +148,30 @@ protected: static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); }; + /** + * Index of all registered "long" flag descriptions (flags like + * `--long`). + */ std::map longFlags; + + /** + * Index of all registered "short" flag descriptions (flags like + * `-s`). + */ std::map shortFlags; + /** + * Process a single flag and its arguments, pulling from an iterator + * of raw CLI args as needed. + */ virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - /* Positional arguments. */ + /** + * Description of positional arguments + * + * These are arguments that do not start with a `-`, and for which + * the order does matter. + */ struct ExpectedArg { std::string label; @@ -144,8 +180,24 @@ protected: std::function completer; }; + /** + * Queue of expected positional argument forms. + * + * Positional arugment descriptions are inserted on the back. + * + * As positional arguments are passed, these are popped from the + * front, until there are hopefully none left as all args that were + * expected in fact were passed. + */ std::list expectedArgs; + /** + * Process some positional arugments + * + * @param finish: We have parsed everything else, and these are the only + * arguments left. Used because we accumulate some "pending args" we might + * have left over. + */ virtual bool processArgs(const Strings & args, bool finish); virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) @@ -204,6 +256,9 @@ public: friend class MultiCommand; + /** + * The parent command, used if this is a subcommand. + */ MultiCommand * parent = nullptr; private: From f7a36f981276a535962c1f5b0d02b00b9cd05298 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 12:52:59 -0400 Subject: [PATCH 146/402] Fix language tests a bit - Remove some stray saved error messages that didn't correspond to any test, because they were renamed in d11faa01b5b16773ba97bcb852b4123992829afc. - Need `--eval` in test failure test in order to get in "read-only" mode where we don't try to write to the store. (The other tests already do this.) - Need `--strict` so top-level attribute sets are still forced, like they are without `--eval`. --- tests/functional/lang.sh | 2 +- .../lang/eval-fail-antiquoted-path.err.exp | 1 - .../lang/eval-fail-bad-antiquote-1.err.exp | 10 ---------- .../lang/eval-fail-bad-antiquote-2.err.exp | 1 - .../lang/eval-fail-bad-antiquote-3.err.exp | 10 ---------- .../eval-fail-bad-string-interpolation-2.err.exp | 2 +- .../lang/eval-fail-dup-dynamic-attrs.err.exp | 12 +++++++++++- .../functional/lang/eval-fail-nonexist-path.err.exp | 2 +- tests/functional/lang/parse-fail-dup-attrs-6.err.exp | 1 - 9 files changed, 14 insertions(+), 27 deletions(-) delete mode 100644 tests/functional/lang/eval-fail-antiquoted-path.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-1.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-2.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-3.err.exp delete mode 100644 tests/functional/lang/parse-fail-dup-attrs-6.err.exp diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index f4760eced..c3acef5ee 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -68,7 +68,7 @@ for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename "$i" .nix) if - expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \ + expectStderr 1 nix-instantiate --eval --strict --show-trace "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then diffAndAccept "$i" err err.exp diff --git a/tests/functional/lang/eval-fail-antiquoted-path.err.exp b/tests/functional/lang/eval-fail-antiquoted-path.err.exp deleted file mode 100644 index 425deba42..000000000 --- a/tests/functional/lang/eval-fail-antiquoted-path.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory diff --git a/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp deleted file mode 100644 index cf94f53bc..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp +++ /dev/null @@ -1,10 +0,0 @@ -error: - … while evaluating a path segment - - at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2: - - 1| "${x: x}" - | ^ - 2| - - error: cannot coerce a function to a string diff --git a/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp deleted file mode 100644 index c8fe39d12..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp deleted file mode 100644 index fbefbc826..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp +++ /dev/null @@ -1,10 +0,0 @@ -error: - … while evaluating a path segment - - at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3: - - 1| ''${x: x}'' - | ^ - 2| - - error: cannot coerce a function to a string diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp index c8fe39d12..dea119ae8 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -1 +1 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' +error: getting status of '/pwd/lang/fnord': No such file or directory diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp index e01f8e6d0..c5fa67523 100644 --- a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -1,4 +1,14 @@ -error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 +error: + … while evaluating the attribute 'set' + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3: + + 1| { + 2| set = { "${"" + "b"}" = 1; }; + | ^ + 3| set = { "${"b" + ""}" = 2; }; + + error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp index c8fe39d12..dea119ae8 100644 --- a/tests/functional/lang/eval-fail-nonexist-path.err.exp +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -1 +1 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' +error: getting status of '/pwd/lang/fnord': No such file or directory diff --git a/tests/functional/lang/parse-fail-dup-attrs-6.err.exp b/tests/functional/lang/parse-fail-dup-attrs-6.err.exp deleted file mode 100644 index 74823fc25..000000000 --- a/tests/functional/lang/parse-fail-dup-attrs-6.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3 From b3fd7db63f78ec736238016745338277703ad1d5 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 16 Oct 2023 13:00:49 -0400 Subject: [PATCH 147/402] Detect cycles in flake follows. This change results in an error thrown as opposed to segfaulting due to stack overflow. Fixes #9144 --- src/libexpr/flake/lockfile.cc | 22 ++++++++++++++--- tests/functional/flakes/follow-paths.sh | 32 ++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 3c202967a..68443211b 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -45,16 +45,26 @@ StorePath LockedNode::computeStorePath(Store & store) const return lockedRef.input.computeStorePath(store); } -std::shared_ptr LockFile::findInput(const InputPath & path) -{ + +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; + auto pathS = printInputPath(path); + auto found = std::find(visited.cbegin(), visited.cend(), pathS); + + if(found != visited.end()) { + std::vector cycle(found, visited.cend()); + cycle.push_back(pathS); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(pathS); + for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = findInput(*follows)) + if (auto p = doFind(root, *follows, visited)) pos = ref(p); else return {}; @@ -66,6 +76,12 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return pos; } +std::shared_ptr LockFile::findInput(const InputPath & path) +{ + std::vector visited; + return doFind(root, path, visited); +} + LockFile::LockFile(const nlohmann::json & json, const Path & path) { auto version = json.value("version", 0); diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index dc97027ac..8573b5511 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -167,7 +167,7 @@ nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override # # The message was # error: input 'B/D' follows a non-existent input 'B/C/D' -# +# # Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. flakeFollowsOverloadA="$TEST_ROOT/follows/overload/flakeA" flakeFollowsOverloadB="$TEST_ROOT/follows/overload/flakeA/flakeB" @@ -230,3 +230,33 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \ nix flake metadata "$flakeFollowsOverloadA" nix flake update "$flakeFollowsOverloadA" nix flake lock "$flakeFollowsOverloadA" + +# Now test follow cycle detection +# We construct the following follows graph: +# +# foo +# / ^ +# / \ +# v \ +# bar -> baz +# The message was +# error: follow cycle detected: [baz -> foo -> bar -> baz] +flakeFollowCycle="$TEST_ROOT/follows/followCycle" + +# Test following path flakerefs. +mkdir -p "$flakeFollowCycle" + +cat > $flakeFollowCycle/flake.nix <&1 && fail "nix flake lock should have failed." || true) +echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" From add066cc7bdd1357ccf471b6ab6e68c470e6f604 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 19:32:47 +0100 Subject: [PATCH 148/402] Fix broken move --- src/libutil/config.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 38d406e8a..8672edaa8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -68,6 +68,7 @@ void AbstractConfig::warnUnknownSettings() void AbstractConfig::reapplyUnknownSettings() { auto unknownSettings2 = std::move(unknownSettings); + unknownSettings = {}; for (auto & s : unknownSettings2) set(s.first, s.second); } From d6066c90f87d9ba8ed1ddd8e2a3bd199f8498d6f Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 16 Oct 2023 15:47:28 -0400 Subject: [PATCH 149/402] Don't convert InputPaths to strings prematurely. --- src/libexpr/flake/lockfile.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 68443211b..f3ea9063f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -2,8 +2,10 @@ #include "store-api.hh" #include "url-parts.hh" +#include #include +#include #include namespace nix::flake { @@ -46,18 +48,18 @@ StorePath LockedNode::computeStorePath(Store & store) const } -static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; - auto pathS = printInputPath(path); - auto found = std::find(visited.cbegin(), visited.cend(), pathS); + auto found = std::find(visited.cbegin(), visited.cend(), path); if(found != visited.end()) { - std::vector cycle(found, visited.cend()); - cycle.push_back(pathS); + std::vector cycle; + std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); + cycle.push_back(printInputPath(path)); throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); } - visited.push_back(pathS); + visited.push_back(path); for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { @@ -78,7 +80,7 @@ static std::shared_ptr doFind(const ref& root, const InputPath & pat std::shared_ptr LockFile::findInput(const InputPath & path) { - std::vector visited; + std::vector visited; return doFind(root, path, visited); } From abf7df2b376a1cdbc00527bc662344fd9232ce05 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 21:47:33 +0100 Subject: [PATCH 150/402] Fix moves that accidentally copy anyway --- src/libexpr/eval.cc | 4 ++-- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/outputs-spec.cc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b18eb5266..5280614a1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -539,7 +539,7 @@ EvalState::EvalState( auto r = resolveSearchPathPath(i.path); if (!r) continue; - auto path = *std::move(r); + auto path = std::move(*r); if (store->isInStore(path)) { try { @@ -1035,7 +1035,7 @@ std::string EvalState::mkOutputStringRaw( /* In practice, this is testing for the case of CA derivations, or dynamic derivations. */ return optStaticOutputPath - ? store->printStorePath(*std::move(optStaticOutputPath)) + ? store->printStorePath(std::move(*optStaticOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index dc4d91079..360c6b70b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -561,7 +561,7 @@ void DerivationGoal::inputsRealised() attempt = fullDrv.tryResolve(worker.store); } assert(attempt); - Derivation drvResolved { *std::move(attempt) }; + Derivation drvResolved { std::move(*attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 817b20b2f..6a02f5ad4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2521,7 +2521,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - *std::move(optCA), + std::move(*optCA), Hash::dummy, }; if (*scratchPath != newInfo0.path) { diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index d943bc111..21c069223 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -63,7 +63,7 @@ std::optional> ExtendedOutputsS auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); if (!specOpt) return std::nullopt; - return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { std::move(*specOpt) } }; } From 54b350d517932ef11de4b3d638d67ffa5cec1634 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 21:48:02 +0100 Subject: [PATCH 151/402] Drop some moves that would happen anyway but forbid NRVO where appicable --- src/libexpr/eval.cc | 2 +- src/libexpr/paths.cc | 2 +- src/libstore/content-address.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5280614a1..511a5f434 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2290,7 +2290,7 @@ BackedStringView EvalState::coerceToString( && (!v2->isList() || v2->listSize() != 0)) result += " "; } - return std::move(result); + return result; } } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 1d690b722..b6a696f47 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -4,7 +4,7 @@ namespace nix { SourcePath EvalState::rootPath(CanonPath path) { - return std::move(path); + return path; } } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e290a8d38..ae91b859b 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -83,7 +83,7 @@ static std::pair parseContentAddressMethodPrefix if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); + return hashType; }; // Switch on prefix From 564922939404db110bcdfa71319470533df54cb4 Mon Sep 17 00:00:00 2001 From: Artturin Date: Mon, 11 Sep 2023 00:19:21 +0300 Subject: [PATCH 152/402] Bindmount files instead of hardlinking or copying to chroot https://github.com/nixos/nix/commit/16591eb3cccf86da8cd3f20c56e2dd847576ff5e#diff-19f999107b609d37cfb22c58e7f0bc1cf76edf1180e238dd6389e03cc279b604 (2013) added support for files to doBind This is work towards allowing users to change the location of chrootRootDir, to, for example, a tmpfs. inspired by trofi on matrix > It looks like build sandbox created by nix-daemon runs on the same filesystem, as /nix/store including things like /tmp which makes all small temporary files hit the disk. Is it intentional? If it is is there an easy way to redirect chroot's root to be tmpfs? dirsInChroot -> pathsInChroot --- src/libstore/build/local-derivation-goal.cc | 64 +++++++-------------- src/libstore/build/local-derivation-goal.hh | 4 +- 2 files changed, 23 insertions(+), 45 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64b55ca6a..a3ebfc681 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -581,7 +581,7 @@ void LocalDerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - dirsInChroot.clear(); + pathsInChroot.clear(); for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; @@ -592,19 +592,19 @@ void LocalDerivationGoal::startBuilder() } size_t p = i.find('='); if (p == std::string::npos) - dirsInChroot[i] = {i, optional}; + pathsInChroot[i] = {i, optional}; else - dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) { throw Error("`sandbox-build-dir` must not contain the storeDir"); } - dirsInChroot[tmpDirInSandbox] = tmpDir; + pathsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ StorePathSet closure; - for (auto & i : dirsInChroot) + for (auto & i : pathsInChroot) try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); @@ -615,7 +615,7 @@ void LocalDerivationGoal::startBuilder() } for (auto & i : closure) { auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); + pathsInChroot.insert_or_assign(p, p); } PathSet allowedPaths = settings.allowedImpureHostPrefixes; @@ -643,7 +643,7 @@ void LocalDerivationGoal::startBuilder() /* Allow files in __impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ - dirsInChroot[i] = {i, true}; + pathsInChroot[i] = {i, true}; } #if __linux__ @@ -711,15 +711,12 @@ void LocalDerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); + pathsInChroot.insert_or_assign(p, r); } /* If we're repairing, checking or rebuilding part of a multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot + rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputsAndOptPaths(worker.store)) { @@ -729,7 +726,7 @@ void LocalDerivationGoal::startBuilder() is already in the sandbox, so we don't need to worry about removing it. */ if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + pathsInChroot.erase(worker.store.printStorePath(*i.second.second)); } if (cgroup) { @@ -787,9 +784,9 @@ void LocalDerivationGoal::startBuilder() } else { auto p = line.find('='); if (p == std::string::npos) - dirsInChroot[line] = line; + pathsInChroot[line] = line; else - dirsInChroot[line.substr(0, p)] = line.substr(p + 1); + pathsInChroot[line.substr(0, p)] = line.substr(p + 1); } } } @@ -1779,7 +1776,7 @@ void LocalDerivationGoal::runChild() /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + if (pathsInChroot.find("/dev") == pathsInChroot.end()) { createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); @@ -1814,34 +1811,15 @@ void LocalDerivationGoal::runChild() ss.push_back(path); if (settings.caFile != "") - dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } - for (auto & i : ss) dirsInChroot.emplace(i, i); + for (auto & i : ss) pathsInChroot.emplace(i, i); /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility #if HAVE_EMBEDDED_SANDBOX_SHELL @@ -1882,7 +1860,7 @@ void LocalDerivationGoal::runChild() if /dev/ptx/ptmx exists). */ if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) + && !pathsInChroot.count("/dev/pts")) { if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { @@ -2017,7 +1995,7 @@ void LocalDerivationGoal::runChild() /* We build the ancestry before adding all inputPaths to the store because we know they'll all have the same parents (the store), and there might be lots of inputs. This isn't particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { Path cur = i.first; while (cur.compare("/") != 0) { cur = dirOf(cur); @@ -2025,7 +2003,7 @@ void LocalDerivationGoal::runChild() } } - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ Path cur = worker.store.storeDir; while (cur.compare("/") != 0) { @@ -2036,7 +2014,7 @@ void LocalDerivationGoal::runChild() /* Add all our input paths to the chroot */ for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; + pathsInChroot[p] = p; } /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2067,7 +2045,7 @@ void LocalDerivationGoal::runChild() without file-write* allowed, access() incorrectly returns EPERM */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.first != i.second.source) throw Error( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 0a05081c7..05c2c3a56 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -86,8 +86,8 @@ struct LocalDerivationGoal : public DerivationGoal : source(source), optional(optional) { } }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; + typedef map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; typedef map Environment; Environment env; From 630c2545d13cd8f6de4d678f19e89dfca375f62e Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:18:18 +0300 Subject: [PATCH 153/402] remove linkOrCopy and use bindmounts for files in addDependency --- src/libstore/build/local-derivation-goal.cc | 62 +++++++-------------- 1 file changed, 21 insertions(+), 41 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a3ebfc681..204acab11 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -387,26 +387,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() } -#if __linux__ -static void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - or fail with EXDEV in OpenAFS - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM && errno != EXDEV) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} -#endif - - void LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) @@ -1559,34 +1539,34 @@ void LocalDerivationGoal::addDependency(const StorePath & path) auto st = lstat(source); - if (S_ISDIR(st.st_mode)) { + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) - throw SysError("entering sandbox user namespace"); - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + if (S_ISDIR(st.st_mode)) createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError("bind mount from '%s' to '%s' failed", source, target); - _exit(0); - })); + _exit(0); + })); - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", From 11e47e7dfbca2f330b693665a4ee38302d1f6ad2 Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:36:25 +0300 Subject: [PATCH 154/402] factor out doBind from runChild --- src/libstore/build/local-derivation-goal.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 204acab11..9592df43a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -386,6 +386,26 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } +#if __linux__ +static void doBind(const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); +}; +#endif void LocalDerivationGoal::startBuilder() { From b8dfa3d53bda6793fe2c2566ae6e63e7c83718d8 Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:36:40 +0300 Subject: [PATCH 155/402] use doBind in addDependency --- src/libstore/build/local-derivation-goal.cc | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9592df43a..fd12b66b9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1552,13 +1552,12 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); if (pathExists(target)) + // There is a similar debug message in doBind, so only run it in this block to not have double messages. + debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - auto st = lstat(source); - /* Bind-mount the path into the sandbox. This requires entering its mount namespace, which is not possible in multithreaded programs. So we do this in a @@ -1571,15 +1570,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) if (setns(sandboxMountNamespace.get(), 0) == -1) throw SysError("entering sandbox mount namespace"); - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); + doBind(source, target); _exit(0); })); From dcc5f801f4fd88e08182b13304cdfa1b3aed8ece Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Tue, 17 Oct 2023 09:39:59 +0530 Subject: [PATCH 156/402] Store info command help updates --- src/nix/{ping-store.cc => store-info.cc} | 8 ++++---- src/nix/{ping-store.md => store-info.md} | 6 +++--- tests/functional/{store-ping.sh => store-info.sh} | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) rename src/nix/{ping-store.cc => store-info.cc} (91%) rename src/nix/{ping-store.md => store-info.md} (82%) rename tests/functional/{store-ping.sh => store-info.sh} (69%) diff --git a/src/nix/ping-store.cc b/src/nix/store-info.cc similarity index 91% rename from src/nix/ping-store.cc rename to src/nix/store-info.cc index 247d9c6bb..a7c595761 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/store-info.cc @@ -17,7 +17,7 @@ struct CmdPingStore : StoreCommand, MixJSON std::string doc() override { return - #include "ping-store.md" + #include "store-info.md" ; } @@ -50,11 +50,11 @@ struct CmdInfoStore : CmdPingStore { void run(nix::ref store) override { - warn("'nix store info' is a deprecated alias for 'nix store ping'"); + warn("'nix store ping' is a deprecated alias for 'nix store info'"); CmdPingStore::run(store); } }; -static auto rCmdPingStore = registerCommand2({"store", "ping"}); -static auto rCmdInfoStore = registerCommand2({"store", "info"}); +static auto rCmdPingStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/ping-store.md b/src/nix/store-info.md similarity index 82% rename from src/nix/ping-store.md rename to src/nix/store-info.md index 8c846791b..f86efd722 100644 --- a/src/nix/ping-store.md +++ b/src/nix/store-info.md @@ -5,19 +5,19 @@ R""( * Test whether connecting to a remote Nix store via SSH works: ```console - # nix store ping --store ssh://mac1 + # nix store info --store ssh://mac1 ``` * Test whether a URL is a valid binary cache: ```console - # nix store ping --store https://cache.nixos.org + # nix store info --store https://cache.nixos.org ``` * Test whether the Nix daemon is up and running: ```console - # nix store ping --store daemon + # nix store info --store daemon ``` # Description diff --git a/tests/functional/store-ping.sh b/tests/functional/store-info.sh similarity index 69% rename from tests/functional/store-ping.sh rename to tests/functional/store-info.sh index 9846c7d3d..c002e50be 100644 --- a/tests/functional/store-ping.sh +++ b/tests/functional/store-info.sh @@ -1,7 +1,7 @@ source common.sh -STORE_INFO=$(nix store ping 2>&1) -STORE_INFO_JSON=$(nix store ping --json) +STORE_INFO=$(nix store info 2>&1) +STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" @@ -11,7 +11,7 @@ if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ - fail "nix store ping on a non-existent store should fail" +expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ + fail "nix store info on a non-existent store should fail" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] From f62b5500ff96cc46e610da13dcf72c134935642d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Oct 2023 14:52:34 +0200 Subject: [PATCH 157/402] fetchTree: Require the flakes experimental feature for the URL syntax --- src/libexpr/primops/fetchTree.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 0335e5f36..3431ff013 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -148,6 +148,11 @@ static void fetchTree( attrs.emplace("url", fixGitURL(url)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"), + .errPos = state.positions[pos] + })); input = fetchers::Input::fromURL(url); } } @@ -180,6 +185,10 @@ static RegisterPrimOp primop_fetchTree({ *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. + > **Note** + > + > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + Here are some examples of how to use `fetchTree`: - Fetch a GitHub repository using the attribute set representation: From 3470cd68c46334d9d2c55e890f7fe50fa9ebe35c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Oct 2023 14:57:29 +0200 Subject: [PATCH 158/402] Mark some fetchers as experimental --- src/libfetchers/github.cc | 5 +++++ src/libfetchers/path.cc | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 291f457f0..6c5c792ec 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -223,6 +223,11 @@ struct GitArchiveInputScheme : InputScheme return {result.tree.storePath, input}; } + + std::optional experimentalFeature() override + { + return Xp::Flakes; + } }; struct GitHubInputScheme : GitArchiveInputScheme diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 01f1be978..092975f5d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -125,6 +125,11 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } + + std::optional experimentalFeature() override + { + return Xp::Flakes; + } }; static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From ff684260950cd31efffbe9804494d42bc226df59 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 17 Oct 2023 11:15:36 -0400 Subject: [PATCH 159/402] Name the protocol version types This makes the code clearer, and will help us replace them with proper structs and get rid of the macros later. --- src/libstore/daemon.cc | 10 +++++----- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/remote-store-connection.hh | 2 +- src/libstore/serve-protocol.hh | 7 +++++++ src/libstore/worker-protocol.hh | 7 +++++++ src/nix-store/nix-store.cc | 2 +- 6 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044..fbff64e89 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -45,9 +45,9 @@ struct TunnelLogger : public Logger Sync state_; - unsigned int clientVersion; + WorkerProto::Version clientVersion; - TunnelLogger(FdSink & to, unsigned int clientVersion) + TunnelLogger(FdSink & to, WorkerProto::Version clientVersion) : to(to), clientVersion(clientVersion) { } void enqueueMsg(const std::string & s) @@ -261,7 +261,7 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn) +static std::vector readDerivedPaths(Store & store, WorkerProto::Version clientVersion, WorkerProto::ReadConn conn) { std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { @@ -274,7 +274,7 @@ static std::vector readDerivedPaths(Store & store, unsigned int cli } static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, + TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { WorkerProto::ReadConn rconn { .from = from }; @@ -1017,7 +1017,7 @@ void processConnection( if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); to << WORKER_MAGIC_2 << PROTOCOL_VERSION; to.flush(); - unsigned int clientVersion = readInt(from); + WorkerProto::Version clientVersion = readInt(from); if (clientVersion < 0x10a) throw Error("the Nix client version is too old"); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 703ded0b2..efced20d0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -45,7 +45,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor std::unique_ptr sshConn; FdSink to; FdSource from; - int remoteVersion; + ServeProto::Version remoteVersion; bool good = true; /** diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index ce4740a9c..8738e4d95 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -30,7 +30,7 @@ struct RemoteStore::Connection * sides support. (If the maximum doesn't exist, we would fail to * establish a connection and produce a value of this type.) */ - unsigned int daemonVersion; + WorkerProto::Version daemonVersion; /** * Whether the remote side trusts us or not. diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index e2345d450..dddb7c54d 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -30,6 +30,13 @@ struct ServeProto */ enum struct Command : uint64_t; + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c84060103..e9fc8d957 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -47,6 +47,13 @@ struct WorkerProto */ enum struct Op : uint64_t; + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9b6c80a75..ea53e49b0 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -818,7 +818,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; out.flush(); - unsigned int clientVersion = readInt(in); + ServeProto::Version clientVersion = readInt(in); ServeProto::ReadConn rconn { .from = in }; ServeProto::WriteConn wconn { .to = out }; From e36c9175f49c0fac4f0511d4cae5ab991fd9cb7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 04:40:49 +0000 Subject: [PATCH 160/402] Add protocol versions to `{Worker,Serve}Proto::*Conn` This will allow us to factor out logic, which is currently scattered inline, into several reusable instances The tests are also updated to support versioning. Currently all Worker and Serve protocol tests are using the minimum version, since no version-specific serialisers have been created yet. But in subsequent commits when that changes, we will test individual versions to ensure complete coverage. --- src/libstore/daemon.cc | 15 ++++- src/libstore/legacy-ssh-store.cc | 2 + src/libstore/path-info.cc | 10 +++- src/libstore/remote-store-connection.hh | 2 + src/libstore/serve-protocol.hh | 8 +-- src/libstore/tests/common-protocol.cc | 73 +++++++++++++++++++++---- src/libstore/tests/protocol.hh | 45 ++++++++------- src/libstore/tests/serve-protocol.cc | 38 +++++++++---- src/libstore/tests/worker-protocol.cc | 49 ++++++++++++----- src/libstore/worker-protocol.hh | 8 +-- src/nix-store/nix-store.cc | 10 +++- 11 files changed, 185 insertions(+), 75 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fbff64e89..d61e97a64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -277,8 +277,14 @@ static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { - WorkerProto::ReadConn rconn { .from = from }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::ReadConn rconn { + .from = from, + .version = clientVersion, + }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; switch (op) { @@ -1052,7 +1058,10 @@ void processConnection( auto temp = trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; WorkerProto::write(*store, wconn, temp); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index efced20d0..9143be72e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -60,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { return ServeProto::ReadConn { .from = from, + .version = remoteVersion, }; } @@ -75,6 +76,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { return ServeProto::WriteConn { .to = to, + .version = remoteVersion, }; } }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f..a9231ce8d 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -141,7 +141,10 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = source }); + WorkerProto::ReadConn { + .from = source, + .version = format, + }); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -163,7 +166,10 @@ void ValidPathInfo::write( sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); WorkerProto::write(store, - WorkerProto::WriteConn { .to = sink }, + WorkerProto::WriteConn { + .to = sink, + .version = format, + }, references); sink << registrationTime << narSize; if (format >= 16) { diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index 8738e4d95..e4a9cacb9 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -70,6 +70,7 @@ struct RemoteStore::Connection { return WorkerProto::ReadConn { .from = from, + .version = daemonVersion, }; } @@ -85,6 +86,7 @@ struct RemoteStore::Connection { return WorkerProto::WriteConn { .to = to, + .version = daemonVersion, }; } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index dddb7c54d..a627c6ad6 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -40,23 +40,19 @@ struct ServeProto /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. - * - * This currently is just a `Source &`, but more fields will be added - * later. */ struct ReadConn { Source & from; + Version version; }; /** * A unidirectional write connection, to be used by the write half of the * canonical serializers below. - * - * This currently is just a `Sink &`, but more fields will be added - * later. */ struct WriteConn { Sink & to; + Version version; }; /** diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc index de57211f0..61c2cb70c 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/src/libstore/tests/common-protocol.cc @@ -13,10 +13,71 @@ namespace nix { const char commonProtoDir[] = "common-protocol"; -using CommonProtoTest = ProtoTest; +class CommonProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + CommonProto::Serialise::read( + *store, + CommonProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + CommonProto::write( + *store, + CommonProto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } CHARACTERIZATION_TEST( - CommonProtoTest, string, "string", (std::tuple { @@ -28,7 +89,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, storePath, "store-path", (std::tuple { @@ -37,7 +97,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, contentAddress, "content-address", (std::tuple { @@ -56,7 +115,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, drvOutput, "drv-output", (std::tuple { @@ -71,7 +129,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, realisation, "realisation", (std::tuple { @@ -103,7 +160,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, vector, "vector", (std::tuple, std::vector, std::vector, std::vector>> { @@ -114,7 +170,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, set, "set", (std::tuple, std::set, std::set, std::set>> { @@ -125,7 +180,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, optionalStorePath, "optional-store-path", (std::tuple, std::optional> { @@ -136,7 +190,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, optionalContentAddress, "optional-content-address", (std::tuple, std::optional> { diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 86a900757..496915745 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -9,26 +9,23 @@ namespace nix { template class ProtoTest : public LibStoreTest { - /** - * Read this as simply `using S = Inner::Serialise;`. - * - * See `LengthPrefixedProtoHelper::S` for the same trick, and its - * rationale. - */ - template using S = typename Proto::template Serialise; - -public: +protected: Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; Path goldenMaster(std::string_view testStem) { return unitTestData + "/" + testStem + ".bin"; } +}; +template +class VersionedProtoTest : public ProtoTest +{ +public: /** * Golden test for `T` reading */ template - void readTest(PathView testStem, T value) + void readTest(PathView testStem, typename Proto::Version version, T value) { if (testAccept()) { @@ -36,13 +33,16 @@ public: } else { - auto expected = readFile(goldenMaster(testStem)); + auto expected = readFile(ProtoTest::goldenMaster(testStem)); T got = ({ StringSource from { expected }; - S::read( - *store, - typename Proto::ReadConn { .from = from }); + Proto::template Serialise::read( + *LibStoreTest::store, + typename Proto::ReadConn { + .from = from, + .version = version, + }); }); ASSERT_EQ(got, value); @@ -53,14 +53,17 @@ public: * Golden test for `T` write */ template - void writeTest(PathView testStem, const T & value) + void writeTest(PathView testStem, typename Proto::Version version, const T & value) { - auto file = goldenMaster(testStem); + auto file = ProtoTest::goldenMaster(testStem); StringSink to; Proto::write( - *store, - typename Proto::WriteConn { .to = to }, + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, value); if (testAccept()) @@ -77,12 +80,12 @@ public: } }; -#define CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VALUE) \ +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ TEST_F(FIXTURE, NAME ## _read) { \ - readTest(STEM, VALUE); \ + readTest(STEM, VERSION, VALUE); \ } \ TEST_F(FIXTURE, NAME ## _write) { \ - writeTest(STEM, VALUE); \ + writeTest(STEM, VERSION, VALUE); \ } } diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc index 5d7df56cf..ba798ab1c 100644 --- a/src/libstore/tests/serve-protocol.cc +++ b/src/libstore/tests/serve-protocol.cc @@ -11,14 +11,22 @@ namespace nix { -const char commonProtoDir[] = "serve-protocol"; +const char serveProtoDir[] = "serve-protocol"; -using ServeProtoTest = ProtoTest; +struct ServeProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + ServeProto::Version defaultVersion = 1 << 8 | 0; +}; -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, string, "string", + defaultVersion, (std::tuple { "", "hi", @@ -27,19 +35,21 @@ CHARACTERIZATION_TEST( "oh no \0\0\0 what was that!", })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, storePath, "store-path", + defaultVersion, (std::tuple { StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, contentAddress, "content-address", + defaultVersion, (std::tuple { ContentAddress { .method = TextIngestionMethod {}, @@ -55,10 +65,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, drvOutput, "drv-output", + defaultVersion, (std::tuple { { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -70,10 +81,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, realisation, "realisation", + defaultVersion, (std::tuple { Realisation { .id = DrvOutput { @@ -102,10 +114,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, "vector", + defaultVersion, (std::tuple, std::vector, std::vector, std::vector>> { { }, { "" }, @@ -113,10 +126,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, set, "set", + defaultVersion, (std::tuple, std::set, std::set, std::set>> { { }, { "" }, @@ -124,10 +138,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, optionalStorePath, "optional-store-path", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { @@ -135,10 +150,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, optionalContentAddress, "optional-content-address", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 59d7ff96d..63fecce96 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -14,12 +14,21 @@ namespace nix { const char workerProtoDir[] = "worker-protocol"; -using WorkerProtoTest = ProtoTest; +struct WorkerProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + WorkerProto::Version defaultVersion = 1 << 8 | 0; +}; -CHARACTERIZATION_TEST( + +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, string, "string", + defaultVersion, (std::tuple { "", "hi", @@ -28,19 +37,21 @@ CHARACTERIZATION_TEST( "oh no \0\0\0 what was that!", })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, storePath, "store-path", + defaultVersion, (std::tuple { StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, contentAddress, "content-address", + defaultVersion, (std::tuple { ContentAddress { .method = TextIngestionMethod {}, @@ -56,10 +67,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, derivedPath, "derived-path", + defaultVersion, (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, @@ -72,10 +84,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, "drv-output", + defaultVersion, (std::tuple { { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -87,10 +100,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, realisation, "realisation", + defaultVersion, (std::tuple { Realisation { .id = DrvOutput { @@ -119,10 +133,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult, "build-result", + defaultVersion, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -177,10 +192,11 @@ CHARACTERIZATION_TEST( t; })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, keyedBuildResult, "keyed-build-result", + defaultVersion, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -213,20 +229,22 @@ CHARACTERIZATION_TEST( t; })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalTrustedFlag, "optional-trusted-flag", + defaultVersion, (std::tuple, std::optional, std::optional> { std::nullopt, std::optional { Trusted }, std::optional { NotTrusted }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, vector, "vector", + defaultVersion, (std::tuple, std::vector, std::vector, std::vector>> { { }, { "" }, @@ -234,10 +252,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, set, "set", + defaultVersion, (std::tuple, std::set, std::set, std::set>> { { }, { "" }, @@ -245,10 +264,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalStorePath, "optional-store-path", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { @@ -256,10 +276,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalContentAddress, "optional-content-address", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index e9fc8d957..2d55d926a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -57,23 +57,19 @@ struct WorkerProto /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. - * - * This currently is just a `Source &`, but more fields will be added - * later. */ struct ReadConn { Source & from; + Version version; }; /** * A unidirectional write connection, to be used by the write half of the * canonical serializers below. - * - * This currently is just a `Sink &`, but more fields will be added - * later. */ struct WriteConn { Sink & to; + Version version; }; /** diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ea53e49b0..11d2e86e3 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -820,8 +820,14 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); ServeProto::Version clientVersion = readInt(in); - ServeProto::ReadConn rconn { .from = in }; - ServeProto::WriteConn wconn { .to = out }; + ServeProto::ReadConn rconn { + .from = in, + .version = clientVersion, + }; + ServeProto::WriteConn wconn { + .to = out, + .version = clientVersion, + }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're From a0f071f1d3b05140fa1c97de5a7612d1b938ffbf Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Wed, 18 Oct 2023 00:12:10 +0530 Subject: [PATCH 161/402] store info sh renamed --- tests/functional/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 643351163..3679349f8 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -113,7 +113,7 @@ nix_tests = \ pass-as-file.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh \ + store-info.sh \ fetchClosure.sh \ completions.sh \ flakes/show.sh \ From 891dfb435943f68f165a0c5921f0af8e9e7362e8 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Wed, 18 Oct 2023 00:14:11 +0530 Subject: [PATCH 162/402] updated store ping to store info in files --- doc/manual/src/advanced-topics/distributed-builds.md | 4 ++-- doc/manual/src/release-notes/rl-2.15.md | 2 +- doc/manual/src/release-notes/rl-2.7.md | 2 +- tests/functional/legacy-ssh-store.sh | 4 ++-- tests/functional/local-store.sh | 4 ++-- tests/functional/nix-copy-ssh-ng.sh | 2 +- tests/functional/remote-store.sh | 8 ++++---- tests/installer/default.nix | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 73a113d35..507c5ecb7 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix store ping --store ssh://mac +$ nix store info --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..4faf0b143 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -44,7 +44,7 @@ (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. - `nix store ping` and `nix doctor` now display this information. + `nix store info` and `nix doctor` now display this information. * The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md index 2f3879422..dd649e166 100644 --- a/doc/manual/src/release-notes/rl-2.7.md +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -24,7 +24,7 @@ [repository](https://github.com/NixOS/bundlers) has various bundlers implemented. -* `nix store ping` now reports the version of the remote Nix daemon. +* `nix store info` now reports the version of the remote Nix daemon. * `nix flake {init,new}` now display information about which files have been created. diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh index 71b716b84..894efccd4 100644 --- a/tests/functional/legacy-ssh-store.sh +++ b/tests/functional/legacy-ssh-store.sh @@ -1,4 +1,4 @@ source common.sh -# Check that store ping trusted doesn't yet work with ssh:// -nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' +# Check that store info trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e 'has("trusted") | not' diff --git a/tests/functional/local-store.sh b/tests/functional/local-store.sh index 89502f864..f7c8eb3f1 100644 --- a/tests/functional/local-store.sh +++ b/tests/functional/local-store.sh @@ -18,5 +18,5 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] -# Ensure store ping trusted works with local store -nix --store ./x store ping --json | jq -e '.trusted' +# Ensure store info trusted works with local store +nix --store ./x store info --json | jq -e '.trusted' diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 45e53c9c0..463b5e0c4 100644 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -9,7 +9,7 @@ rm -rf "$remoteRoot" outPath=$(nix-build --no-out-link dependencies.nix) -nix store ping --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" +nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" # Regression test for https://github.com/NixOS/nix/issues/6253 nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 50e6f24b9..5c7bfde46 100644 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -5,17 +5,17 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor -# Ensure that store ping trusted works with ssh-ng:// -nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' +# Ensure that store info trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e '.trusted' startDaemon if isDaemonNewer "2.15pre0"; then # Ensure that ping works trusted with new daemon - nix store ping --json | jq -e '.trusted' + nix store info --json | jq -e '.trusted' else # And the the field is absent with the old daemon - nix store ping --json | jq -e 'has("trusted") | not' + nix store info --json | jq -e 'has("trusted") | not' fi # Test import-from-derivation through the daemon. diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 49cfd2bcc..238c6ac8e 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -213,7 +213,7 @@ let source /etc/bashrc || true nix-env --version - nix --extra-experimental-features nix-command store ping + nix --extra-experimental-features nix-command store info out=\$(nix-build --no-substitute -E 'derivation { name = "foo"; system = "x86_64-linux"; builder = "/bin/sh"; args = ["-c" "echo foobar > \$out"]; }') [[ \$(cat \$out) = foobar ]] From ea38605d116e1c8db55a8a8cb7400724b931458d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Oct 2023 15:32:31 +0200 Subject: [PATCH 163/402] Introduce FSInputAccessor and use it Backported from the lazy-trees branch. Note that this doesn't yet use the access control features of FSInputAccessor. --- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval.cc | 35 +++-- src/libexpr/eval.hh | 3 + src/libexpr/flake/flake.cc | 6 +- src/libexpr/nixexpr.hh | 7 +- src/libexpr/parser.y | 9 +- src/libexpr/paths.cc | 3 +- src/libexpr/primops.cc | 18 +-- src/libexpr/tests/json.cc | 2 +- src/libexpr/tests/libexpr.hh | 15 ++- src/libexpr/value.hh | 17 ++- src/libfetchers/fs-input-accessor.cc | 141 ++++++++++++++++++++ src/libfetchers/fs-input-accessor.hh | 33 +++++ src/libfetchers/input-accessor.cc | 189 ++++++++++++++++++++------- src/libfetchers/input-accessor.hh | 105 ++++++++++++--- src/libstore/store-api.cc | 14 +- src/libstore/store-api.hh | 15 ++- src/nix/main.cc | 14 +- 18 files changed, 502 insertions(+), 126 deletions(-) create mode 100644 src/libfetchers/fs-input-accessor.cc create mode 100644 src/libfetchers/fs-input-accessor.hh diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index ab654c1b0..d12345710 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -132,7 +132,7 @@ std::pair findPackageFilename(EvalState & state, Value & v if (colon == std::string::npos) fail(); std::string filename(fn, 0, colon); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); - return {CanonPath(fn.substr(0, colon)), lineno}; + return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); abort(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 511a5f434..87791fa52 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -12,6 +12,7 @@ #include "function-trace.hh" #include "profiles.hh" #include "print.hh" +#include "fs-input-accessor.hh" #include #include @@ -503,6 +504,18 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) + , rootFS( + makeFSInputAccessor( + CanonPath::root, + evalSettings.restrictEval || evalSettings.pureEval + ? std::optional>(std::set()) + : std::nullopt, + [](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = evalSettings.pureEval + ? "in pure evaluation mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); + })) , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) , store(store) , buildStore(buildStore ? buildStore : store) @@ -518,6 +531,8 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { + rootFS->allowPath(CanonPath::root); // FIXME + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); @@ -599,7 +614,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) */ Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath); + if (hasPrefix(abspath, corepkgsPrefix)) return rootPath(CanonPath(abspath)); for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { @@ -617,7 +632,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) /* Resolve symlinks. */ debug("checking access to '%s'", abspath); - SourcePath path = CanonPath(canonPath(abspath, true)); + SourcePath path = rootPath(CanonPath(canonPath(abspath, true))); for (auto & i : *allowedPaths) { if (isDirOrInDir(path.path.abs(), i)) { @@ -649,12 +664,12 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(CanonPath(uri)); + checkSourcePath(rootPath(CanonPath(uri))); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(CanonPath(std::string(uri, 7))); + checkSourcePath(rootPath(CanonPath(std::string(uri, 7)))); return; } @@ -950,7 +965,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context) void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(path.path.abs())); + mkPath(&*path.accessor, makeImmutableString(path.path.abs())); } @@ -2037,7 +2052,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nPath) { if (!context.empty()) state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); - v.mkPath(CanonPath(canonPath(str()))); + v.mkPath(state.rootPath(CanonPath(canonPath(str())))); } else v.mkStringMove(c_str(), context); } @@ -2236,7 +2251,7 @@ BackedStringView EvalState::coerceToString( !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. - v._path + v._path.path : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) : std::string(v.path().path.abs()); @@ -2329,7 +2344,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return CanonPath(path); + return rootPath(CanonPath(path)); } @@ -2429,7 +2444,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.string_view().compare(v2.string_view()) == 0; case nPath: - return strcmp(v1._path, v2._path) == 0; + return + v1._path.accessor == v2._path.accessor + && strcmp(v1._path.path, v2._path.path) == 0; case nNull: return true; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index c95558952..4bb0a6a0b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -24,6 +24,7 @@ class EvalState; class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; +struct FSInputAccessor; /** @@ -211,6 +212,8 @@ public: Bindings emptyBindings; + const ref rootFS; + const SourcePath derivationInternal; /** diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index a6212c12f..2f91ad433 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -223,9 +223,9 @@ static Flake getFlake( throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); Value vInfo; - state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack + state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); + expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); @@ -741,7 +741,7 @@ void callFlake(EvalState & state, state.vCallFlake = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "call-flake.nix.gen.hh" - , CanonPath::root), **state.vCallFlake); + , state.rootPath(CanonPath::root)), **state.vCallFlake); } state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e766996c8..10099d49e 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -20,7 +20,6 @@ MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); -MakeError(RestrictedPathError, Error); /** * Position objects. @@ -200,9 +199,13 @@ struct ExprString : Expr struct ExprPath : Expr { + ref accessor; std::string s; Value v; - ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; + ExprPath(ref accessor, std::string s) : accessor(accessor), s(std::move(s)) + { + v.mkPath(&*accessor, this->s.c_str()); + } Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 70228e1e2..78436c6a2 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,6 +64,7 @@ struct StringToken { #include "parser-tab.hh" #include "lexer-tab.hh" +#include "fs-input-accessor.hh" YY_DECL; @@ -520,7 +521,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(std::move(path)); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -530,7 +531,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(std::move(path)); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } ; @@ -756,11 +757,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ auto r = *rOpt; Path res = suffix == "" ? r : concatStrings(r, "/", suffix); - if (pathExists(res)) return CanonPath(canonPath(res)); + if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); } if (hasPrefix(path, "nix/")) - return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); + return rootPath(CanonPath(concatStrings(corepkgsPrefix, path.substr(4)))); debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index b6a696f47..099607638 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,10 +1,11 @@ #include "eval.hh" +#include "fs-input-accessor.hh" namespace nix { SourcePath EvalState::rootPath(CanonPath path) { - return path; + return {rootFS, std::move(path)}; } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3bed7c5aa..dd3fa12ee 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -202,7 +202,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , CanonPath::root), **state.vImportedDrvToDerivation); + , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation); } state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); @@ -213,7 +213,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v else if (path2 == corepkgsPrefix + "fetchurl.nix") { state.eval(state.parseExprFromString( #include "fetchurl.nix.gen.hh" - , CanonPath::root), v); + , state.rootPath(CanonPath::root)), v); } else { @@ -599,7 +599,8 @@ struct CompareValues case nString: return v1->string_view().compare(v2->string_view()) < 0; case nPath: - return strcmp(v1->_path, v2->_path) < 0; + // FIXME: handle accessor? + return strcmp(v1->_path.path, v2->_path.path) < 0; case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -2203,7 +2204,7 @@ static void addPath( path = evalSettings.pureEval && expectedHash ? path - : state.checkSourcePath(CanonPath(path)).path.abs(); + : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs(); PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); @@ -2236,9 +2237,10 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - StorePath dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); + // FIXME + if (method != FileIngestionMethod::Recursive) + throw Error("'recursive = false' is not implemented"); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); @@ -4447,7 +4449,7 @@ void EvalState::createBaseEnv() // the parser needs two NUL bytes as terminators; one of them // is implied by being a C string. "\0"; - eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation); + eval(parse(code, sizeof(code), derivationInternal, rootPath(CanonPath::root), staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index 7586bdd9b..f4cc118d6 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -62,7 +62,7 @@ namespace nix { // not supported by store 'dummy'" thrown in the test body. TEST_F(JSONValueTest, DISABLED_Path) { Value v; - v.mkPath("test"); + v.mkPath(state.rootPath(CanonPath("/test"))); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); } } /* namespace nix */ diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index 97b05ac46..968431446 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -103,14 +103,17 @@ namespace nix { } MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) { - if (arg.type() != nPath) { - *result_listener << "Expected a path got " << arg.type(); - return false; - } else if (std::string_view(arg._path) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str(); + if (arg.type() != nPath) { + *result_listener << "Expected a path got " << arg.type(); + return false; + } else { + auto path = arg.path(); + if (path.path != CanonPath(p)) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path; return false; } - return true; + } + return true; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 7a1165cac..622e613ea 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -189,7 +189,12 @@ public: const char * c_str; const char * * context; // must be in sorted order } string; - const char * _path; + + struct { + InputAccessor * accessor; + const char * path; + } _path; + Bindings * attrs; struct { size_t size; @@ -286,11 +291,12 @@ public: void mkPath(const SourcePath & path); - inline void mkPath(const char * path) + inline void mkPath(InputAccessor * accessor, const char * path) { clearValue(); internalType = tPath; - _path = path; + _path.accessor = accessor; + _path.path = path; } inline void mkNull() @@ -437,7 +443,10 @@ public: SourcePath path() const { assert(internalType == tPath); - return SourcePath{CanonPath(_path)}; + return SourcePath { + .accessor = ref(_path.accessor->shared_from_this()), + .path = CanonPath(CanonPath::unchecked_t(), _path.path) + }; } std::string_view string_view() const diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc new file mode 100644 index 000000000..a35955465 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.cc @@ -0,0 +1,141 @@ +#include "fs-input-accessor.hh" +#include "store-api.hh" + +namespace nix { + +struct FSInputAccessorImpl : FSInputAccessor +{ + CanonPath root; + std::optional> allowedPaths; + MakeNotAllowedError makeNotAllowedError; + + FSInputAccessorImpl( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) + : root(root) + , allowedPaths(std::move(allowedPaths)) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { + } + + std::string readFile(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return nix::readFile(absPath.abs()); + } + + bool pathExists(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + return isAllowed(absPath) && nix::pathExists(absPath.abs()); + } + + Stat lstat(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + auto st = nix::lstat(absPath.abs()); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + DirEntries res; + for (auto & entry : nix::readDirectory(absPath.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + if (isAllowed(absPath + entry.name)) + res.emplace(entry.name, type); + } + return res; + } + + std::string readLink(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return nix::readLink(absPath.abs()); + } + + CanonPath makeAbsPath(const CanonPath & path) + { + return root + path; + } + + void checkAllowed(const CanonPath & absPath) override + { + if (!isAllowed(absPath)) + throw makeNotAllowedError + ? makeNotAllowedError(absPath) + : RestrictedPathError("access to path '%s' is forbidden", absPath); + } + + bool isAllowed(const CanonPath & absPath) + { + if (!absPath.isWithin(root)) + return false; + + if (allowedPaths) { + auto p = absPath.removePrefix(root); + if (!p.isAllowed(*allowedPaths)) + return false; + } + + return true; + } + + void allowPath(CanonPath path) override + { + if (allowedPaths) + allowedPaths->insert(std::move(path)); + } + + bool hasAccessControl() override + { + return (bool) allowedPaths; + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return makeAbsPath(path); + } +}; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(root, std::move(allowedPaths), std::move(makeNotAllowedError)); +} + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError) +{ + return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError)); +} + +SourcePath getUnfilteredRootPath(CanonPath path) +{ + static auto rootFS = makeFSInputAccessor(CanonPath::root); + return {rootFS, path}; +} + +} diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh new file mode 100644 index 000000000..19a5211c8 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +class StorePath; +class Store; + +struct FSInputAccessor : InputAccessor +{ + virtual void checkAllowed(const CanonPath & absPath) = 0; + + virtual void allowPath(CanonPath path) = 0; + + virtual bool hasAccessControl() = 0; +}; + +typedef std::function MakeNotAllowedError; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths = {}, + MakeNotAllowedError && makeNotAllowedError = {}); + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError = {}); + +SourcePath getUnfilteredRootPath(CanonPath path); + +} diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index f37a8058b..fec16e99f 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -3,12 +3,149 @@ namespace nix { +static std::atomic nextNumber{0}; + +InputAccessor::InputAccessor() + : number(++nextNumber) +{ +} + +// FIXME: merge with archive.cc. +void InputAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) +{ + auto dumpContents = [&](const CanonPath & path) + { + // FIXME: pipe + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; + + std::function dump; + + dump = [&](const CanonPath & path) { + checkInterrupt(); + + auto st = lstat(path); + + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (/* archiveSettings.useCaseHack */ false) { // FIXME + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); +} + +Hash InputAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashType ht) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +StorePath InputAccessor::fetchToStore( + ref store, + const CanonPath & path, + std::string_view name, + PathFilter * filter, + RepairFlag repair) +{ + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); + + auto source = sinkToSource([&](Sink & sink) { + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + }); + + auto storePath = + settings.readOnlyMode + ? store->computeStorePathFromDump(*source, name).first + : store->addToStoreFromDump(*source, name, FileIngestionMethod::Recursive, htSHA256, repair); + + return storePath; +} + +std::optional InputAccessor::maybeLstat(const CanonPath & path) +{ + // FIXME: merge these into one operation. + if (!pathExists(path)) + return {}; + return lstat(path); +} + +std::string InputAccessor::showPath(const CanonPath & path) +{ + return path.abs(); +} + +SourcePath InputAccessor::root() +{ + return {ref(shared_from_this()), CanonPath::root}; +} + std::ostream & operator << (std::ostream & str, const SourcePath & path) { str << path.to_string(); return str; } +StorePath SourcePath::fetchToStore( + ref store, + std::string_view name, + PathFilter * filter, + RepairFlag repair) const +{ + return accessor->fetchToStore(store, path, name, filter, repair); +} + std::string_view SourcePath::baseName() const { return path.baseName().value_or("source"); @@ -18,60 +155,12 @@ SourcePath SourcePath::parent() const { auto p = path.parent(); assert(p); - return std::move(*p); -} - -InputAccessor::Stat SourcePath::lstat() const -{ - auto st = nix::lstat(path.abs()); - return InputAccessor::Stat { - .type = - S_ISREG(st.st_mode) ? InputAccessor::tRegular : - S_ISDIR(st.st_mode) ? InputAccessor::tDirectory : - S_ISLNK(st.st_mode) ? InputAccessor::tSymlink : - InputAccessor::tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; -} - -std::optional SourcePath::maybeLstat() const -{ - // FIXME: merge these into one operation. - if (!pathExists()) - return {}; - return lstat(); -} - -InputAccessor::DirEntries SourcePath::readDirectory() const -{ - InputAccessor::DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = InputAccessor::Type::tRegular; break; - case DT_LNK: type = InputAccessor::Type::tSymlink; break; - case DT_DIR: type = InputAccessor::Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; -} - -StorePath SourcePath::fetchToStore( - ref store, - std::string_view name, - PathFilter * filter, - RepairFlag repair) const -{ - return - settings.readOnlyMode - ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first - : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair); + return {accessor, std::move(*p)}; } SourcePath SourcePath::resolveSymlinks() const { - SourcePath res(CanonPath::root); + auto res = accessor->root(); int linksAllowed = 1024; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5a2f17f62..a0f08e295 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -5,14 +5,29 @@ #include "archive.hh" #include "canon-path.hh" #include "repair-flag.hh" +#include "hash.hh" namespace nix { +MakeError(RestrictedPathError, Error); + +struct SourcePath; class StorePath; class Store; -struct InputAccessor +struct InputAccessor : public std::enable_shared_from_this { + const size_t number; + + InputAccessor(); + + virtual ~InputAccessor() + { } + + virtual std::string readFile(const CanonPath & path) = 0; + + virtual bool pathExists(const CanonPath & path) = 0; + enum Type { tRegular, tSymlink, tDirectory, /** @@ -32,9 +47,63 @@ struct InputAccessor bool isExecutable = false; // regular files only }; + virtual Stat lstat(const CanonPath & path) = 0; + + std::optional maybeLstat(const CanonPath & path); + typedef std::optional DirEntry; typedef std::map DirEntries; + + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashType ht = htSHA256); + + StorePath fetchToStore( + ref store, + const CanonPath & path, + std::string_view name = "source", + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + + /* Return a corresponding path in the root filesystem, if + possible. This is only possible for inputs that are + materialized in the root filesystem. */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const InputAccessor & x) const + { + return number == x.number; + } + + bool operator < (const InputAccessor & x) const + { + return number < x.number; + } + + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + + virtual std::string showPath(const CanonPath & path); + + SourcePath root(); + + /* Return the maximum last-modified time of the files in this + tree, if available. */ + virtual std::optional getLastModified() + { + return std::nullopt; + } }; /** @@ -45,12 +114,9 @@ struct InputAccessor */ struct SourcePath { + ref accessor; CanonPath path; - SourcePath(CanonPath path) - : path(std::move(path)) - { } - std::string_view baseName() const; /** @@ -64,39 +130,42 @@ struct SourcePath * return its contents; otherwise throw an error. */ std::string readFile() const - { return nix::readFile(path.abs()); } + { return accessor->readFile(path); } /** * Return whether this `SourcePath` denotes a file (of any type) * that exists */ bool pathExists() const - { return nix::pathExists(path.abs()); } + { return accessor->pathExists(path); } /** * Return stats about this `SourcePath`, or throw an exception if * it doesn't exist. */ - InputAccessor::Stat lstat() const; + InputAccessor::Stat lstat() const + { return accessor->lstat(path); } /** * Return stats about this `SourcePath`, or std::nullopt if it * doesn't exist. */ - std::optional maybeLstat() const; + std::optional maybeLstat() const + { return accessor->maybeLstat(path); } /** * If this `SourcePath` denotes a directory (not a symlink), * return its directory entries; otherwise throw an error. */ - InputAccessor::DirEntries readDirectory() const; + InputAccessor::DirEntries readDirectory() const + { return accessor->readDirectory(path); } /** * If this `SourcePath` denotes a symlink, return its target; * otherwise throw an error. */ std::string readLink() const - { return nix::readLink(path.abs()); } + { return accessor->readLink(path); } /** * Dump this `SourcePath` to `sink` as a NAR archive. @@ -104,7 +173,7 @@ struct SourcePath void dumpPath( Sink & sink, PathFilter & filter = defaultPathFilter) const - { return nix::dumpPath(path.abs(), sink, filter); } + { return accessor->dumpPath(path, sink, filter); } /** * Copy this `SourcePath` to the Nix store. @@ -120,7 +189,7 @@ struct SourcePath * it has a physical location. */ std::optional getPhysicalPath() const - { return path; } + { return accessor->getPhysicalPath(path); } std::string to_string() const { return path.abs(); } @@ -129,7 +198,7 @@ struct SourcePath * Append a `CanonPath` to this path. */ SourcePath operator + (const CanonPath & x) const - { return {path + x}; } + { return {accessor, path + x}; } /** * Append a single component `c` to this path. `c` must not @@ -137,21 +206,21 @@ struct SourcePath * and `c`. */ SourcePath operator + (std::string_view c) const - { return {path + c}; } + { return {accessor, path + c}; } bool operator == (const SourcePath & x) const { - return path == x.path; + return std::tie(accessor, path) == std::tie(x.accessor, x.path); } bool operator != (const SourcePath & x) const { - return path != x.path; + return std::tie(accessor, path) != std::tie(x.accessor, x.path); } bool operator < (const SourcePath & x) const { - return path < x.path; + return std::tie(accessor, path) < std::tie(x.accessor, x.path); } /** diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100..e761c0e96 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -225,12 +225,16 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const +std::pair Store::computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashType hashAlgo, + const StorePathSet & references) const { - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); + HashSink sink(hashAlgo); + dump.drainInto(sink); + auto h = sink.finish().first; FixedOutputInfo caInfo { .method = method, .hash = h, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4e233bac3..bd69999f8 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -292,14 +292,15 @@ public: StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** - * Preparatory part of addToStore(). - * - * @return the store path to which srcPath is to be copied - * and the cryptographic hash of the contents of srcPath. + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. */ - std::pair computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = {}) const; /** * Preparatory part of addTextToStore(). diff --git a/src/nix/main.cc b/src/nix/main.cc index 879baa5f5..6cd7ace51 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -204,30 +204,30 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( #include "generate-manpage.nix.gen.hh" - , CanonPath::root), *vGenerateManpage); + , state.rootPath(CanonPath::root)), *vGenerateManpage); auto vUtils = state.allocValue(); state.cacheFile( - CanonPath("/utils.nix"), CanonPath("/utils.nix"), + state.rootPath(CanonPath("/utils.nix")), state.rootPath(CanonPath("/utils.nix")), state.parseExprFromString( #include "utils.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vUtils); auto vSettingsInfo = state.allocValue(); state.cacheFile( - CanonPath("/generate-settings.nix"), CanonPath("/generate-settings.nix"), + state.rootPath(CanonPath("/generate-settings.nix")), state.rootPath(CanonPath("/generate-settings.nix")), state.parseExprFromString( #include "generate-settings.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vSettingsInfo); auto vStoreInfo = state.allocValue(); state.cacheFile( - CanonPath("/generate-store-info.nix"), CanonPath("/generate-store-info.nix"), + state.rootPath(CanonPath("/generate-store-info.nix")), state.rootPath(CanonPath("/generate-store-info.nix")), state.parseExprFromString( #include "generate-store-info.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vStoreInfo); auto vDump = state.allocValue(); From df73c6eb8cb14fe11f2d5b27e2e28135c7c5dbaf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Oct 2023 17:34:58 +0200 Subject: [PATCH 164/402] Introduce MemoryInputAccessor and use it for corepkgs MemoryInputAccessor is an in-memory virtual filesystem that returns files like . This removes the need for special hacks to handle those files. --- doc/manual/generate-manpage.nix | 2 +- doc/manual/local.mk | 2 +- src/libexpr/eval.cc | 77 ++++++++++++++++-------- src/libexpr/eval.hh | 32 +++++----- src/libexpr/flake/flake.cc | 10 +-- src/libexpr/parser.y | 5 +- src/libexpr/primops.cc | 29 +++------ src/libexpr/tests/error_traces.cc | 14 ++--- src/libfetchers/memory-input-accessor.cc | 54 +++++++++++++++++ src/libfetchers/memory-input-accessor.hh | 15 +++++ src/nix/main.cc | 34 ++++------- 11 files changed, 176 insertions(+), 98 deletions(-) create mode 100644 src/libfetchers/memory-input-accessor.cc create mode 100644 src/libfetchers/memory-input-accessor.hh diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index d81eac182..14136016d 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -2,7 +2,7 @@ let inherit (builtins) attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique; + inherit (import ) attrsToList concatStrings optionalString filterAttrs trim squash unique; showStoreDocs = import ./generate-store-info.nix; in diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 702fb1ff8..2e9f08678 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -32,7 +32,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 87791fa52..a28bb2101 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -13,6 +13,7 @@ #include "profiles.hh" #include "print.hh" #include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" #include #include @@ -516,7 +517,16 @@ EvalState::EvalState( : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); })) - , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) + , corepkgsFS(makeMemoryInputAccessor()) + , internalFS(makeMemoryInputAccessor()) + , derivationInternal{corepkgsFS->addFile( + CanonPath("derivation-internal.nix"), + #include "primops/derivation.nix.gen.hh" + )} + , callFlakeInternal{internalFS->addFile( + CanonPath("call-flake.nix"), + #include "flake/call-flake.nix.gen.hh" + )} , store(store) , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) @@ -531,7 +541,8 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { - rootFS->allowPath(CanonPath::root); // FIXME + // For now, don't rely on FSInputAccessor for access control. + rootFS->allowPath(CanonPath::root); countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; @@ -570,6 +581,11 @@ EvalState::EvalState( } } + corepkgsFS->addFile( + CanonPath("fetchurl.nix"), + #include "fetchurl.nix.gen.hh" + ); + createBaseEnv(); } @@ -600,6 +616,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & SourcePath EvalState::checkSourcePath(const SourcePath & path_) { + if (path_.accessor != rootFS) return path_; if (!allowedPaths) return path_; auto i = resolvedPaths.find(path_.path.abs()); @@ -614,8 +631,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) */ Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return rootPath(CanonPath(abspath)); - for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; @@ -1180,24 +1195,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial if (!e) e = parseExprFromFile(checkSourcePath(resolvedPath)); - cacheFile(path, resolvedPath, e, v, mustBeTrivial); -} - - -void EvalState::resetFileCache() -{ - fileEvalCache.clear(); - fileParseCache.clear(); -} - - -void EvalState::cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial) -{ fileParseCache[resolvedPath] = e; try { @@ -1226,6 +1223,13 @@ void EvalState::cacheFile( } +void EvalState::resetFileCache() +{ + fileEvalCache.clear(); + fileParseCache.clear(); +} + + void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); @@ -2341,10 +2345,31 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); - if (path == "" || path[0] != '/') - error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return rootPath(CanonPath(path)); + try { + forceValue(v, pos); + + if (v.type() == nString) { + copyContext(v, context); + auto s = v.string_view(); + if (!hasPrefix(s, "/")) + error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow(); + return rootPath(CanonPath(s)); + } + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } + + if (v.type() == nPath) + return v.path(); + + if (v.type() == nAttrs) { + auto i = v.attrs->find(sOutPath); + if (i != v.attrs->end()) + return coerceToPath(pos, *i->value, context, errorCtx); + } + + error("cannot coerce %1% to a path", showType(v)).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4bb0a6a0b..048dff42b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -25,6 +25,7 @@ class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; struct FSInputAccessor; +struct MemoryInputAccessor; /** @@ -212,10 +213,26 @@ public: Bindings emptyBindings; + /** + * The accessor for the root filesystem. + */ const ref rootFS; + /** + * The in-memory filesystem for paths. + */ + const ref corepkgsFS; + + /** + * In-memory filesystem for internal, non-user-callable Nix + * expressions like call-flake.nix. + */ + const ref internalFS; + const SourcePath derivationInternal; + const SourcePath callFlakeInternal; + /** * Store used to materialise .drv files. */ @@ -226,7 +243,6 @@ public: */ const ref buildStore; - RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; /** @@ -408,16 +424,6 @@ public: */ void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); - /** - * Like `evalFile`, but with an already parsed expression. - */ - void cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial = false); - void resetFileCache(); /** @@ -427,7 +433,7 @@ public: SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** - * Try to resolve a search path value (not the optinal key part) + * Try to resolve a search path value (not the optional key part) * * If the specified search path element is a URI, download it. * @@ -832,8 +838,6 @@ struct InvalidPathError : EvalError #endif }; -static const std::string corepkgsPrefix{"/__corepkgs__/"}; - template void ErrorBuilder::debugThrow() { diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2f91ad433..acc84ea4f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -737,14 +737,10 @@ void callFlake(EvalState & state, vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); - if (!state.vCallFlake) { - state.vCallFlake = allocRootValue(state.allocValue()); - state.eval(state.parseExprFromString( - #include "call-flake.nix.gen.hh" - , state.rootPath(CanonPath::root)), **state.vCallFlake); - } + auto vCallFlake = state.allocValue(); + state.evalFile(state.callFlakeInternal, *vCallFlake); - state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 78436c6a2..19812cc49 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,7 +64,6 @@ struct StringToken { #include "parser-tab.hh" #include "lexer-tab.hh" -#include "fs-input-accessor.hh" YY_DECL; @@ -650,6 +649,8 @@ formal #include "fetchers.hh" #include "store-api.hh" #include "flake/flake.hh" +#include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" namespace nix { @@ -761,7 +762,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ } if (hasPrefix(path, "nix/")) - return rootPath(CanonPath(concatStrings(corepkgsPrefix, path.substr(4)))); + return {corepkgsFS, CanonPath(path.substr(3))}; debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dd3fa12ee..f1c24a75c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - StringMap rewrites = state.realiseContext(context); - - auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); + if (!context.empty()) { + auto rewrites = state.realiseContext(context); + auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); + return {path.accessor, CanonPath(realPath)}; + } return flags.checkForPureEval - ? state.checkSourcePath(realPath) - : realPath; + ? state.checkSourcePath(path) + : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } - else if (path2 == corepkgsPrefix + "fetchurl.nix") { - state.eval(state.parseExprFromString( - #include "fetchurl.nix.gen.hh" - , state.rootPath(CanonPath::root)), v); - } - else { if (!vScope) state.evalFile(path, v); @@ -1486,7 +1482,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); NixStringContext context; - auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; + auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -2257,7 +2253,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto path = state.coerceToPath(pos, *args[1], context, - "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } @@ -4444,12 +4440,7 @@ void EvalState::createBaseEnv() /* Note: we have to initialize the 'derivation' constant *after* building baseEnv/staticBaseEnv because it uses 'builtins'. */ - char code[] = - #include "primops/derivation.nix.gen.hh" - // the parser needs two NUL bytes as terminators; one of them - // is implied by being a C string. - "\0"; - eval(parse(code, sizeof(code), derivationInternal, rootPath(CanonPath::root), staticBaseEnv), *vDerivation); + evalFile(derivationInternal, *vDerivation); } diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 285651256..6da8a5954 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a path", "a list"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,8 +309,8 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string", "a Boolean"), - hintfmt("while evaluating the first argument passed to builtins.storePath")); + hintfmt("cannot coerce %s to a path", "a Boolean"), + hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a path", "a list"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -377,13 +377,13 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", EvalError, hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] ./.", TypeError, diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc new file mode 100644 index 000000000..817d063ba --- /dev/null +++ b/src/libfetchers/memory-input-accessor.cc @@ -0,0 +1,54 @@ +#include "memory-input-accessor.hh" + +namespace nix { + +struct MemoryInputAccessorImpl : MemoryInputAccessor +{ + std::map files; + + std::string readFile(const CanonPath & path) override + { + auto i = files.find(path); + if (i == files.end()) + throw Error("file '%s' does not exist", path); + return i->second; + } + + bool pathExists(const CanonPath & path) override + { + auto i = files.find(path); + return i != files.end(); + } + + Stat lstat(const CanonPath & path) override + { + auto i = files.find(path); + if (i != files.end()) + return Stat { .type = tRegular, .isExecutable = false }; + throw Error("file '%s' does not exist", path); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return {}; + } + + std::string readLink(const CanonPath & path) override + { + throw UnimplementedError("MemoryInputAccessor::readLink"); + } + + SourcePath addFile(CanonPath path, std::string && contents) override + { + files.emplace(path, std::move(contents)); + + return {ref(shared_from_this()), std::move(path)}; + } +}; + +ref makeMemoryInputAccessor() +{ + return make_ref(); +} + +} diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh new file mode 100644 index 000000000..b75b02bfd --- /dev/null +++ b/src/libfetchers/memory-input-accessor.hh @@ -0,0 +1,15 @@ +#include "input-accessor.hh" + +namespace nix { + +/** + * An input accessor for an in-memory file system. + */ +struct MemoryInputAccessor : InputAccessor +{ + virtual SourcePath addFile(CanonPath path, std::string && contents) = 0; +}; + +ref makeMemoryInputAccessor(); + +} diff --git a/src/nix/main.cc b/src/nix/main.cc index 6cd7ace51..ee3878cc7 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -12,6 +12,7 @@ #include "finally.hh" #include "loggers.hh" #include "markdown.hh" +#include "memory-input-accessor.hh" #include #include @@ -206,29 +207,20 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) #include "generate-manpage.nix.gen.hh" , state.rootPath(CanonPath::root)), *vGenerateManpage); - auto vUtils = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/utils.nix")), state.rootPath(CanonPath("/utils.nix")), - state.parseExprFromString( - #include "utils.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vUtils); + state.corepkgsFS->addFile( + CanonPath("utils.nix"), + #include "utils.nix.gen.hh" + ); - auto vSettingsInfo = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/generate-settings.nix")), state.rootPath(CanonPath("/generate-settings.nix")), - state.parseExprFromString( - #include "generate-settings.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vSettingsInfo); + state.corepkgsFS->addFile( + CanonPath("/generate-settings.nix"), + #include "generate-settings.nix.gen.hh" + ); - auto vStoreInfo = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/generate-store-info.nix")), state.rootPath(CanonPath("/generate-store-info.nix")), - state.parseExprFromString( - #include "generate-store-info.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vStoreInfo); + state.corepkgsFS->addFile( + CanonPath("/generate-store-info.nix"), + #include "generate-store-info.nix.gen.hh" + ); auto vDump = state.allocValue(); vDump->mkString(toplevel.dumpCli()); From e9ddf0b4008262542b422c76f60c3fb80982cd74 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 10:53:47 +0800 Subject: [PATCH 165/402] Simplify parseHashTypeOpt Remove redundant "else" after "return". Use std::nullopt to increase readability. --- src/libutil/hash.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d94..6d56c1989 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -389,10 +389,10 @@ Hash compressHash(const Hash & hash, unsigned int newSize) std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else if (s == "sha512") return htSHA512; - else return std::optional {}; + if (s == "sha1") return htSHA1; + if (s == "sha256") return htSHA256; + if (s == "sha512") return htSHA512; + return std::nullopt; } HashType parseHashType(std::string_view s) From aff177d8609c392b4edf59819a086440cca101a7 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 11:00:14 +0800 Subject: [PATCH 166/402] Elaborate the "unknown hash algorithm" error List the allowed hash formats --- src/libexpr/tests/error_traces.cc | 2 +- src/libutil/hash.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 285651256..1af601ec8 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -1084,7 +1084,7 @@ namespace nix { ASSERT_TRACE1("hashString \"foo\" \"content\"", UsageError, - hintfmt("unknown hash algorithm '%s'", "foo")); + hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); ASSERT_TRACE2("hashString \"sha256\" {}", TypeError, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 6d56c1989..251440363 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -401,7 +401,7 @@ HashType parseHashType(std::string_view s) if (opt_h) return *opt_h; else - throw UsageError("unknown hash algorithm '%1%'", s); + throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } std::string_view printHashType(HashType ht) From 838c70f62116328ce01cb41a01886e4f1b9a727f Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Tue, 10 Oct 2023 18:53:01 +0800 Subject: [PATCH 167/402] treewide: Rename hashBase to hashFormat hashBase is ambiguous, since it's not about the digital bases, but about the format of hashes. Base16, Base32 and Base64 are all character maps for binary encoding. Rename the enum Base to HashFormat. Rename variables of type HashFormat from [hash]Base to hashFormat, including CmdHashBase::hashFormat and CmdToBase::hashFormat. --- src/libstore/store-api.cc | 6 ++--- src/libstore/store-api.hh | 2 +- src/libutil/hash.cc | 8 +++---- src/libutil/hash.hh | 4 ++-- src/libutil/tests/hash.cc | 16 +++++++------- src/nix/hash.cc | 46 +++++++++++++++++++-------------------- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100..fa23f976e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -938,7 +938,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor json Store::pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase, + HashFormat hashFormat, AllowInvalidFlag allowInvalid) { json::array_t jsonList = json::array(); @@ -951,7 +951,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, jsonPath["path"] = printStorePath(info->path); jsonPath["valid"] = true; - jsonPath["narHash"] = info->narHash.to_string(hashBase, true); + jsonPath["narHash"] = info->narHash.to_string(hashFormat, true); jsonPath["narSize"] = info->narSize; { @@ -993,7 +993,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, if (!narInfo->url.empty()) jsonPath["url"] = narInfo->url; if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); + jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true); if (narInfo->fileSize) jsonPath["downloadSize"] = narInfo->fileSize; if (showClosureSize) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4e233bac3..f30ccc950 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -676,7 +676,7 @@ public: */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base32, + HashFormat hashFormat = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 251440363..401a65ec3 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -115,14 +115,14 @@ std::string printHash16or32(const Hash & hash) } -std::string Hash::to_string(Base base, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeType) const { std::string s; - if (base == SRI || includeType) { + if (hashFormat == SRI || includeType) { s += printHashType(type); - s += base == SRI ? '-' : ':'; + s += hashFormat == SRI ? '-' : ':'; } - switch (base) { + switch (hashFormat) { case Base16: s += printHash16(*this); break; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index c3aa5cd81..73c2183ef 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -23,7 +23,7 @@ extern std::set hashTypes; extern const std::string base32Chars; -enum Base : int { Base64, Base32, Base16, SRI }; +enum HashFormat : int { Base64, Base32, Base16, SRI }; struct Hash @@ -114,7 +114,7 @@ public: * or base-64. By default, this is prefixed by the hash type * (e.g. "sha256:"). */ - std::string to_string(Base base, bool includeType) const; + std::string to_string(HashFormat hashFormat, bool includeType) const; std::string gitRev() const { diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index e4e928b3b..d14e5b26d 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -18,28 +18,28 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { @@ -47,7 +47,7 @@ namespace nix { auto s = "abc"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -55,7 +55,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -63,7 +63,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); @@ -74,7 +74,7 @@ namespace nix { auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9feca9345..6af6bd19f 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - Base base = SRI; + HashFormat hashFormat = SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&base, SRI}, + .handler = {&hashFormat, SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&base, Base64}, + .handler = {&hashFormat, Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&base, Base32}, + .handler = {&hashFormat, Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&base, Base16}, + .handler = {&hashFormat, Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -94,18 +94,18 @@ struct CmdHashBase : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(base, base == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == SRI)); } } }; struct CmdToBase : Command { - Base base; + HashFormat hashFormat; std::optional ht; std::vector args; - CmdToBase(Base base) : base(base) + CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { addFlag(Flag::mkHashTypeOptFlag("type", &ht)); expectArgs("strings", &args); @@ -114,16 +114,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - base == Base16 ? "base-16" : - base == Base32 ? "base-32" : - base == Base64 ? "base-64" : + hashFormat == Base16 ? "base-16" : + hashFormat == Base32 ? "base-32" : + hashFormat == Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == SRI)); } }; @@ -162,7 +162,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional ht; bool flat = false; - Base base = Base16; + HashFormat hashFormat = Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector ss; @@ -173,10 +173,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") base = Base16; - else if (*arg == "--base32") base = Base32; - else if (*arg == "--base64") base = Base64; - else if (*arg == "--sri") base = SRI; + else if (*arg == "--base16") hashFormat = Base16; + else if (*arg == "--base32") hashFormat = Base32; + else if (*arg == "--base64") hashFormat = Base64; + else if (*arg == "--sri") hashFormat = SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -184,19 +184,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - base = Base16; + hashFormat = Base16; } else if (*arg == "--to-base32") { op = opTo; - base = Base32; + hashFormat = Base32; } else if (*arg == "--to-base64") { op = opTo; - base = Base64; + hashFormat = Base64; } else if (*arg == "--to-sri") { op = opTo; - base = SRI; + hashFormat = SRI; } else if (*arg != "" && arg->at(0) == '-') return false; @@ -209,14 +209,14 @@ static int compatNixHash(int argc, char * * argv) CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); if (!ht.has_value()) ht = htMD5; cmd.ht = ht.value(); - cmd.base = base; + cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(base); + CmdToBase cmd(hashFormat); cmd.args = ss; if (ht.has_value()) cmd.ht = ht; cmd.run(); From 5043e6cf4ea537dfe599470797c5b310ab0e94b9 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 18 Oct 2023 17:08:06 +0800 Subject: [PATCH 168/402] Document HashFormat --- src/libutil/hash.hh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 73c2183ef..0db9bcd6d 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -23,7 +23,21 @@ extern std::set hashTypes; extern const std::string base32Chars; -enum HashFormat : int { Base64, Base32, Base16, SRI }; +/** + * @brief Enumeration representing the hash formats. + */ +enum HashFormat : int { + /// @brief Base 64 encoding. + /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). + Base64, + /// @brief Nix-specific base-32 encoding. @see base32Chars + Base32, + /// @brief Lowercase hexadecimal encoding. @see base16Chars + Base16, + /// @brief ":", format of the SRI integrity attribute. + /// @see W3C recommendation [Subresource Intergrity](https://www.w3.org/TR/SRI/). + SRI +}; struct Hash From e026f3e1ae533ab476e58258c7d84814189df9ff Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 13 Oct 2023 09:48:15 +0800 Subject: [PATCH 169/402] treewide: Reference HashFormat members with scope Base* -> HashFormat::Base* --- perl/lib/Nix/Store.xs | 12 +++--- src/libexpr/eval-cache.cc | 2 +- src/libexpr/primops.cc | 4 +- src/libexpr/primops/fetchTree.cc | 4 +- src/libfetchers/fetchers.cc | 4 +- src/libfetchers/git.cc | 4 +- src/libfetchers/github.cc | 8 ++-- src/libfetchers/mercurial.cc | 4 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 +-- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/content-address.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 12 +++--- src/libstore/downstream-placeholder.cc | 4 +- src/libstore/export-import.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 14 +++---- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/optimise-store.cc | 4 +- src/libstore/path-info.cc | 4 +- src/libstore/path.cc | 2 +- src/libstore/realisation.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 6 +-- src/libstore/store-api.hh | 2 +- src/libutil/hash.cc | 16 +++---- src/libutil/hash.hh | 4 +- src/nix-store/nix-store.cc | 8 ++-- src/nix/flake.cc | 8 ++-- src/nix/hash.cc | 46 ++++++++++----------- src/nix/path-info.cc | 2 +- src/nix/prefetch.cc | 4 +- src/nix/verify.cc | 4 +- 37 files changed, 108 insertions(+), 108 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index e96885e4c..08f812b31 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -78,7 +78,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashType(algo), path).first; - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashType(algo), path); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashType(algo), s); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { auto h = Hash::parseAny(s, parseHashType(algo)); - auto s = h.to_string(toBase32 ? Base32 : Base16, false); + auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 391b32a77..10fc799a9 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -50,7 +50,7 @@ struct AttrDb Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3bed7c5aa..830579dbf 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1758,7 +1758,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -3760,7 +3760,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3431ff013..976037ff9 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -31,7 +31,7 @@ void emitTreeAttrs( auto narHash = input.getNarHash(); assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -297,7 +297,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); + *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); } state.allowAndSetStorePathString(storePath, v); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e3d3ac562..c54c39cf0 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -147,12 +147,12 @@ std::pair Input::fetch(ref store) const }; auto narHash = store->queryPathInfo(tree.storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); + to_string(), tree.actualPath, prevNarHash->to_string(HashFormat::SRI, true), narHash.to_string(HashFormat::SRI, true)); } if (auto prevLastModified = getLastModified()) { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7e9f34790..c1a6dce43 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -46,7 +46,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(Base32, false); + hashString(htSHA256, key).to_string(HashFormat::Base32, false); } // Returns the name of the HEAD branch. @@ -417,7 +417,7 @@ struct GitInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); }; auto getLockedAttrs = [&]() diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 6c5c792ec..d7450defe 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -125,7 +125,7 @@ struct GitArchiveInputScheme : InputScheme auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); + if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); return ParsedURL { .scheme = type(), .path = path, @@ -296,7 +296,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://api.%s/repos/%s/%s/tarball/%s"; const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); return DownloadUrl { url, headers }; } @@ -362,7 +362,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; @@ -449,7 +449,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 51fd1ed42..b0d2d1909 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -206,7 +206,7 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && hash->type != htSHA1) - throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); }; @@ -252,7 +252,7 @@ struct MercurialInputScheme : InputScheme } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e3a41e75e..269a56526 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -250,7 +250,7 @@ struct CurlInputScheme : InputScheme // NAR hashes are preferred over file hashes since tar/zip // files don't have a canonical representation. if (auto narHash = input.getNarHash()) - url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b4fea693f..2a91233ec 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -164,7 +164,7 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6a02f5ad4..40a0385af 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1066,7 +1066,7 @@ void LocalDerivationGoal::initTmpDir() { env[i.first] = i.second; } else { auto hash = hashString(htSHA256, i.first); - std::string fn = ".attr-" + hash.to_string(Base32, false); + std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -2583,8 +2583,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true))); } if (!newInfo0.references.empty()) delayedException = std::make_exception_ptr( diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 7d7924d77..357800333 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index ae91b859b..52c60154c 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -60,7 +60,7 @@ std::string ContentAddress::render() const + makeFileIngestionPrefix(method); }, }, method.raw) - + this->hash.to_string(Base32, true); + + this->hash.to_string(HashFormat::Base32, true); } /** diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044..5860dd548 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -334,7 +334,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash.to_string(HashFormat::Base16, false); break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fc17e520c..a5ceb29dc 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -542,7 +542,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(HashFormat::Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -775,7 +775,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto & dof = std::get(i.second.raw); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.hash.to_string(Base16, false) + ":" + + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -820,7 +820,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].value.insert(outputName); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); } } @@ -925,7 +925,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.hash.to_string(Base16, false); + << dof.ca.hash.to_string(HashFormat::Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -957,7 +957,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); + return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); } @@ -1162,7 +1162,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.hash.to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 7e3f7548d..ca9f7476e 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,7 +5,7 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(Base32, false); + return "/" + hash.to_string(HashFormat::Base32, false); } @@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(Base32, false) + + compressed.to_string(HashFormat::Base32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { hashString(htSHA256, clearText) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 87b2f8741..91b7e30db 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); teeSink << exportMagic diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 516cbef83..fb7895817 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -43,7 +43,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(Base32, false); + std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 703ded0b2..46c90ee31 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << ServeProto::Command::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(HashFormat::Base16, false); ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17b4ecc73..1c2f6023a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -826,7 +826,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmts->RegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -933,7 +933,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -1236,7 +1236,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1252,8 +1252,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - specified.hash.to_string(Base32, true), - actualHash.hash.to_string(Base32, true)); + specified.hash.to_string(HashFormat::Base32, true), + actualHash.hash.to_string(HashFormat::Base32, true)); } } @@ -1545,7 +1545,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1578,7 +1578,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f..cdbcf7e74 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -332,9 +332,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash.to_string(HashFormat::Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d17253741..ee2ddfd81 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -105,10 +105,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash && fileHash->type == htSHA256); - res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; + res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4a79cf4a1..23c6a41e4 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -146,10 +146,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f..7ad3247da 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash.to_string(HashFormat::Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -161,7 +161,7 @@ void ValidPathInfo::write( if (includePath) sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(Base16, false); + << narHash.to_string(HashFormat::Base16, false); WorkerProto::write(store, WorkerProto::WriteConn { .to = sink }, references); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 3c6b9fc10..ec3e53232 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 559483ce3..4ba2123d8 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -39,7 +39,7 @@ struct DrvOutput { std::string to_string() const; std::string strHash() const - { return drvHash.to_string(Base16, true); } + { return drvHash.to_string(HashFormat::Base16, true); } static DrvOutput parse(const std::string &); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a639346d1..1704bfbb0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -541,7 +541,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fa23f976e..0c58cca0c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -154,7 +154,7 @@ StorePath Store::makeStorePath(std::string_view type, StorePath Store::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { - return makeStorePath(type, hash.to_string(Base16, true), name); + return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } @@ -192,7 +192,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf hashString(htSHA256, "fixed:out:" + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + info.hash.to_string(HashFormat::Base16, true) + ":"), name); } } @@ -884,7 +884,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash.to_string(HashFormat::Base16, false) + "\n"; s += fmt("%1%\n", info->narSize); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f30ccc950..2d27f7d51 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -676,7 +676,7 @@ public: */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat = Base32, + HashFormat hashFormat = HashFormat::Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 401a65ec3..b58b5f08d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -111,26 +111,26 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { assert(hash.type); - return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); + return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false); } std::string Hash::to_string(HashFormat hashFormat, bool includeType) const { std::string s; - if (hashFormat == SRI || includeType) { + if (hashFormat == HashFormat::SRI || includeType) { s += printHashType(type); - s += hashFormat == SRI ? '-' : ':'; + s += hashFormat == HashFormat::SRI ? '-' : ':'; } switch (hashFormat) { - case Base16: + case HashFormat::Base16: s += printHash16(*this); break; - case Base32: + case HashFormat::Base32: s += printHash32(*this); break; - case Base64: - case SRI: + case HashFormat::Base64: + case HashFormat::SRI: s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } @@ -267,7 +267,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) if (!ht) throw BadHash("empty hash requires explicit hash type"); Hash h(*ht); - warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else return Hash::parseAny(hashStr, ht); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0db9bcd6d..783e30496 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -132,12 +132,12 @@ public: std::string gitRev() const { - return to_string(Base16, false); + return to_string(HashFormat::Base16, false); } std::string gitShortRev() const { - return std::string(to_string(Base16, false), 0, 7); + return std::string(to_string(HashFormat::Base16, false), 0, 7); } static Hash dummy; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9b6c80a75..5e494bcbf 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -406,7 +406,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -769,8 +769,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + current.first.to_string(HashFormat::Base32, true)); status = 1; } } @@ -892,7 +892,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(Base32, true) + out << info->narHash.to_string(HashFormat::Base32, true) << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a77b5fcb8..ceb112c03 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -179,7 +179,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); + j["revision"] = rev->to_string(HashFormat::Base16, false); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) @@ -206,7 +206,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", - rev->to_string(Base16, false)); + rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -1345,7 +1345,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(tree.storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); logger->cout(res.dump()); @@ -1353,7 +1353,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), store->printStorePath(tree.storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 6af6bd19f..d6595dcca 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - HashFormat hashFormat = SRI; + HashFormat hashFormat = HashFormat::SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&hashFormat, SRI}, + .handler = {&hashFormat, HashFormat::SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&hashFormat, Base64}, + .handler = {&hashFormat, HashFormat::Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&hashFormat, Base32}, + .handler = {&hashFormat, HashFormat::Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&hashFormat, Base16}, + .handler = {&hashFormat, HashFormat::Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -94,7 +94,7 @@ struct CmdHashBase : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(hashFormat, hashFormat == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI)); } } }; @@ -114,16 +114,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - hashFormat == Base16 ? "base-16" : - hashFormat == Base32 ? "base-32" : - hashFormat == Base64 ? "base-64" : + hashFormat == HashFormat::Base16 ? "base-16" : + hashFormat == HashFormat::Base32 ? "base-32" : + hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; @@ -133,10 +133,10 @@ struct CmdHash : NixMultiCommand : MultiCommand({ {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, - {"to-base16", []() { return make_ref(Base16); }}, - {"to-base32", []() { return make_ref(Base32); }}, - {"to-base64", []() { return make_ref(Base64); }}, - {"to-sri", []() { return make_ref(SRI); }}, + {"to-base16", []() { return make_ref(HashFormat::Base16); }}, + {"to-base32", []() { return make_ref(HashFormat::Base32); }}, + {"to-base64", []() { return make_ref(HashFormat::Base64); }}, + {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) { } @@ -162,7 +162,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional ht; bool flat = false; - HashFormat hashFormat = Base16; + HashFormat hashFormat = HashFormat::Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector ss; @@ -173,10 +173,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") hashFormat = Base16; - else if (*arg == "--base32") hashFormat = Base32; - else if (*arg == "--base64") hashFormat = Base64; - else if (*arg == "--sri") hashFormat = SRI; + else if (*arg == "--base16") hashFormat = HashFormat::Base16; + else if (*arg == "--base32") hashFormat = HashFormat::Base32; + else if (*arg == "--base64") hashFormat = HashFormat::Base64; + else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -184,19 +184,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - hashFormat = Base16; + hashFormat = HashFormat::Base16; } else if (*arg == "--to-base32") { op = opTo; - hashFormat = Base32; + hashFormat = HashFormat::Base32; } else if (*arg == "--to-base64") { op = opTo; - hashFormat = Base64; + hashFormat = HashFormat::Base64; } else if (*arg == "--to-sri") { op = opTo; - hashFormat = SRI; + hashFormat = HashFormat::SRI; } else if (*arg != "" && arg->at(0) == '-') return false; diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 613c5b191..c16864d30 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -90,7 +90,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << store->pathInfoToJSON( // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid).dump(); + true, showClosureSize, HashFormat::SRI, AllowInvalid).dump(); } else { diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b67d381ca..3ed7946a8 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -310,13 +310,13 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", url, store->printStorePath(storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 0b306cc11..adaa33c0c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -108,8 +108,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + hash.first.to_string(HashFormat::Base32, true)); } } From 231b0fca6df668f904dfb86ef858db556caebdff Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 13 Oct 2023 18:48:36 +0800 Subject: [PATCH 170/402] Migrate HashFormat to scoped enumeration (enum struct) --- src/libutil/hash.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 783e30496..d2bf1cb23 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -26,7 +26,7 @@ extern const std::string base32Chars; /** * @brief Enumeration representing the hash formats. */ -enum HashFormat : int { +enum struct HashFormat : int { /// @brief Base 64 encoding. /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). Base64, From 6b47635180c740b7478f187b3ae9e35b36deeed7 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 11:03:16 +0800 Subject: [PATCH 171/402] Add helper function parseHashFormat[Opt] printHashFormat Add hash format analogy of parseHashTypeOpt, parseHashType, and printHashType. Co-authored-by: Valentin Gagarin --- src/libutil/hash.cc | 35 +++++++++++++++++++++++++++++++++++ src/libutil/hash.hh | 15 +++++++++++++++ src/libutil/tests/hash.cc | 15 +++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index b58b5f08d..e297c245b 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -386,6 +386,41 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } +std::optional parseHashFormatOpt(std::string_view hashFormatName) +{ + if (hashFormatName == "base16") return HashFormat::Base16; + if (hashFormatName == "base32") return HashFormat::Base32; + if (hashFormatName == "base64") return HashFormat::Base64; + if (hashFormatName == "sri") return HashFormat::SRI; + return std::nullopt; +} + +HashFormat parseHashFormat(std::string_view hashFormatName) +{ + auto opt_f = parseHashFormatOpt(hashFormatName); + if (opt_f) + return *opt_f; + throw UsageError("unknown hash format '%1%', expect 'base16', 'base32', 'base64', or 'sri'", hashFormatName); +} + +std::string_view printHashFormat(HashFormat HashFormat) +{ + switch (HashFormat) { + case HashFormat::Base64: + return "base64"; + case HashFormat::Base32: + return "base32"; + case HashFormat::Base16: + return "base16"; + case HashFormat::SRI: + return "sri"; + default: + // illegal hash base enum value internally, as opposed to external input + // which should be validated with nice error message. + assert(false); + } +} + std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index d2bf1cb23..cab3e6eca 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -181,6 +181,21 @@ HashResult hashPath(HashType ht, const Path & path, */ Hash compressHash(const Hash & hash, unsigned int newSize); +/** + * Parse a string representing a hash format. + */ +HashFormat parseHashFormat(std::string_view hashFormatName); + +/** + * std::optional version of parseHashFormat that doesn't throw error. + */ +std::optional parseHashFormatOpt(std::string_view hashFormatName); + +/** + * The reverse of parseHashFormat. + */ +std::string_view printHashFormat(HashFormat hashFormat); + /** * Parse a string representing a hash type. */ diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index d14e5b26d..9a5ebbb30 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -79,6 +79,21 @@ namespace nix { "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); } + + /* ---------------------------------------------------------------------------- + * parseHashFormat, parseHashFormatOpt, printHashFormat + * --------------------------------------------------------------------------*/ + + TEST(hashFormat, testRoundTripPrintParse) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) { + ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); + ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); + } + } + + TEST(hashFormat, testParseHashFormatOptException) { + ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); + } } namespace rc { From 5088e6563a4e88b7e6615623d3ed655fdebd3345 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li <44064051+ShamrockLee@users.noreply.github.com> Date: Sun, 29 Jan 2023 14:03:06 +0800 Subject: [PATCH 172/402] primops: add builtins.convertHash Co-authored-by: Valentin Gagarin --- doc/manual/src/release-notes/rl-next.md | 2 + src/libexpr/primops.cc | 95 +++++++++++++++++++ .../functional/lang/eval-okay-convertHash.exp | 1 + .../functional/lang/eval-okay-convertHash.nix | 31 ++++++ 4 files changed, 129 insertions(+) create mode 100644 tests/functional/lang/eval-okay-convertHash.exp create mode 100644 tests/functional/lang/eval-okay-convertHash.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c905f445f..46ce2abac 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,4 +8,6 @@ - Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. +- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + - `builtins.fetchTree` is now marked as stable. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 830579dbf..14beb069f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3774,6 +3774,101 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); +static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash"); + auto &inputAttrs = args[0]->attrs; + + Bindings::iterator iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'"); + auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); + + Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); + std::optional ht = std::nullopt; + if (iteratorHashAlgo != inputAttrs->end()) { + ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + } + + Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); + HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); + + v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI)); +} + +static RegisterPrimOp primop_convertHash({ + .name = "__convertHash", + .args = {"args"}, + .doc = R"( + Return the specified representation of a hash string, based on the attributes presented in *args*: + + - `hash` + + The hash to be converted. + The hash format is detected automatically. + + - `hashAlgo` + + The algorithm used to create the hash. Must be one of + - `"md5"` + - `"sha1"` + - `"sha256"` + - `"sha512"` + + The attribute may be omitted when `hash` is an [SRI hash](https://www.w3.org/TR/SRI/#the-integrity-attribute) or when the hash is prefixed with the hash algorithm name followed by a colon. + That `:` syntax is supported for backwards compatibility with existing tooling. + + - `toHashFormat` + + The format of the resulting hash. Must be one of + - `"base16"` + - `"base32"` + - `"base64"` + - `"sri"` + + The result hash is the *toHashFormat* representation of the hash *hash*. + + > **Example** + > + > Convert a SHA256 hash in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > hashAlgo = "sha256"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + + > **Example** + > + > Convert a SHA256 hash in SRI to Base16: + > + > ```nix + > builtins.convertHash { + > hash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + > toHashFormat = "base16"; + > } + > ``` + > + > "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + > **Example** + > + > Convert a hash in the form `:` in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + )", + .fun = prim_convertHash, +}); + struct RegexCache { // TODO use C++20 transparent comparison when available diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp new file mode 100644 index 000000000..60e0a3c49 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -0,0 +1 @@ +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix new file mode 100644 index 000000000..cf4909aaf --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -0,0 +1,31 @@ +let + hashAlgos = [ "md5" "md5" "md5" "sha1" "sha1" "sha1" "sha256" "sha256" "sha256" "sha512" "sha512" "sha512" ]; + hashesBase16 = import ./eval-okay-hashstring.exp; + map2 = f: { fsts, snds }: if fsts == [ ] then [ ] else [ (f (builtins.head fsts) (builtins.head snds)) ] ++ map2 f { fsts = builtins.tail fsts; snds = builtins.tail snds; }; + map2' = f: fsts: snds: map2 f { inherit fsts snds; }; + getOutputHashes = hashes: { + hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + getOutputHashesColon = hashes: { + hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + outputHashes = getOutputHashes hashesBase16; +in +# map2'` +assert map2' (s1: s2: s1 + s2) [ "a" "b" ] [ "c" "d" ] == [ "ac" "bd" ]; +# hashesBase16 +assert outputHashes.hashesBase16 == hashesBase16; +# standard SRI hashes +assert outputHashes.hashesSRI == (map2' (hashAlgo: hashBody: hashAlgo + "-" + hashBody) hashAlgos outputHashes.hashesBase64); +# without prefix +assert builtins.all (x: getOutputHashes x == outputHashes) (builtins.attrValues outputHashes); +# colon-separated. +# Note that colon prefix must not be applied to the standard SRI. e.g. "sha256:sha256-..." is illegal. +assert builtins.all (x: getOutputHashesColon x == outputHashes) (with outputHashes; [ hashesBase16 hashesBase32 hashesBase64 ]); +outputHashes From d2c0051784e4a338254445f8f680abf1e37e9024 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Oct 2023 23:35:07 +0200 Subject: [PATCH 173/402] Remove obsolete corepkgs references --- doc/manual/local.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 2e9f08678..8bf16e9dd 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -125,7 +125,7 @@ $(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $ @mv $@.tmp $@ $(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @@ -141,7 +141,7 @@ $(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin @mv $@.tmp $@ $(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ # Generate the HTML manual. From 34c559352579b9e1a501ec2fc435f3e21cf6e78f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 10:57:45 +0200 Subject: [PATCH 174/402] add a link to all maintainer meeting notes linking to the discourse category will by default show a view sorted by most recent post, which makes it hard to find particular meeting notes. this also adds a procedural detail about the notes, to make that more explicit and less dependent on being present in the meetings. --- maintainers/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 8eeb47e8b..5be4f9d04 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -50,7 +50,9 @@ The team meets twice a week: 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. -Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +[All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). ## Project board protocol From 9adac237e70003625b626cef04979ce746e9dd7e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 11:40:27 +0200 Subject: [PATCH 175/402] update link to label GitHub now displays a banner and has a dedicated page[1] for good first issues, but that uses a different label name as we had in place. I renamed the label on GitHub, this is updating the link. [1]: https://github.com/NixOS/nix/contribute --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73210b303..556365375 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 1. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. - Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. + Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue) should be relatively easy to fix and are likely to get merged quickly. Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. From 36b15d905e8ca6f5eb4f73fb842600b6e091708e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 11:47:10 +0200 Subject: [PATCH 176/402] link to popular issues from the contributing guide this also adds a hint to contributors about making far-reaching changes, complementing the recent update to the maintainers' handbook on how to deal with those. --- CONTRIBUTING.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73210b303..71970f307 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. + If you are proficient with C++, addressing one of the [popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) will be highly appreciated by maintainers and Nix users all over the world. + For far-reaching changes, please investigate possible blockers and design implications, and coordinate with maintainers before investing too much time in writing code that may not end up getting merged. + If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. 2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. From 12214fef09151e26abfcaea0095ab157388822e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:19:10 +0200 Subject: [PATCH 177/402] InputAccessor::fetchToStore(): Support arbitrary ingestion methods --- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 5 +---- src/libfetchers/input-accessor.cc | 8 +++++--- src/libfetchers/input-accessor.hh | 3 +++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a28bb2101..3b0fe93e1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2329,7 +2329,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); + auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f1c24a75c..f416aa639 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2233,10 +2233,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - // FIXME - if (method != FileIngestionMethod::Recursive) - throw Error("'recursive = false' is not implemented"); - auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, &filter, state.repair); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index fec16e99f..63f1f4ad2 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -96,6 +96,7 @@ StorePath InputAccessor::fetchToStore( ref store, const CanonPath & path, std::string_view name, + FileIngestionMethod method, PathFilter * filter, RepairFlag repair) { @@ -107,8 +108,8 @@ StorePath InputAccessor::fetchToStore( auto storePath = settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name).first - : store->addToStoreFromDump(*source, name, FileIngestionMethod::Recursive, htSHA256, repair); + ? store->computeStorePathFromDump(*source, name, method, htSHA256).first + : store->addToStoreFromDump(*source, name, method, htSHA256, repair); return storePath; } @@ -140,10 +141,11 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path) StorePath SourcePath::fetchToStore( ref store, std::string_view name, + FileIngestionMethod method, PathFilter * filter, RepairFlag repair) const { - return accessor->fetchToStore(store, path, name, filter, repair); + return accessor->fetchToStore(store, path, name, method, filter, repair); } std::string_view SourcePath::baseName() const diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index a0f08e295..003c7226c 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -6,6 +6,7 @@ #include "canon-path.hh" #include "repair-flag.hh" #include "hash.hh" +#include "content-address.hh" namespace nix { @@ -73,6 +74,7 @@ struct InputAccessor : public std::enable_shared_from_this ref store, const CanonPath & path, std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); @@ -181,6 +183,7 @@ struct SourcePath StorePath fetchToStore( ref store, std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair) const; From f16af08e8393420f6234ab59bde716dd145703d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:20:50 +0200 Subject: [PATCH 178/402] Fix macOS compilation --- src/libexpr/eval.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3b0fe93e1..7180b9fbe 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -616,7 +616,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & SourcePath EvalState::checkSourcePath(const SourcePath & path_) { - if (path_.accessor != rootFS) return path_; + // Don't check non-rootFS accessors, they're in a different namespace. + if (path_.accessor != ref(rootFS)) return path_; + if (!allowedPaths) return path_; auto i = resolvedPaths.find(path_.path.abs()); From 06c57899e306bd6a19fce5d83b2ffce487067cdb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:22:05 +0200 Subject: [PATCH 179/402] Remove FIXME --- src/libexpr/primops.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f416aa639..b971fd052 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -595,7 +595,9 @@ struct CompareValues case nString: return v1->string_view().compare(v2->string_view()) < 0; case nPath: - // FIXME: handle accessor? + // Note: we don't take the accessor into account + // since it's not obvious how to compare them in a + // reproducible way. return strcmp(v1->_path.path, v2->_path.path) < 0; case nList: // Lexicographic comparison From 9bc7b4f463b6a06746bd1658d0170865e3fc8337 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 19 Oct 2023 14:39:41 +0200 Subject: [PATCH 180/402] doc: generic closure supported key types (#9183) * doc: generic closure supported key types Co-authored-by: Valentin Gagarin --- src/libexpr/primops.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 14beb069f..e5c5eb5b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -734,6 +734,14 @@ static RegisterPrimOp primop_genericClosure(PrimOp { ``` [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] ``` + + `key` can be one of the following types: + - [Number](@docroot@/language/values.md#type-number) + - [Boolean](@docroot@/language/values.md#type-boolean) + - [String](@docroot@/language/values.md#type-string) + - [Path](@docroot@/language/values.md#type-path) + - [List](@docroot@/language/values.md#list) + )", .fun = prim_genericClosure, }); From fb6a3910c4b4ebec2ab2e422a7454b8f3ba71f85 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:45:26 +0200 Subject: [PATCH 181/402] Move most of InputAccessor into libutil --- src/libfetchers/input-accessor.cc | 102 ---------------------------- src/libfetchers/input-accessor.hh | 85 +---------------------- src/libutil/source-accessor.cc | 108 ++++++++++++++++++++++++++++++ src/libutil/source-accessor.hh | 96 ++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 185 deletions(-) create mode 100644 src/libutil/source-accessor.cc create mode 100644 src/libutil/source-accessor.hh diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 63f1f4ad2..9bddf7c2e 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -3,95 +3,6 @@ namespace nix { -static std::atomic nextNumber{0}; - -InputAccessor::InputAccessor() - : number(++nextNumber) -{ -} - -// FIXME: merge with archive.cc. -void InputAccessor::dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter) -{ - auto dumpContents = [&](const CanonPath & path) - { - // FIXME: pipe - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); - }; - - std::function dump; - - dump = [&](const CanonPath & path) { - checkInterrupt(); - - auto st = lstat(path); - - sink << "("; - - if (st.type == tRegular) { - sink << "type" << "regular"; - if (st.isExecutable) - sink << "executable" << ""; - dumpContents(path); - } - - else if (st.type == tDirectory) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (/* archiveSettings.useCaseHack */ false) { // FIXME - std::string name(i.first); - size_t pos = i.first.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%s'", path + i.first); - name.erase(pos); - } - if (!unhacked.emplace(name, i.first).second) - throw Error("file name collision in between '%s' and '%s'", - (path + unhacked[name]), - (path + i.first)); - } else - unhacked.emplace(i.first, i.first); - - for (auto & i : unhacked) - if (filter((path + i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + i.second); - sink << ")"; - } - } - - else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%s' has an unsupported type", path); - - sink << ")"; - }; - - sink << narVersionMagic1; - dump(path); -} - -Hash InputAccessor::hashPath( - const CanonPath & path, - PathFilter & filter, - HashType ht) -{ - HashSink sink(ht); - dumpPath(path, sink, filter); - return sink.finish().first; -} - StorePath InputAccessor::fetchToStore( ref store, const CanonPath & path, @@ -114,19 +25,6 @@ StorePath InputAccessor::fetchToStore( return storePath; } -std::optional InputAccessor::maybeLstat(const CanonPath & path) -{ - // FIXME: merge these into one operation. - if (!pathExists(path)) - return {}; - return lstat(path); -} - -std::string InputAccessor::showPath(const CanonPath & path) -{ - return path.abs(); -} - SourcePath InputAccessor::root() { return {ref(shared_from_this()), CanonPath::root}; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 003c7226c..524a94cd1 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -1,11 +1,9 @@ #pragma once +#include "source-accessor.hh" #include "ref.hh" #include "types.hh" -#include "archive.hh" -#include "canon-path.hh" #include "repair-flag.hh" -#include "hash.hh" #include "content-address.hh" namespace nix { @@ -16,60 +14,8 @@ struct SourcePath; class StorePath; class Store; -struct InputAccessor : public std::enable_shared_from_this +struct InputAccessor : SourceAccessor, std::enable_shared_from_this { - const size_t number; - - InputAccessor(); - - virtual ~InputAccessor() - { } - - virtual std::string readFile(const CanonPath & path) = 0; - - virtual bool pathExists(const CanonPath & path) = 0; - - enum Type { - tRegular, tSymlink, tDirectory, - /** - Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. - - Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. - - Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. - */ - tMisc - }; - - struct Stat - { - Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only - }; - - virtual Stat lstat(const CanonPath & path) = 0; - - std::optional maybeLstat(const CanonPath & path); - - typedef std::optional DirEntry; - - typedef std::map DirEntries; - - virtual DirEntries readDirectory(const CanonPath & path) = 0; - - virtual std::string readLink(const CanonPath & path) = 0; - - virtual void dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter = defaultPathFilter); - - Hash hashPath( - const CanonPath & path, - PathFilter & filter = defaultPathFilter, - HashType ht = htSHA256); - StorePath fetchToStore( ref store, const CanonPath & path, @@ -78,34 +24,7 @@ struct InputAccessor : public std::enable_shared_from_this PathFilter * filter = nullptr, RepairFlag repair = NoRepair); - /* Return a corresponding path in the root filesystem, if - possible. This is only possible for inputs that are - materialized in the root filesystem. */ - virtual std::optional getPhysicalPath(const CanonPath & path) - { return std::nullopt; } - - bool operator == (const InputAccessor & x) const - { - return number == x.number; - } - - bool operator < (const InputAccessor & x) const - { - return number < x.number; - } - - void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); - - virtual std::string showPath(const CanonPath & path); - SourcePath root(); - - /* Return the maximum last-modified time of the files in this - tree, if available. */ - virtual std::optional getLastModified() - { - return std::nullopt; - } }; /** diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc new file mode 100644 index 000000000..23ec55c89 --- /dev/null +++ b/src/libutil/source-accessor.cc @@ -0,0 +1,108 @@ +#include "source-accessor.hh" +#include "archive.hh" + +namespace nix { + +static std::atomic nextNumber{0}; + +SourceAccessor::SourceAccessor() + : number(++nextNumber) +{ +} + +// FIXME: merge with archive.cc. +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) +{ + auto dumpContents = [&](const CanonPath & path) + { + // FIXME: pipe + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; + + std::function dump; + + dump = [&](const CanonPath & path) { + checkInterrupt(); + + auto st = lstat(path); + + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (/* archiveSettings.useCaseHack */ false) { // FIXME + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); +} + +Hash SourceAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashType ht) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +std::optional SourceAccessor::maybeLstat(const CanonPath & path) +{ + // FIXME: merge these into one operation. + if (!pathExists(path)) + return {}; + return lstat(path); +} + +std::string SourceAccessor::showPath(const CanonPath & path) +{ + return path.abs(); +} + +} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh new file mode 100644 index 000000000..c2d35d6b5 --- /dev/null +++ b/src/libutil/source-accessor.hh @@ -0,0 +1,96 @@ +#pragma once + +#include "canon-path.hh" +#include "hash.hh" + +namespace nix { + +/** + * A read-only filesystem abstraction. This is used by the Nix + * evaluator and elsewhere for accessing sources in various + * filesystem-like entities (such as the real filesystem, tarballs or + * Git repositories). + */ +struct SourceAccessor +{ + const size_t number; + + SourceAccessor(); + + virtual ~SourceAccessor() + { } + + virtual std::string readFile(const CanonPath & path) = 0; + + virtual bool pathExists(const CanonPath & path) = 0; + + enum Type { + tRegular, tSymlink, tDirectory, + /** + Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. + + Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. + + Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. + */ + tMisc + }; + + struct Stat + { + Type type = tMisc; + //uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + }; + + virtual Stat lstat(const CanonPath & path) = 0; + + std::optional maybeLstat(const CanonPath & path); + + typedef std::optional DirEntry; + + typedef std::map DirEntries; + + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashType ht = htSHA256); + + /* Return a corresponding path in the root filesystem, if + possible. This is only possible for filesystems that are + materialized in the root filesystem. */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const SourceAccessor & x) const + { + return number == x.number; + } + + bool operator < (const SourceAccessor & x) const + { + return number < x.number; + } + + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + + virtual std::string showPath(const CanonPath & path); + + /* Return the maximum last-modified time of the files in this + tree, if available. */ + virtual std::optional getLastModified() + { + return std::nullopt; + } +}; + +} From 9f572eb0e35d3017a160cef89b7417cf6b4a53e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 15:07:56 +0200 Subject: [PATCH 182/402] Unify the two implementations of dumpPath() --- src/libutil/archive.cc | 183 +++++++++++++++++++-------------- src/libutil/source-accessor.cc | 72 ------------- 2 files changed, 108 insertions(+), 147 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0f0cae7c0..e48fc7522 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -14,6 +14,7 @@ #include "archive.hh" #include "util.hh" #include "config.hh" +#include "source-accessor.hh" namespace nix { @@ -36,91 +37,134 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, off_t size, - Sink & sink) +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) { - sink << "contents" << size; + auto dumpContents = [&](const CanonPath & path) + { + /* It would be nice if this was streaming, but we need the + size before the contents. */ + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%1%'", path); + std::function dump; - std::vector buf(65536); - size_t left = size; + dump = [&](const CanonPath & path) { + checkInterrupt(); - while (left > 0) { - auto n = std::min(left, buf.size()); - readFull(fd.get(), buf.data(), n); - left -= n; - sink({buf.data(), n}); - } + auto st = lstat(path); - writePadding(size, sink); + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); } -static time_t dump(const Path & path, Sink & sink, PathFilter & filter) +struct FSSourceAccessor : SourceAccessor { - checkInterrupt(); + time_t mtime = 0; // most recent mtime seen - auto st = lstat(path); - time_t result = st.st_mtime; - - sink << "("; - - if (S_ISREG(st.st_mode)) { - sink << "type" << "regular"; - if (st.st_mode & S_IXUSR) - sink << "executable" << ""; - dumpContents(path, st.st_size, sink); + std::string readFile(const CanonPath & path) override + { + return nix::readFile(path.abs()); } - else if (S_ISDIR(st.st_mode)) { - sink << "type" << "directory"; + bool pathExists(const CanonPath & path) override + { + return nix::pathExists(path.abs()); + } - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (archiveSettings.useCaseHack) { - std::string name(i.name); - size_t pos = i.name.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%1%'", path + "/" + i.name); - name.erase(pos); - } - if (!unhacked.emplace(name, i.name).second) - throw Error("file name collision in between '%1%' and '%2%'", - (path + "/" + unhacked[name]), - (path + "/" + i.name)); - } else - unhacked.emplace(i.name, i.name); + Stat lstat(const CanonPath & path) override + { + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; + } - for (auto & i : unhacked) - if (filter(path + "/" + i.first)) { - sink << "entry" << "(" << "name" << i.first << "node"; - auto tmp_mtime = dump(path + "/" + i.second, sink, filter); - if (tmp_mtime > result) { - result = tmp_mtime; - } - sink << ")"; + DirEntries readDirectory(const CanonPath & path) override + { + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; } + res.emplace(entry.name, type); + } + return res; } - else if (S_ISLNK(st.st_mode)) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%1%' has an unsupported type", path); - - sink << ")"; - - return result; -} + std::string readLink(const CanonPath & path) override + { + return nix::readLink(path.abs()); + } +}; time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - sink << narVersionMagic1; - return dump(path, sink, filter); + FSSourceAccessor accessor; + accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); + return accessor.mtime; } void dumpPath(const Path & path, Sink & sink, PathFilter & filter) @@ -141,17 +185,6 @@ static SerialisationError badArchive(const std::string & s) } -#if 0 -static void skipGeneric(Source & source) -{ - if (readString(source) == "(") { - while (readString(source) != ")") - skipGeneric(source); - } -} -#endif - - static void parseContents(ParseSink & sink, Source & source, const Path & path) { uint64_t size = readLongLong(source); diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 23ec55c89..e3adee5f1 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,78 +10,6 @@ SourceAccessor::SourceAccessor() { } -// FIXME: merge with archive.cc. -void SourceAccessor::dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter) -{ - auto dumpContents = [&](const CanonPath & path) - { - // FIXME: pipe - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); - }; - - std::function dump; - - dump = [&](const CanonPath & path) { - checkInterrupt(); - - auto st = lstat(path); - - sink << "("; - - if (st.type == tRegular) { - sink << "type" << "regular"; - if (st.isExecutable) - sink << "executable" << ""; - dumpContents(path); - } - - else if (st.type == tDirectory) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (/* archiveSettings.useCaseHack */ false) { // FIXME - std::string name(i.first); - size_t pos = i.first.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%s'", path + i.first); - name.erase(pos); - } - if (!unhacked.emplace(name, i.first).second) - throw Error("file name collision in between '%s' and '%s'", - (path + unhacked[name]), - (path + i.first)); - } else - unhacked.emplace(i.first, i.first); - - for (auto & i : unhacked) - if (filter((path + i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + i.second); - sink << ")"; - } - } - - else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%s' has an unsupported type", path); - - sink << ")"; - }; - - sink << narVersionMagic1; - dump(path); -} - Hash SourceAccessor::hashPath( const CanonPath & path, PathFilter & filter, From 50156302c033323895ababb6c190086cbc5cec46 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 15:20:10 +0200 Subject: [PATCH 183/402] Deduplicate FSSourceAccessor and FSInputAccessor --- src/libfetchers/fs-input-accessor.cc | 31 +++++------------ src/libutil/archive.cc | 52 +--------------------------- src/libutil/source-accessor.cc | 49 ++++++++++++++++++++++++++ src/libutil/source-accessor.hh | 24 +++++++++++++ 4 files changed, 82 insertions(+), 74 deletions(-) diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index a35955465..e40faf03f 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -3,7 +3,7 @@ namespace nix { -struct FSInputAccessorImpl : FSInputAccessor +struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor { CanonPath root; std::optional> allowedPaths; @@ -23,28 +23,20 @@ struct FSInputAccessorImpl : FSInputAccessor { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return nix::readFile(absPath.abs()); + return PosixSourceAccessor::readFile(absPath); } bool pathExists(const CanonPath & path) override { auto absPath = makeAbsPath(path); - return isAllowed(absPath) && nix::pathExists(absPath.abs()); + return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); } Stat lstat(const CanonPath & path) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - auto st = nix::lstat(absPath.abs()); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; + return PosixSourceAccessor::lstat(absPath); } DirEntries readDirectory(const CanonPath & path) override @@ -52,16 +44,9 @@ struct FSInputAccessorImpl : FSInputAccessor auto absPath = makeAbsPath(path); checkAllowed(absPath); DirEntries res; - for (auto & entry : nix::readDirectory(absPath.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - if (isAllowed(absPath + entry.name)) - res.emplace(entry.name, type); - } + for (auto & entry : PosixSourceAccessor::readDirectory(absPath)) + if (isAllowed(absPath + entry.first)) + res.emplace(entry); return res; } @@ -69,7 +54,7 @@ struct FSInputAccessorImpl : FSInputAccessor { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return nix::readLink(absPath.abs()); + return PosixSourceAccessor::readLink(absPath); } CanonPath makeAbsPath(const CanonPath & path) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index e48fc7522..6508ba807 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -110,59 +110,9 @@ void SourceAccessor::dumpPath( } -struct FSSourceAccessor : SourceAccessor -{ - time_t mtime = 0; // most recent mtime seen - - std::string readFile(const CanonPath & path) override - { - return nix::readFile(path.abs()); - } - - bool pathExists(const CanonPath & path) override - { - return nix::pathExists(path.abs()); - } - - Stat lstat(const CanonPath & path) override - { - auto st = nix::lstat(path.abs()); - mtime = std::max(mtime, st.st_mtime); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; - } - - DirEntries readDirectory(const CanonPath & path) override - { - DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; - } - - std::string readLink(const CanonPath & path) override - { - return nix::readLink(path.abs()); - } -}; - - time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - FSSourceAccessor accessor; + PosixSourceAccessor accessor; accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); return accessor.mtime; } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index e3adee5f1..d5c8cbcdd 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -33,4 +33,53 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } +std::string PosixSourceAccessor::readFile(const CanonPath & path) +{ + return nix::readFile(path.abs()); +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + return nix::pathExists(path.abs()); +} + +SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +{ + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + return nix::readLink(path.abs()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return path; +} + } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index c2d35d6b5..a251ae31c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -93,4 +93,28 @@ struct SourceAccessor } }; +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : SourceAccessor +{ + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + std::string readFile(const CanonPath & path) override; + + bool pathExists(const CanonPath & path) override; + + Stat lstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; +}; + } From 5be7705ddf9397846ca9391d464af1cc27110f06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 19:20:21 +0200 Subject: [PATCH 184/402] Remove stuff we don't need yet --- src/libexpr/eval.cc | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7180b9fbe..8235b83c5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -505,18 +505,7 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) - , rootFS( - makeFSInputAccessor( - CanonPath::root, - evalSettings.restrictEval || evalSettings.pureEval - ? std::optional>(std::set()) - : std::nullopt, - [](const CanonPath & path) -> RestrictedPathError { - auto modeInformation = evalSettings.pureEval - ? "in pure evaluation mode (use '--impure' to override)" - : "in restricted mode"; - throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); - })) + , rootFS(makeFSInputAccessor(CanonPath::root)) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -541,9 +530,6 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { - // For now, don't rely on FSInputAccessor for access control. - rootFS->allowPath(CanonPath::root); - countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); From 87c4f4a972ce732962cc0881d848d8bf979893c2 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:24:13 +0100 Subject: [PATCH 185/402] libutil: Move some non-template implememntations from config.hh to config.cc --- src/libutil/config.cc | 45 +++++++++++++++++++++++++++++++++++++++++++ src/libutil/config.hh | 31 +++++++---------------------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8672edaa8..f5c7fb7d5 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -9,6 +9,10 @@ namespace nix { +Config::Config(StringMap initials) + : AbstractConfig(std::move(initials)) +{ } + bool Config::set(const std::string & name, const std::string & value) { bool append = false; @@ -59,6 +63,10 @@ void Config::addSetting(AbstractSetting * setting) } } +AbstractConfig::AbstractConfig(StringMap initials) + : unknownSettings(std::move(initials)) +{ } + void AbstractConfig::warnUnknownSettings() { for (auto & s : unknownSettings) @@ -199,6 +207,13 @@ AbstractSetting::AbstractSetting( { } +AbstractSetting::~AbstractSetting() +{ + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); +} + nlohmann::json AbstractSetting::toJSON() { return nlohmann::json(toJSONObject()); @@ -220,6 +235,9 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } + +bool AbstractSetting::isOverridden() const { return overridden; } + template<> std::string BaseSetting::parse(const std::string & str) const { return str; @@ -385,11 +403,33 @@ static Path parsePath(const AbstractSetting & s, const std::string & str) return canonPath(str); } +PathSetting::PathSetting(Config * options, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting(def, true, name, description, aliases) +{ + options->addSetting(this); +} + Path PathSetting::parse(const std::string & str) const { return parsePath(*this, str); } + +OptionalPathSetting::OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting>(def, true, name, description, aliases) +{ + options->addSetting(this); +} + + std::optional OptionalPathSetting::parse(const std::string & str) const { if (str == "") @@ -398,6 +438,11 @@ std::optional OptionalPathSetting::parse(const std::string & str) const return parsePath(*this, str); } +void OptionalPathSetting::operator =(const std::optional & v) +{ + this->assign(v); +} + bool GlobalConfig::set(const std::string & name, const std::string & value) { for (auto & config : *configRegistrations) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 96c2cd75d..47c7be473 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -52,9 +52,7 @@ class AbstractConfig protected: StringMap unknownSettings; - AbstractConfig(const StringMap & initials = {}) - : unknownSettings(initials) - { } + AbstractConfig(StringMap initials = {}); public: @@ -163,9 +161,7 @@ private: public: - Config(const StringMap & initials = {}) - : AbstractConfig(initials) - { } + Config(StringMap initials = {}); bool set(const std::string & name, const std::string & value) override; @@ -206,12 +202,7 @@ protected: const std::set & aliases, std::optional experimentalFeature = std::nullopt); - virtual ~AbstractSetting() - { - // Check against a gcc miscompilation causing our constructor - // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). - assert(created == 123); - } + virtual ~AbstractSetting(); virtual void set(const std::string & value, bool append = false) = 0; @@ -229,7 +220,7 @@ protected: virtual void convertToArg(Args & args, const std::string & category); - bool isOverridden() const { return overridden; } + bool isOverridden() const; }; /** @@ -365,11 +356,7 @@ public: const Path & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); Path parse(const std::string & str) const override; @@ -391,15 +378,11 @@ public: const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting>(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); std::optional parse(const std::string & str) const override; - void operator =(const std::optional & v) { this->assign(v); } + void operator =(const std::optional & v); }; struct GlobalConfig : public AbstractConfig From 55f06b6f30fa212d48cfd6d121a845863fb2980b Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:24:54 +0100 Subject: [PATCH 186/402] libutil: Remove non-needed constructor --- src/libutil/config.cc | 4 ++-- src/libutil/config.hh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index f5c7fb7d5..8e06273ee 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -33,9 +33,9 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { - _settings.emplace(setting->name, Config::SettingData(false, setting)); + _settings.emplace(setting->name, Config::SettingData{false, setting}); for (auto & alias : setting->aliases) - _settings.emplace(alias, Config::SettingData(true, setting)); + _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 47c7be473..f09a9b0b8 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -148,9 +148,6 @@ public: { bool isAlias; AbstractSetting * setting; - SettingData(bool isAlias, AbstractSetting * setting) - : isAlias(isAlias), setting(setting) - { } }; typedef std::map Settings; From b0f4ac29d3c0534b91fff7b37191d4d7d5a96395 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:25:17 +0100 Subject: [PATCH 187/402] libutil: Use c++ style cast --- src/libutil/config.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index f09a9b0b8..38c3ce0c4 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -312,8 +312,7 @@ public: template std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) { - str << (const T &) opt; - return str; + return str << static_cast(opt); } template From 42f26eb42e3b115f39570bb14445d7e5f33220d7 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 20 Oct 2023 02:45:47 +0200 Subject: [PATCH 188/402] doc: complexity for '?' operator (#9184) Co-authored-by: Valentin Gagarin Co-authored-by: Robert Hensing --- doc/manual/src/language/operators.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index a22c71109..07b43a881 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -63,6 +63,8 @@ The result is a [Boolean] value. [Has attribute]: #has-attribute +After evaluating *attrset* and *attrpath*, the computational complexity is O(log(*n*)) for *n* attributes in the *attrset* + ## Arithmetic Numbers are type-compatible: From e58566a057692bbfbd30a6639248adfcf81df108 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 20 Oct 2023 05:11:03 +0200 Subject: [PATCH 189/402] doc: add reference to hasAttr in `?` operator (#9185) Co-authored-by: Valentin Gagarin --- doc/manual/src/language/operators.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 07b43a881..cc825b4cf 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -59,6 +59,8 @@ An attribute path is a dot-separated list of [attribute names](./values.md#attri Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. The result is a [Boolean] value. +See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) + [Boolean]: ./values.md#type-boolean [Has attribute]: #has-attribute From 9277eb276bf0a942e88fcf499f6a6b9c262be853 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 19 Oct 2023 16:59:35 +0200 Subject: [PATCH 190/402] libstore: Add apple-virt to system features when available I'm sure that we'll adjust the implementation over time, but this at least discerns between an apple silicon bare metal machine and a tart VM. --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libstore/globals.cc | 31 +++++++++++++++++++++++++ src/libstore/globals.hh | 4 ++++ 3 files changed, 37 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 46ce2abac..d0b516dfb 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,6 +8,8 @@ - Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). - `builtins.fetchTree` is now marked as stable. diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5a4cb1824..9c25d9868 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -24,6 +24,9 @@ #include "config-impl.hh" +#ifdef __APPLE__ +#include +#endif namespace nix { @@ -154,6 +157,29 @@ unsigned int Settings::getDefaultCores() return concurrency; } +#if __APPLE__ +static bool hasVirt() { + + int hasVMM; + int hvSupport; + size_t size; + + size = sizeof(hasVMM); + if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) { + if (hasVMM) + return false; + } + + // whether the kernel and hardware supports virt + size = sizeof(hvSupport); + if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) { + return hvSupport == 1; + } else { + return false; + } +} +#endif + StringSet Settings::getDefaultSystemFeatures() { /* For backwards compatibility, accept some "features" that are @@ -170,6 +196,11 @@ StringSet Settings::getDefaultSystemFeatures() features.insert("kvm"); #endif + #if __APPLE__ + if (hasVirt()) + features.insert("apple-virt"); + #endif + return features; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 5ec332417..ff7d3df1e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -714,6 +714,10 @@ public: System features are user-defined, but Nix sets the following defaults: + - `apple-virt` + + Included on darwin if virtualization is available. + - `kvm` Included by default if `/dev/kvm` is accessible. From bb645c5d02025158c8dbb0c2a60e7cba8251f062 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Oct 2023 10:19:07 +0200 Subject: [PATCH 191/402] system-features doc: kvm is Linux-only --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ff7d3df1e..e90f70f5f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -720,7 +720,7 @@ public: - `kvm` - Included by default if `/dev/kvm` is accessible. + Included on Linux if `/dev/kvm` is accessible. - `nixos-test`, `benchmark`, `big-parallel` From df10dc630fd13082e13400ec46acefe4ad85a715 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 12:36:18 +0200 Subject: [PATCH 192/402] Doxygen Co-authored-by: John Ericson --- src/libutil/source-accessor.hh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index a251ae31c..4074b2e63 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -65,9 +65,11 @@ struct SourceAccessor PathFilter & filter = defaultPathFilter, HashType ht = htSHA256); - /* Return a corresponding path in the root filesystem, if - possible. This is only possible for filesystems that are - materialized in the root filesystem. */ + /** + * Return a corresponding path in the root filesystem, if + * possible. This is only possible for filesystems that are + * materialized in the root filesystem. + */ virtual std::optional getPhysicalPath(const CanonPath & path) { return std::nullopt; } From bacceaea919ddb26d1882fec16462cdcb4e1d740 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 12:40:46 +0200 Subject: [PATCH 193/402] Move getLastModified(), remove setPathDisplay() --- src/libfetchers/input-accessor.hh | 9 +++++++++ src/libutil/source-accessor.hh | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 524a94cd1..5dc05a363 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -16,6 +16,15 @@ class Store; struct InputAccessor : SourceAccessor, std::enable_shared_from_this { + /** + * Return the maximum last-modified time of the files in this + * tree, if available. + */ + virtual std::optional getLastModified() + { + return std::nullopt; + } + StorePath fetchToStore( ref store, const CanonPath & path, diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 4074b2e63..53408eb6c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -83,16 +83,7 @@ struct SourceAccessor return number < x.number; } - void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); - virtual std::string showPath(const CanonPath & path); - - /* Return the maximum last-modified time of the files in this - tree, if available. */ - virtual std::optional getLastModified() - { - return std::nullopt; - } }; /** From 173abec0bce03016b001c415822793a309c40e0b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 13:04:39 +0200 Subject: [PATCH 194/402] coerceToPath(): Handle __toString, add tests --- src/libexpr/eval.cc | 27 ++++++++++--------- src/libexpr/tests/error_traces.cc | 8 +++--- .../functional/lang/eval-okay-pathexists.nix | 2 ++ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8235b83c5..e3ecb987c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2335,29 +2335,32 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext { try { forceValue(v, pos); - - if (v.type() == nString) { - copyContext(v, context); - auto s = v.string_view(); - if (!hasPrefix(s, "/")) - error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow(); - return rootPath(CanonPath(s)); - } } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } + /* Handle path values directly, without coercing to a string. */ if (v.type() == nPath) return v.path(); + /* Similarly, handle __toString where the result may be a path + value. */ if (v.type() == nAttrs) { - auto i = v.attrs->find(sOutPath); - if (i != v.attrs->end()) - return coerceToPath(pos, *i->value, context, errorCtx); + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToPath(pos, v1, context, errorCtx); + } } - error("cannot coerce %1% to a path", showType(v)).withTrace(pos, errorCtx).debugThrow(); + /* Any other value should be coercable to a string, interpreted + relative to the root filesystem. */ + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); + if (path == "" || path[0] != '/') + error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); + return rootPath(CanonPath(path)); } diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 6da8a5954..9cd9f0a60 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,7 +309,7 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a path", "a Boolean"), + hintfmt("cannot coerce %s to a string", "a Boolean"), hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -377,7 +377,7 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix index c5e7a62de..31697f66a 100644 --- a/tests/functional/lang/eval-okay-pathexists.nix +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -25,5 +25,7 @@ builtins.pathExists (./lib.nix) && builtins.pathExists (builtins.toString ./. + "/../lang/..//") && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) +&& builtins.pathExists (builtins.toPath { __toString = x: builtins.toString ./lib.nix; }) +&& builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; }) && builtins.pathExists ./lib.nix && !builtins.pathExists ./bla.nix From 7a086a32bce3338bdc1ce669a51599a99bc17b9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 13:32:15 +0200 Subject: [PATCH 195/402] fetchToStore(): Handle flat ingestion method and add test --- src/libfetchers/input-accessor.cc | 5 ++++- tests/functional/lang/eval-okay-path.exp | 2 +- tests/functional/lang/eval-okay-path.nix | 22 +++++++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 9bddf7c2e..488350849 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -14,7 +14,10 @@ StorePath InputAccessor::fetchToStore( Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); auto source = sinkToSource([&](Sink & sink) { - dumpPath(path, sink, filter ? *filter : defaultPathFilter); + if (method == FileIngestionMethod::Recursive) + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + else + sink(readFile(path)); // FIXME: stream }); auto storePath = diff --git a/tests/functional/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp index 3ce7f8283..635e2243a 100644 --- a/tests/functional/lang/eval-okay-path.exp +++ b/tests/functional/lang/eval-okay-path.exp @@ -1 +1 @@ -"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" +[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ] diff --git a/tests/functional/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix index e67168cf3..599b33541 100644 --- a/tests/functional/lang/eval-okay-path.nix +++ b/tests/functional/lang/eval-okay-path.nix @@ -1,7 +1,15 @@ -builtins.path - { path = ./.; - filter = path: _: baseNameOf path == "data"; - recursive = true; - sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; - name = "output"; - } +[ + (builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + }) + (builtins.path + { path = ./data; + recursive = false; + sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm"; + name = "output"; + }) +] From 57db3be9e448042814d386def2d8af16a2a4c4b9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 16:36:41 +0200 Subject: [PATCH 196/402] SourceAccessor::readFile(): Support reading into a sink --- src/libfetchers/fs-input-accessor.cc | 7 +++- src/libfetchers/input-accessor.cc | 2 +- src/libutil/archive.cc | 15 ++++--- src/libutil/source-accessor.cc | 58 +++++++++++++++++++++++++++- src/libutil/source-accessor.hh | 25 +++++++++++- 5 files changed, 94 insertions(+), 13 deletions(-) diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index e40faf03f..3444c4643 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -19,11 +19,14 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor { } - std::string readFile(const CanonPath & path) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return PosixSourceAccessor::readFile(absPath); + PosixSourceAccessor::readFile(absPath, sink, sizeCallback); } bool pathExists(const CanonPath & path) override diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 488350849..d1d450cf7 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -17,7 +17,7 @@ StorePath InputAccessor::fetchToStore( if (method == FileIngestionMethod::Recursive) dumpPath(path, sink, filter ? *filter : defaultPathFilter); else - sink(readFile(path)); // FIXME: stream + readFile(path, sink); }); auto storePath = diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6508ba807..0cd54e5db 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -44,12 +44,15 @@ void SourceAccessor::dumpPath( { auto dumpContents = [&](const CanonPath & path) { - /* It would be nice if this was streaming, but we need the - size before the contents. */ - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); + sink << "contents"; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + sink << _size; + }); + assert(size); + writePadding(*size, sink); }; std::function dump; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d5c8cbcdd..2d03d3d7a 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,6 +10,28 @@ SourceAccessor::SourceAccessor() { } +std::string SourceAccessor::readFile(const CanonPath & path) +{ + StringSink sink; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + }); + assert(size && *size == sink.s.size()); + return std::move(sink.s); +} + +void SourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + auto s = readFile(path); + sizeCallback(s.size()); + sink(s); +} + Hash SourceAccessor::hashPath( const CanonPath & path, PathFilter & filter, @@ -33,9 +55,41 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } -std::string PosixSourceAccessor::readFile(const CanonPath & path) +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) { - return nix::readFile(path.abs()); + // FIXME: add O_NOFOLLOW since symlinks should be resolved by the + // caller? + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::vector buf(64 * 1024); + while (left) { + checkInterrupt(); + ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } } bool PosixSourceAccessor::pathExists(const CanonPath & path) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 53408eb6c..f3504c9bb 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -5,6 +5,8 @@ namespace nix { +struct Sink; + /** * A read-only filesystem abstraction. This is used by the Nix * evaluator and elsewhere for accessing sources in various @@ -20,7 +22,23 @@ struct SourceAccessor virtual ~SourceAccessor() { } - virtual std::string readFile(const CanonPath & path) = 0; + /** + * Return the contents of a file as a string. + */ + virtual std::string readFile(const CanonPath & path); + + /** + * Write the contents of a file as a sink. `sizeCallback` must be + * called with the size of the file before any data is written to + * the sink. + * + * Note: subclasses of `SourceAccessor` need to implement at least + * one of the `readFile()` variants. + */ + virtual void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback = [](uint64_t size){}); virtual bool pathExists(const CanonPath & path) = 0; @@ -97,7 +115,10 @@ struct PosixSourceAccessor : SourceAccessor */ time_t mtime = 0; - std::string readFile(const CanonPath & path) override; + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; bool pathExists(const CanonPath & path) override; From bcf5c31950a01d73c17cee301d3dd363c91cd23d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 16:58:33 +0200 Subject: [PATCH 197/402] Add future FIXME --- src/libexpr/eval.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e3ecb987c..d26cde423 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2461,6 +2461,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nPath: return + // FIXME: compare accessors by their fingerprint. v1._path.accessor == v2._path.accessor && strcmp(v1._path.path, v2._path.path) == 0; From af302267e5e02c8b373779fac979f20e094e7cfa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 17:18:42 +0200 Subject: [PATCH 198/402] Input::hasAllInfo(): Remove --- src/libfetchers/fetchers.cc | 9 +-------- src/libfetchers/fetchers.hh | 11 ----------- src/libfetchers/git.cc | 9 --------- src/libfetchers/github.cc | 5 ----- src/libfetchers/indirect.cc | 5 ----- src/libfetchers/mercurial.cc | 7 ------- src/libfetchers/path.cc | 5 ----- src/libfetchers/tarball.cc | 6 ------ 8 files changed, 1 insertion(+), 56 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c54c39cf0..000609f09 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -89,11 +89,6 @@ Attrs Input::toAttrs() const return attrs; } -bool Input::hasAllInfo() const -{ - return getNarHash() && scheme && scheme->hasAllInfo(*this); -} - bool Input::operator ==(const Input & other) const { return attrs == other.attrs; @@ -117,7 +112,7 @@ std::pair Input::fetch(ref store) const /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (hasAllInfo()) { + if (getNarHash()) { try { auto storePath = computeStorePath(*store); @@ -175,8 +170,6 @@ std::pair Input::fetch(ref store) const input.locked = true; - assert(input.hasAllInfo()); - return {std::move(tree), input}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b55aabb6f..f52175e41 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -79,15 +79,6 @@ public: */ bool isLocked() const { return locked; } - /** - * Check whether the input carries all necessary info required - * for cache insertion and substitution. - * These fields are used to uniquely identify cached trees - * within the "tarball TTL" window without necessarily - * indicating that the input's origin is unchanged. - */ - bool hasAllInfo() const; - bool operator ==(const Input & other) const; bool contains(const Input & other) const; @@ -144,8 +135,6 @@ struct InputScheme virtual ParsedURL toURL(const Input & input) const; - virtual bool hasAllInfo(const Input & input) const = 0; - virtual Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c1a6dce43..26b8987d6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -321,15 +321,6 @@ struct GitInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - bool maybeDirty = !input.getRef(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - return - maybeGetIntAttr(input.attrs, "lastModified") - && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); - } - Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index d7450defe..b824140a6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -132,11 +132,6 @@ struct GitArchiveInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override - { - return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); - } - Input applyOverrides( const Input & _input, std::optional ref, diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index e09d8bfb8..947849802 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -78,11 +78,6 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - return false; - } - Input applyOverrides( const Input & _input, std::optional ref, diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index b0d2d1909..f830a3271 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -98,13 +98,6 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - // FIXME: ugly, need to distinguish between dirty and clean - // default trees. - return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); - } - Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 092975f5d..d829609b5 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -66,11 +66,6 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override - { - return true; - } - std::optional getSourcePath(const Input & input) override { return getStrAttr(input.attrs, "path"); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 269a56526..ba47d59a7 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -253,12 +253,6 @@ struct CurlInputScheme : InputScheme url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } - - bool hasAllInfo(const Input & input) const override - { - return true; - } - }; struct FileInputScheme : CurlInputScheme From 3e6b9f9357e4573740a4d9e477c4f4fa69c74fa5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:17:33 -0400 Subject: [PATCH 199/402] Remove `prevInfos` as its dead code It is unused since 8e0946e8df968391d1430af8377bdb51204e4666 removed support for the repeat and enforce-determinism options. --- src/libstore/build/local-derivation-goal.hh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 0a05081c7..8191af7a6 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -120,14 +120,6 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; - /** - * Path registration info from the previous round, if we're - * building multiple times. Since this contains the hash, it - * allows us to compare whether two rounds produced the same - * result. - */ - std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } From 862d16436b7606bba9064e2bf2a4c48bd883bb42 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:22:34 -0400 Subject: [PATCH 200/402] Remove the `ValidPathInfo` `==` operator It is dead code. It was added in 8e0946e8df968391d1430af8377bdb51204e4666 as part of the repeated / enforce-determinism feature, but that was removed in 8fdd156a650f9b2ce9ae8cd74edcf16225478292. It is not good because it skips many fields. For testing purposes we will soon want to add a new one that doesn't skip fields, but we want to make sure making == sensitive to those fields won't change how Nix works. Proving in this commit that the old version is dead code achieves that. --- src/libstore/path-info.hh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 221523622..fea6d0e5f 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,14 +72,6 @@ struct ValidPathInfo */ std::optional ca; - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } - /** * Return a fingerprint of the store path to be used in binary * cache signatures. It contains the store path, the base-32 From 0f7e9d051340531332b7c6854675006cc8a16f50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 19:13:35 +0200 Subject: [PATCH 201/402] Input: Remove 'direct' field --- src/libfetchers/fetchers.cc | 5 +++++ src/libfetchers/fetchers.hh | 6 ++++-- src/libfetchers/indirect.cc | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 000609f09..5f6167868 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -84,6 +84,11 @@ std::string Input::to_string() const return toURL().to_string(); } +bool Input::isDirect() const +{ + return !scheme || scheme->isDirect(*this); +} + Attrs Input::toAttrs() const { return attrs; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index f52175e41..4119dd17b 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -35,7 +35,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; bool locked = false; - bool direct = true; /** * path of the parent of this input, used for relative path resolution @@ -71,7 +70,7 @@ public: * Check whether this is a "direct" input, that is, not * one that goes through a registry. */ - bool isDirect() const { return direct; } + bool isDirect() const; /** * Check whether this is a "locked" input, that is, @@ -152,6 +151,9 @@ struct InputScheme * Is this `InputScheme` part of an experimental feature? */ virtual std::optional experimentalFeature(); + + virtual bool isDirect(const Input & input) const + { return true; } }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 947849802..9a71df3d4 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -41,7 +41,6 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? Input input; - input.direct = false; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -63,7 +62,6 @@ struct IndirectInputScheme : InputScheme throw BadURL("'%s' is not a valid flake ID", id); Input input; - input.direct = false; input.attrs = attrs; return input; } @@ -98,6 +96,9 @@ struct IndirectInputScheme : InputScheme { return Xp::Flakes; } + + bool isDirect(const Input & input) const override + { return false; } }; static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From 85e5ac403fa9adb82105664bb6c4df501d4a0a25 Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Fri, 20 Oct 2023 10:28:26 -0700 Subject: [PATCH 202/402] docker: publish images to ghcr.io (#8066) * docker: publish images to ghcr.io docker.com announced their intention to remove the free plan used by OSS. The nixos/nix image is essential to various CI runs to build with nix. To provide a continuity plan, this commit pushes the image to ghcr.io as well. Co-authored-by: Sandro --- .github/workflows/ci.yml | 23 +++++++++++++++++++ .../src/installation/installing-docker.md | 8 +++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9c24dad..afe4dc2e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,9 @@ jobs: docker_push_image: needs: [check_secrets, tests] + permissions: + contents: read + packages: write if: >- github.event_name == 'push' && github.ref_name == 'master' && @@ -126,6 +129,9 @@ jobs: - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master + # We'll deploy the newly built image to both Docker Hub and Github Container Registry. + # + # Push to Docker Hub first - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -133,3 +139,20 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - run: docker push nixos/nix:$NIX_VERSION - run: docker push nixos/nix:master + # Push to GitHub Container Registry as well + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION + docker tag nix:$NIX_VERSION $IMAGE_ID:master + docker push $IMAGE_ID:$NIX_VERSION + docker push $IMAGE_ID:master diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 9d6d8f2d9..6f77d6a57 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -3,14 +3,14 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker run -ti nixos/nix -Unable to find image 'nixos/nix:latest' locally -latest: Pulling from nixos/nix +$ docker run -ti ghcr.io/nixos/nix +Unable to find image 'ghcr.io/nixos/nix:latest' locally +latest: Pulling from ghcr.io/nixos/nix 5843afab3874: Pull complete b52bf13f109c: Pull complete 1e2415612aa3: Pull complete Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff -Status: Downloaded newer image for nixos/nix:latest +Status: Downloaded newer image for ghcr.io/nixos/nix:latest 35ca4ada6e96:/# nix --version nix (Nix) 2.3.12 35ca4ada6e96:/# exit From 935c9981de03248fd5687c5d0363f572af723144 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 19:50:21 +0200 Subject: [PATCH 203/402] Remove fetchers::Tree and move tarball-related stuff into its own header --- src/libcmd/common-eval-args.cc | 5 ++-- src/libexpr/flake/flake.cc | 34 ++++++++++----------- src/libexpr/flake/flake.hh | 6 ++-- src/libexpr/flake/flakeref.cc | 6 ++-- src/libexpr/flake/flakeref.hh | 2 +- src/libexpr/parser.y | 6 ++-- src/libexpr/primops/fetchMercurial.cc | 6 ++-- src/libexpr/primops/fetchTree.cc | 13 ++++---- src/libfetchers/fetchers.cc | 18 +++++------ src/libfetchers/fetchers.hh | 41 ++----------------------- src/libfetchers/github.cc | 5 ++-- src/libfetchers/registry.cc | 2 +- src/libfetchers/tarball.cc | 7 +++-- src/libfetchers/tarball.hh | 43 +++++++++++++++++++++++++++ src/nix-channel/nix-channel.cc | 2 +- src/nix/flake.cc | 18 +++++------ 16 files changed, 111 insertions(+), 103 deletions(-) create mode 100644 src/libfetchers/tarball.hh diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e36bda52f..e6df49a7a 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,6 +9,7 @@ #include "flake/flakeref.hh" #include "store-api.hh" #include "command.hh" +#include "tarball.hh" namespace nix { @@ -168,14 +169,14 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index a6212c12f..b32c7c086 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -15,7 +15,7 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +typedef std::pair FetchedFlake; typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( @@ -34,7 +34,7 @@ static std::optional lookupInFlakeCache( return std::nullopt; } -static std::tuple fetchOrSubstituteTree( +static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, bool allowLookup, @@ -61,16 +61,16 @@ static std::tuple fetchOrSubstituteTree( flakeCache.push_back({originalRef, *fetched}); } - auto [tree, lockedRef] = *fetched; + auto [storePath, lockedRef] = *fetched; debug("got tree '%s' from '%s'", - state.store->printStorePath(tree.storePath), lockedRef); + state.store->printStorePath(storePath), lockedRef); - state.allowPath(tree.storePath); + state.allowPath(storePath); - assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(tree), resolvedRef, lockedRef}; + return {std::move(storePath), resolvedRef, lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -202,21 +202,21 @@ static Flake getFlake( FlakeCache & flakeCache, InputPath lockRootPath) { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. - auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); + auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true); - if (!isInDir(flakeFile, sourceInfo.actualPath)) + if (!isInDir(flakeFile, state.store->toRealPath(storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(sourceInfo.storePath)); + lockedRef, state.store->printStorePath(storePath)); Flake flake { .originalRef = originalRef, .resolvedRef = resolvedRef, .lockedRef = lockedRef, - .sourceInfo = std::make_shared(std::move(sourceInfo)) + .storePath = storePath, }; if (!pathExists(flakeFile)) @@ -346,7 +346,7 @@ LockedFlake lockFlake( // FIXME: symlink attack auto oldLockFile = LockFile::read( lockFlags.referenceLockFilePath.value_or( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); + state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); @@ -574,7 +574,7 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast(oldLock) : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -598,7 +598,7 @@ LockedFlake lockFlake( }; // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); + auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true); computeLocks( flake.inputs, @@ -729,7 +729,7 @@ void callFlake(EvalState & state, emitTreeAttrs( state, - *lockedFlake.flake.sourceInfo, + lockedFlake.flake.storePath, lockedFlake.flake.lockedRef.input, *vRootSrc, false, @@ -893,7 +893,7 @@ Fingerprint LockedFlake::getFingerprint() const // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), + flake.storePath.to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c1d1b71e5..d5ad3eade 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -10,8 +10,6 @@ namespace nix { class EvalState; -namespace fetchers { struct Tree; } - namespace flake { struct FlakeInput; @@ -84,7 +82,7 @@ struct Flake */ bool forceDirty = false; std::optional description; - std::shared_ptr sourceInfo; + StorePath storePath; FlakeInputs inputs; /** * 'nixConfig' attribute @@ -193,7 +191,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback = false, diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 93ad33a33..16f45ace7 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -272,10 +272,10 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } -std::pair FlakeRef::fetchTree(ref store) const +std::pair FlakeRef::fetchTree(ref store) const { - auto [tree, lockedInput] = input.fetch(store); - return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; + auto [storePath, lockedInput] = input.fetch(store); + return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index ffb2e50de..5d78f49b6 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -63,7 +63,7 @@ struct FlakeRef static FlakeRef fromAttrs(const fetchers::Attrs & attrs); - std::pair fetchTree(ref store) const; + std::pair fetchTree(ref store) const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 70228e1e2..8b73bd056 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -646,7 +646,7 @@ formal #include "eval.hh" #include "filetransfer.hh" -#include "fetchers.hh" +#include "tarball.hh" #include "store-api.hh" #include "flake/flake.hh" @@ -783,7 +783,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath; res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ @@ -797,7 +797,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + auto storePath = flakeRef.resolve(store).fetchTree(store).first; res = { store->toRealPath(storePath) }; } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index b9ff01c16..e76ce455d 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -71,10 +71,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to @@ -86,7 +86,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("revCount").mkInt(*revCount); v.mkAttrs(attrs2); - state.allowPath(tree.storePath); + state.allowPath(storePath); } static RegisterPrimOp r_fetchMercurial({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 976037ff9..a99b0e500 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -5,6 +5,7 @@ #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "tarball.hh" #include "url.hh" #include @@ -15,7 +16,7 @@ namespace nix { void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback, @@ -25,7 +26,7 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -165,11 +166,11 @@ static void fetchTree( state.checkURI(input.toURLString()); - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetch(state.store); - state.allowPath(tree.storePath); + state.allowPath(storePath); - emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); + emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -288,7 +289,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5f6167868..5688c4dc1 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -109,7 +109,7 @@ bool Input::contains(const Input & other) const return false; } -std::pair Input::fetch(ref store) const +std::pair Input::fetch(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); @@ -126,7 +126,7 @@ std::pair Input::fetch(ref store) const debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; + return {std::move(storePath), *this}; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } @@ -141,18 +141,16 @@ std::pair Input::fetch(ref store) const } }(); - Tree tree { - .actualPath = store->toRealPath(storePath), - .storePath = storePath, - }; - - auto narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(HashFormat::SRI, true), narHash.to_string(HashFormat::SRI, true)); + to_string(), + store->printStorePath(storePath), + prevNarHash->to_string(HashFormat::SRI, true), + narHash.to_string(HashFormat::SRI, true)); } if (auto prevLastModified = getLastModified()) { @@ -175,7 +173,7 @@ std::pair Input::fetch(ref store) const input.locked = true; - return {std::move(tree), input}; + return {std::move(storePath), input}; } Input Input::applyOverrides( diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 4119dd17b..ac605ff8e 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -13,12 +13,6 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Tree -{ - Path actualPath; - StorePath storePath; -}; - struct InputScheme; /** @@ -83,10 +77,10 @@ public: bool contains(const Input & other) const; /** - * Fetch the input into the Nix store, returning the location in - * the Nix store and the locked input. + * Fetch the entire input into the Nix store, returning the + * location in the Nix store and the locked input. */ - std::pair fetch(ref store) const; + std::pair fetch(ref store) const; Input applyOverrides( std::optional ref, @@ -158,33 +152,4 @@ struct InputScheme void registerInputScheme(std::shared_ptr && fetcher); -struct DownloadFileResult -{ - StorePath storePath; - std::string etag; - std::string effectiveUrl; - std::optional immutableUrl; -}; - -DownloadFileResult downloadFile( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); - -struct DownloadTarballResult -{ - Tree tree; - time_t lastModified; - std::optional immutableUrl; -}; - -DownloadTarballResult downloadTarball( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); - } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index b824140a6..617fc7468 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -7,6 +7,7 @@ #include "git.hh" #include "fetchers.hh" #include "fetch-settings.hh" +#include "tarball.hh" #include #include @@ -213,10 +214,10 @@ struct GitArchiveInputScheme : InputScheme {"rev", rev->gitRev()}, {"lastModified", uint64_t(result.lastModified)} }, - result.tree.storePath, + result.storePath, true); - return {result.tree.storePath, input}; + return {result.storePath, input}; } std::optional experimentalFeature() override diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 43c03beec..a0fff9ceb 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,5 +1,5 @@ #include "registry.hh" -#include "fetchers.hh" +#include "tarball.hh" #include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ba47d59a7..e1ea9b58b 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,3 +1,4 @@ +#include "tarball.hh" #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" @@ -133,7 +134,7 @@ DownloadTarballResult downloadTarball( if (cached && !cached->expired) return { - .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .storePath = std::move(cached->storePath), .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; @@ -174,7 +175,7 @@ DownloadTarballResult downloadTarball( locked); return { - .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .storePath = std::move(*unpackedStorePath), .lastModified = lastModified, .immutableUrl = res.immutableUrl, }; @@ -307,7 +308,7 @@ struct TarballInputScheme : CurlInputScheme if (result.lastModified && !input.attrs.contains("lastModified")) input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); - return {result.tree.storePath, std::move(input)}; + return {result.storePath, std::move(input)}; } }; diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh new file mode 100644 index 000000000..9e6b50b31 --- /dev/null +++ b/src/libfetchers/tarball.hh @@ -0,0 +1,43 @@ +#pragma once + +#include "types.hh" +#include "path.hh" + +#include + +namespace nix { +class Store; +} + +namespace nix::fetchers { + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; + std::optional immutableUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +struct DownloadTarballResult +{ + StorePath storePath; + time_t lastModified; + std::optional immutableUrl; +}; + +DownloadTarballResult downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +} diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 95f401441..4504441fa 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -4,9 +4,9 @@ #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" -#include "fetchers.hh" #include "eval-settings.hh" // for defexpr #include "util.hh" +#include "tarball.hh" #include #include diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ceb112c03..b4e4156c0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -186,7 +186,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["path"] = store->printStorePath(flake.storePath); j["locks"] = lockedFlake.lockFile.toJSON(); logger->cout("%s", j.dump()); } else { @@ -202,7 +202,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.sourceInfo->storePath)); + store->printStorePath(flake.storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -976,7 +976,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.sourceInfo->storePath); + sources.insert(flake.flake.storePath); // FIXME: use graph output, handle cycles. std::function traverse; @@ -988,7 +988,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + : (*inputNode)->lockedRef.input.fetch(store).first; if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); @@ -1005,7 +1005,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, + {"path", store->printStorePath(flake.flake.storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; logger->cout("%s", jsonRoot); @@ -1339,12 +1339,12 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON { auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); - auto [tree, lockedRef] = resolvedRef.fetchTree(store); - auto hash = store->queryPathInfo(tree.storePath)->narHash; + auto [storePath, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(storePath)->narHash; if (json) { auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(tree.storePath); + res["storePath"] = store->printStorePath(storePath); res["hash"] = hash.to_string(HashFormat::SRI, true); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); @@ -1352,7 +1352,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), - store->printStorePath(tree.storePath), + store->printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); } } From 97a0c08873496dbd6f53adb253e159ecf700d616 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 Oct 2023 21:17:28 +0200 Subject: [PATCH 204/402] Expand derivation examples (#9048) Also use fancier formatting so the example blocks are easier to discern from the description. Co-authored-by: John Ericson --- doc/manual/src/glossary.md | 1 + doc/manual/src/language/derivations.md | 214 +++++++++++++++++++------ 2 files changed, 162 insertions(+), 53 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 611d9576f..d49d5e52e 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -207,6 +207,7 @@ - [output]{#gloss-output} A [store object] produced by a [derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. [output]: #gloss-output diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index d80a7dd48..2aded5527 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -8,8 +8,6 @@ It outputs an attribute set, and produces a [store derivation] as a side effect [store derivation]: @docroot@/glossary.md#gloss-store-derivation - - ## Input attributes ### Required @@ -17,11 +15,22 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) A symbolic name for the derivation. - It is added to the [store derivation]'s [path](@docroot@/glossary.md#gloss-store-path) and its [output paths][output path]. + It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - Example: `name = "hello";` + [store path]: @docroot@/glossary.md#gloss-store-path + + > **Example** + > + > ```nix + > derivation { + > name = "hello"; + > # ... + > } + > ``` + > + > The store derivation's path will be `/nix/store/-hello.drv`. + > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` - The store derivation's path will be `/nix/store/-hello.drv`, and the output paths will be of the form `/nix/store/-hello[-]` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -29,77 +38,175 @@ It outputs an attribute set, and produces a [store derivation] as a side effect A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. - Examples: - - `system = "x86_64-linux";` - - `system = builtins.currentSystem;` - - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. - [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system + > **Example** + > + > Declare a derivation to be built on a specific system type: + > + > ```nix + > derivation { + > # ... + > system = "x86_64-linux"; + > # ... + > } + > ``` + + > **Example** + > + > Declare a derivation to be built on the system type that evaluates the expression: + > + > ```nix + > derivation { + > # ... + > system = builtins.currentSystem; + > # ... + > } + > ``` + > + > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + - [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) Path to an executable that will perform the build. - Examples: + > **Example** + > + > Use the file located at `/bin/bash` as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > # ... + > }; + > ``` - `builder = "/bin/bash";` + - `builder = ./builder.sh;` + > **Example** + > + > Copy a local file to the Nix store for use as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = ./builder.sh; + > # ... + > }; + > ``` - `builder = "${pkgs.python}/bin/python";` + + + > **Example** + > + > Use a file from another derivation as the builder executable: + > + > ```nix + > let pkgs = import {}; in + > derivation { + > # ... + > builder = "${pkgs.python}/bin/python"; + > # ... + > }; + > ``` ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ ]` +- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ ]` Command-line arguments to be passed to the [`builder`](#attr-builder) executable. - Example: `args = [ "-c" "echo hello world > $out" ];` + > **Example** + > + > Pass arguments to Bash to interpret a shell command: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > args = [ "-c" "echo hello world > $out" ]; + > # ... + > }; + > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ "out" ]` +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ "out" ]` Symbolic outputs of the derivation. - Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [output path]. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [store path]. - [output path]: @docroot@/glossary.md#gloss-output-path - - By default, a derivation produces a single output path called `out`. - However, derivations can produce multiple output paths. + By default, a derivation produces a single output called `out`. + However, derivations can produce multiple outputs. This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. - Examples: + > **Example** + > + > Imagine a library package that provides a dynamic library, header files, and documentation. + > A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + > Thus, the library package could specify: + > + > ```nix + > derivation { + > # ... + > outputs = [ "lib" "dev" "doc" ]; + > # ... + > } + > ``` + > + > This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. + > The builder would typically do something like + > + > ```bash + > ./configure \ + > --libdir=$lib/lib \ + > --includedir=$dev/include \ + > --docdir=$doc/share/doc + > ``` + > + > for an Autoconf-style package. - Imagine a library package that provides a dynamic library, header files, and documentation. - A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. - Thus, the library package could specify: + The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. - ```nix - derivation { - # ... - outputs = [ "lib" "dev" "doc" ]; - # ... - } - ``` + > **Example** + > + > + > ```nix + > derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > } + > ``` + > + > The store derivation path will be `/nix/store/-example.drv`. + > The output paths will be + > - `/nix/store/-example-lib` + > - `/nix/store/-example-dev` + > - `/nix/store/-example-doc` + > - `/nix/store/-example` - This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. - The builder would typically do something like + You can refer to each output of a derivation by selecting it as an attribute. + The first element of `outputs` determines the *default output* and ends up at the top-level. - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$dev/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. - - You can refer to each output of a derivation by selecting it as an attribute, e.g. `myPackage.lib` or `myPackage.doc`. - - The first element of `outputs` determines the *default output*. - Therefore, in the given example, `myPackage` is equivalent to `myPackage.lib`. + > **Example** + > + > Select an output by attribute name: + > + > ```nix + > let + > myPackage = derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > }; + > in myPackage.dev + > ``` + > + > Since `lib` is the first output, `myPackage` is equivalent to `myPackage.lib`. @@ -123,8 +230,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect reside in the Nix store. - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. + present derivation. The environment variable is set to the [store path] of the derivation's default [output](#attr-outputs). - Lists of the previous types are also allowed. They are simply concatenated, separated by spaces. @@ -132,6 +238,8 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - `true` is passed as the string `1`, `false` and `null` are passed as an empty string. + + ## Builder execution The [`builder`](#attr-builder) is executed as follows: From 96c58550b839ca415a2cef0f7a5ae51f3e86f028 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:45:06 -0400 Subject: [PATCH 205/402] Test more derived paths --- src/libstore/tests/worker-protocol.cc | 11 ++++++++++- .../libstore/worker-protocol/derived-path.bin | Bin 120 -> 248 bytes 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 63fecce96..b10192cc1 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -72,10 +72,19 @@ VERSIONED_CHARACTERIZATION_TEST( derivedPath, "derived-path", defaultVersion, - (std::tuple { + (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, }, + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, DerivedPath::Built { .drvPath = makeConstantStorePathRef(StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path.bin index bb1a81ac6d096cddf510a457e7506e6a81ee66a3..0729b2690e2f99dc8122b013e4ceaa4a51ebd3d6 100644 GIT binary patch delta 39 mcmb>U!8pN!(Rd=G8JAv4Q5gdWm`(%= Date: Fri, 26 May 2023 14:48:11 -0400 Subject: [PATCH 206/402] Systematize the worker protocol derived path serialiser It was some ad-hoc functions to account for versions, while the already factored-out serializer just supported the latest version. Now, we can fold that version-specific logic into the factored out one, and so we do. --- src/libstore/daemon.cc | 18 ++-------- src/libstore/remote-store.cc | 33 ++---------------- src/libstore/tests/worker-protocol.cc | 29 +++++++++++++-- src/libstore/worker-protocol.cc | 26 ++++++++++++-- .../worker-protocol/derived-path-1.29.bin | Bin 0 -> 184 bytes ...derived-path.bin => derived-path-1.30.bin} | Bin 6 files changed, 56 insertions(+), 50 deletions(-) create mode 100644 unit-test-data/libstore/worker-protocol/derived-path-1.29.bin rename unit-test-data/libstore/worker-protocol/{derived-path.bin => derived-path-1.30.bin} (100%) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e117b507f..27c0b6431 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -261,18 +261,6 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, WorkerProto::Version clientVersion, WorkerProto::ReadConn conn) -{ - std::vector reqs; - if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = WorkerProto::Serialise>::read(store, conn); - } else { - for (auto & s : readStrings(conn.from)) - reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); - } - return reqs; -} - static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) @@ -538,7 +526,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPaths: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -563,7 +551,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPathsWithResults: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; mode = (BuildMode) readInt(from); @@ -938,7 +926,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryMissing: { - auto targets = readDerivedPaths(*store, clientVersion, rconn); + auto targets = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1704bfbb0..482c2ae01 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -655,33 +655,6 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, } catch (...) { return callback.rethrow(); } } -static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector & reqs) -{ - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) { - WorkerProto::write(store, conn, reqs); - } else { - Strings ss; - for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(store)); - }, - [&](const StorePath & drvPath) { - throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", - store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.daemonVersion), - GET_PROTOCOL_MINOR(conn.daemonVersion)); - }, - [&](std::monostate) { - throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); - }, - }, sOrDrvPath); - } - conn.to << ss; - } -} - void RemoteStore::copyDrvsFromEvalStore( const std::vector & paths, std::shared_ptr evalStore) @@ -711,7 +684,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeDerivedPaths(*this, *conn, drvPaths); + WorkerProto::write(*this, *conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -735,7 +708,7 @@ std::vector RemoteStore::buildPathsWithResults( if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; - writeDerivedPaths(*this, *conn, paths); + WorkerProto::write(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); return WorkerProto::Serialise>::read(*this, *conn); @@ -929,7 +902,7 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << WorkerProto::Op::QueryMissing; - writeDerivedPaths(*this, *conn, targets); + WorkerProto::write(*this, *conn, targets); conn.processStderr(); willBuild = WorkerProto::Serialise::read(*this, *conn); willSubstitute = WorkerProto::Serialise::read(*this, *conn); diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index b10192cc1..b3e857bc4 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -69,9 +69,32 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - derivedPath, - "derived-path", - defaultVersion, + derivedPath_1_29, + "derived-path-1.29", + 1 << 8 | 29, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_30, + "derived-path-1.30", + 1 << 8 | 30, (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 415e66f16..0f3f20ca5 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -51,12 +51,34 @@ void WorkerProto::Serialise>::write(const Store & sto DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); - return DerivedPath::parseLegacy(store, s); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + return DerivedPath::parseLegacy(store, s); + } else { + return parsePathWithOutputs(store, s).toDerivedPath(); + } } void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) { - conn.to << req.to_string_legacy(store); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + conn.to << req.to_string_legacy(store); + } else { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(req); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + conn.to << s.to_string(store); + }, + [&](const StorePath & drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MINOR(conn.version)); + }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but protocols do not support that. Try upgrading the Nix on the other end of this connection"); + }, + }, sOrDrvPath); + } } diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin new file mode 100644 index 0000000000000000000000000000000000000000..05ea7678aa058344a5ca4cb1e9e7333e9cf8fe20 GIT binary patch literal 184 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F&X-j5{vXwipsz`&B@oVSfNwN F001Y$HlP3i literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path.bin rename to unit-test-data/libstore/worker-protocol/derived-path-1.30.bin From ab822af0dfee341be131ad285a2cba5362a3f2dc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Feb 2021 18:42:46 +0000 Subject: [PATCH 207/402] Factor out serialization for `BuildResult` Worker Protocol: Note that the worker protocol already had a serialization for `BuildResult`; this was added in a4604f19284254ac98f19a13ff7c2216de7fe176. It didn't have any versioning support because at that time reusable seralizers were not away for the protocol version. It could thus only be used for new messages also introduced in that commit. Now that we do support versioning in reusable serializers, we can expand it to support all known versions and use it in many more places. The exist test data becomes the version 1.29 tests: note that those files' contents are unchanged. 1.28 and 1.27 tests are added to cover the older code-paths. The keyered build result test only has 1.29 because the keying was also added in a4604f19284254ac98f19a13ff7c2216de7fe176; the older serializations are always used unkeyed. Serve Protocol: Conversely, no attempt was made to factor out such a serializer for the serve protocol, so our work there in this commit for that protocol proceeds from scratch. --- src/libstore/daemon.cc | 11 +- src/libstore/legacy-ssh-store.cc | 15 +-- src/libstore/remote-store.cc | 15 +-- src/libstore/serve-protocol.cc | 43 +++++++ src/libstore/serve-protocol.hh | 6 + src/libstore/tests/serve-protocol.cc | 113 +++++++++++++++++- src/libstore/tests/worker-protocol.cc | 80 ++++++++++++- src/libstore/worker-protocol.cc | 49 ++++---- src/nix-store/nix-store.cc | 12 +- .../serve-protocol/build-result-2.2.bin | Bin 0 -> 80 bytes .../serve-protocol/build-result-2.3.bin | Bin 0 -> 176 bytes .../build-result-2.6.bin} | Bin .../worker-protocol/build-result-1.27.bin | Bin 0 -> 80 bytes .../worker-protocol/build-result-1.28.bin | Bin 0 -> 648 bytes .../worker-protocol/build-result-1.29.bin | Bin 0 -> 744 bytes ...result.bin => keyed-build-result-1.29.bin} | Bin 16 files changed, 268 insertions(+), 76 deletions(-) create mode 100644 unit-test-data/libstore/serve-protocol/build-result-2.2.bin create mode 100644 unit-test-data/libstore/serve-protocol/build-result-2.3.bin rename unit-test-data/libstore/{worker-protocol/build-result.bin => serve-protocol/build-result-2.6.bin} (100%) create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.27.bin create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.28.bin create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.29.bin rename unit-test-data/libstore/worker-protocol/{keyed-build-result.bin => keyed-build-result-1.29.bin} (100%) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 27c0b6431..19afe4388 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -635,16 +635,7 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); - to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; - } - if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); - } + WorkerProto::write(*store, wconn, res); break; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d6e24521a..c712f7eb1 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -319,20 +319,7 @@ public: conn->to.flush(); - BuildResult status; - status.status = (BuildResult::Status) readInt(conn->from); - conn->from >> status.errorMsg; - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = ServeProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - status.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return status; + return ServeProto::Serialise::read(*this, *conn); } void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 482c2ae01..ef648e01b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -788,20 +788,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res; - res.status = (BuildResult::Status) readInt(conn->from); - conn->from >> res.errorMsg; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { - conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return res; + return WorkerProto::Serialise::read(*this, *conn); } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 16a62b5bc..97a0ddf0e 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" +#include "build-result.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "archive.hh" @@ -12,4 +13,46 @@ namespace nix { /* protocol-specific definitions */ +BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +{ + BuildResult status; + status.status = (BuildResult::Status) readInt(conn.from); + conn.from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.from + >> status.timesBuilt + >> status.isNonDeterministic + >> status.startTime + >> status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + auto builtOutputs = ServeProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } + return status; +} + +void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +{ + conn.to + << status.status + << status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << status.timesBuilt + << status.isNonDeterministic + << status.startTime + << status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + ServeProto::write(store, conn, builtOutputs); + } +} + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index a627c6ad6..ba159f6e9 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -16,6 +16,9 @@ namespace nix { class Store; struct Source; +// items being serialised +struct BuildResult; + /** * The "serve protocol", used by ssh:// stores. @@ -136,6 +139,9 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ }; +template<> +DECLARE_SERVE_SERIALISER(BuildResult); + template DECLARE_SERVE_SERIALISER(std::vector); template diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc index ba798ab1c..c8ac87a04 100644 --- a/src/libstore/tests/serve-protocol.cc +++ b/src/libstore/tests/serve-protocol.cc @@ -19,7 +19,7 @@ struct ServeProtoTest : VersionedProtoTest * For serializers that don't care about the minimum version, we * used the oldest one: 1.0. */ - ServeProto::Version defaultVersion = 1 << 8 | 0; + ServeProto::Version defaultVersion = 2 << 8 | 0; }; VERSIONED_CHARACTERIZATION_TEST( @@ -114,6 +114,117 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_2, + "build-result-2.2", + 2 << 8 | 2, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_3, + "build-result-2.3", + 2 << 8 | 3, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_6, + "build-result-2.6", + 2 << 8 | 6, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index b3e857bc4..e0ce340d8 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -167,9 +167,77 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - buildResult, - "build-result", - defaultVersion, + buildResult_1_27, + "build-result-1.27", + 1 << 8 | 27, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_28, + "build-result-1.28", + 1 << 8 | 28, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29, + "build-result-1.29", + 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -226,9 +294,9 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - keyedBuildResult, - "keyed-build-result", - defaultVersion, + keyedBuildResult_1_29, + "keyed-build-result-1.29", + 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t { diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 0f3f20ca5..f69322a61 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -103,17 +103,21 @@ BuildResult WorkerProto::Serialise::read(const Store & store, Worke { BuildResult res; res.status = (BuildResult::Status) readInt(conn.from); - conn.from - >> res.errorMsg - >> res.timesBuilt - >> res.isNonDeterministic - >> res.startTime - >> res.stopTime; - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); + conn.from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.from + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + auto builtOutputs = WorkerProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } return res; } @@ -121,15 +125,20 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto { conn.to << res.status - << res.errorMsg - << res.timesBuilt - << res.isNonDeterministic - << res.startTime - << res.stopTime; - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputs); + << res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.to + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputs); + } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e78720f92..e4dd94585 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -959,17 +959,7 @@ static void opServe(Strings opFlags, Strings opArgs) MonitorFdHup monitor(in.fd); auto status = store->buildDerivation(drvPath, drv); - out << status.status << status.errorMsg; - - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) - out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : status.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - ServeProto::write(*store, wconn, builtOutputs); - } - + ServeProto::write(*store, wconn, status); break; } diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/unit-test-data/libstore/serve-protocol/build-result-2.2.bin new file mode 100644 index 0000000000000000000000000000000000000000..ae684778bc26addba4bf1b3e49cc30edf5f038fa GIT binary patch literal 80 gcmZQ&fBf0AiU4H~;_u literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/unit-test-data/libstore/serve-protocol/build-result-2.3.bin new file mode 100644 index 0000000000000000000000000000000000000000..d51e08dfc0d1715210e8a616a5403f03f4b2a46c GIT binary patch literal 176 vcmZQ&fBf0AiU4H~;_u literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/unit-test-data/libstore/worker-protocol/build-result-1.28.bin new file mode 100644 index 0000000000000000000000000000000000000000..74bcd5cf98b828fb63a931ef7810f7fabc805ca5 GIT binary patch literal 648 zcmc&wK?=e!5EQ|aK0%e!+`)6%Si?(*`6_8xaxz$KI6Xx-2t!SGa{k;2f8i+RMR2A;`6>RitBjDY7{lnANJD3OVh8WW_c zkTRG+zDX!Yj%68orEsd#Y$KG=*{FoWL-7`MFAQl%7RmZ0!PYe3jk66aF4r+L$O`tu z&uq-x(J#Q)LAOdzsy>VTJDdcofzX)BfXe~O6CwQ^%woR?}>#sX4il$bfs;n zmv%`YOQ~vvTo;t-%xH@l(n4vSK8bRZQHc`kI^B)Ih0TkNGRhXy8rqZN5BnYj(ieFo zAJ+t*u7l`;??iPt&V)lzi91dfGZFf@g4iVAZN4+jUVRU7o>ol_o!fedeM@Pl_mAVf T^ROZOQyyvZZF!s Date: Tue, 8 Mar 2022 23:09:26 +0000 Subject: [PATCH 208/402] Move `ValidPathInfo` serialization code to `worker-protocol.{cc.hh}` It does not belong with the data type itself. This also materializes the fact that `copyPath` does not do any version negotiation just just hard-codes "16". The non-standard interface of these serializers makes it harder to test, but this is fixed in the next commit which then adds those tests. --- src/libstore/daemon.cc | 4 +-- src/libstore/path-info.cc | 54 --------------------------------- src/libstore/path-info.hh | 5 --- src/libstore/remote-store.cc | 11 +++++-- src/libstore/store-api.cc | 11 ++++++- src/libstore/worker-protocol.cc | 45 ++++++++++++++++++++++++++- src/libstore/worker-protocol.hh | 12 ++++++++ 7 files changed, 76 insertions(+), 66 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 19afe4388..0afaaccba 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -422,7 +422,7 @@ static void performOp(TunnelLogger * logger, ref store, }(); logger->stopWork(); - pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); + WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { HashType hashAlgo; std::string baseName; @@ -819,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); + WorkerProto::Serialise::write(*store, wconn, *info, false); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index fb507a847..51fb5f02a 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,6 +1,4 @@ #include "path-info.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "store-api.hh" namespace nix { @@ -99,7 +97,6 @@ Strings ValidPathInfo::shortRefs() const return refs; } - ValidPathInfo::ValidPathInfo( const Store & store, std::string_view name, @@ -128,55 +125,4 @@ ValidPathInfo::ValidPathInfo( }, std::move(ca).raw); } - -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) -{ - return read(source, store, format, store.parseStorePath(readString(source))); -} - -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path) -{ - auto deriver = readString(source); - auto narHash = Hash::parseAny(readString(source), htSHA256); - ValidPathInfo info(path, narHash); - if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { - .from = source, - .version = format, - }); - source >> info.registrationTime >> info.narSize; - if (format >= 16) { - source >> info.ultimate; - info.sigs = readStrings(source); - info.ca = ContentAddress::parseOpt(readString(source)); - } - return info; -} - - -void ValidPathInfo::write( - Sink & sink, - const Store & store, - unsigned int format, - bool includePath) const -{ - if (includePath) - sink << store.printStorePath(path); - sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(HashFormat::Base16, false); - WorkerProto::write(store, - WorkerProto::WriteConn { - .to = sink, - .version = format, - }, - references); - sink << registrationTime << narSize; - if (format >= 16) { - sink << ultimate - << sigs - << renderContentAddress(ca); - } -} - } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index fea6d0e5f..919fe32f1 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -121,11 +121,6 @@ struct ValidPathInfo std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } - - static ValidPathInfo read(Source & source, const Store & store, unsigned int format); - static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path); - - void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const; }; typedef std::map ValidPathInfos; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ef648e01b..aaea5c4a2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -332,7 +332,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } info = std::make_shared( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); + WorkerProto::Serialise::read(*this, *conn, StorePath{path})); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -445,7 +445,7 @@ ref RemoteStore::addCAToStore( } return make_ref( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); + WorkerProto::Serialise::read(*this, *conn)); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); @@ -570,7 +570,12 @@ void RemoteStore::addMultipleToStore( auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); for (auto & [pathInfo, pathSource] : pathsToCopy) { - pathInfo.write(sink, *this, 16); + WorkerProto::Serialise::write(*this, + WorkerProto::WriteConn { + .to = sink, + .version = 16, + }, + pathInfo); pathSource->drainInto(sink); } }); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0c58cca0c..785b26062 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,6 +11,9 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +// FIXME this should not be here, see TODO below on +// `addMultipleToStore`. +#include "worker-protocol.hh" #include #include @@ -357,7 +360,13 @@ void Store::addMultipleToStore( { auto expected = readNum(source); for (uint64_t i = 0; i < expected; ++i) { - auto info = ValidPathInfo::read(source, *this, 16); + // FIXME we should not be using the worker protocol here, let + // alone the worker protocol with a hard-coded version! + auto info = WorkerProto::Serialise::read(*this, + WorkerProto::ReadConn { + .from = source, + .version = 16, + }); info.ultimate = false; addToStore(info, source, repair, checkSigs); } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index f69322a61..94f8c8811 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,7 +6,7 @@ #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "archive.hh" -#include "derivations.hh" +#include "path-info.hh" #include @@ -142,4 +142,47 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } +ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +{ + auto path = WorkerProto::Serialise::read(store, conn); + return WorkerProto::Serialise::read(store, conn, std::move(path)); +} + +ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn, StorePath && path) +{ + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + ValidPathInfo info(path, narHash); + if (deriver != "") info.deriver = store.parseStorePath(deriver); + info.references = WorkerProto::Serialise::read(store, conn); + conn.from >> info.registrationTime >> info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.from >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + } + return info; +} + +void WorkerProto::Serialise::write( + const Store & store, + WriteConn conn, + const ValidPathInfo & pathInfo, + bool includePath) +{ + if (includePath) + conn.to << store.printStorePath(pathInfo.path); + conn.to + << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") + << pathInfo.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(store, conn, pathInfo.references); + conn.to << pathInfo.registrationTime << pathInfo.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.to + << pathInfo.ultimate + << pathInfo.sigs + << renderContentAddress(pathInfo.ca); + } +} + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 2d55d926a..d35e3c678 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -31,6 +31,7 @@ struct Source; struct DerivedPath; struct BuildResult; struct KeyedBuildResult; +struct ValidPathInfo; enum TrustedFlag : bool; @@ -220,4 +221,15 @@ template DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ +/* These are a non-standard form for historical reasons. */ + +template<> +struct WorkerProto::Serialise +{ + static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn); + static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn, StorePath && path); + + static void write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo, bool includePath = true); +}; + } From 70f8b96c11af275b5766dca0a49737803d1e0339 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 17:13:44 -0400 Subject: [PATCH 209/402] Factor out `UnkeyedValidPathInfo` and test This makes the path info serialisers ideomatic again, which allows me to test them. --- src/libstore/daemon.cc | 2 +- src/libstore/path-info.cc | 23 ++- src/libstore/path-info.hh | 31 ++-- src/libstore/remote-store.cc | 3 +- src/libstore/tests/worker-protocol.cc | 153 ++++++++++++++++++ src/libstore/worker-protocol.cc | 24 +-- src/libstore/worker-protocol.hh | 16 +- .../unkeyed-valid-path-info-1.15.bin | Bin 0 -> 328 bytes .../worker-protocol/valid-path-info-1.15.bin | Bin 0 -> 488 bytes .../worker-protocol/valid-path-info-1.16.bin | Bin 0 -> 952 bytes 10 files changed, 218 insertions(+), 34 deletions(-) create mode 100644 unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin create mode 100644 unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin create mode 100644 unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0afaaccba..007ffc05a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -819,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - WorkerProto::Serialise::write(*store, wconn, *info, false); + WorkerProto::write(*store, wconn, static_cast(*info)); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 51fb5f02a..ab39e71f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,6 +3,25 @@ namespace nix { +GENERATE_CMP_EXT( + , + UnkeyedValidPathInfo, + me->deriver, + me->narHash, + me->references, + me->registrationTime, + me->narSize, + //me->id, + me->ultimate, + me->sigs, + me->ca); + +GENERATE_CMP_EXT( + , + ValidPathInfo, + me->path, + static_cast(*me)); + std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) @@ -102,8 +121,8 @@ ValidPathInfo::ValidPathInfo( std::string_view name, ContentAddressWithReferences && ca, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) - , narHash(narHash) + : UnkeyedValidPathInfo(narHash) + , path(store.makeFixedOutputPathFromCA(name, ca)) { std::visit(overloaded { [this](TextInfo && ti) { diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 919fe32f1..a82e643ae 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -32,9 +32,8 @@ struct SubstitutablePathInfo typedef std::map SubstitutablePathInfos; -struct ValidPathInfo +struct UnkeyedValidPathInfo { - StorePath path; std::optional deriver; /** * \todo document this @@ -72,6 +71,20 @@ struct ValidPathInfo */ std::optional ca; + UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default; + + UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; + + DECLARE_CMP(UnkeyedValidPathInfo); + + virtual ~UnkeyedValidPathInfo() { } +}; + +struct ValidPathInfo : UnkeyedValidPathInfo { + StorePath path; + + DECLARE_CMP(ValidPathInfo); + /** * Return a fingerprint of the store path to be used in binary * cache signatures. It contains the store path, the base-32 @@ -84,11 +97,11 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - /** - * @return The `ContentAddressWithReferences` that determines the - * store path for a content-addressed store object, `std::nullopt` - * for an input-addressed store object. - */ + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ std::optional contentAddressWithReferences() const; /** @@ -114,8 +127,8 @@ struct ValidPathInfo ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; - ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; + ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(path) { }; ValidPathInfo(const Store & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index aaea5c4a2..7bdc25433 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -332,7 +332,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } info = std::make_shared( - WorkerProto::Serialise::read(*this, *conn, StorePath{path})); + StorePath{path}, + WorkerProto::Serialise::read(*this, *conn)); } callback(std::move(info)); } catch (...) { callback.rethrow(); } diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index e0ce340d8..ad5943c69 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -329,6 +329,159 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + unkeyedValidPathInfo_1_15, + "unkeyed-valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_15, + "valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_16, + "valid-path-info-1.16", + 1 << 8 | 16, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.ultimate = true; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalTrustedFlag, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 94f8c8811..d618b9bd8 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -145,14 +145,24 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); - return WorkerProto::Serialise::read(store, conn, std::move(path)); + return ValidPathInfo { + std::move(path), + WorkerProto::Serialise::read(store, conn), + }; } -ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn, StorePath && path) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) +{ + WorkerProto::write(store, conn, pathInfo.path); + WorkerProto::write(store, conn, static_cast(pathInfo)); +} + + +UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { auto deriver = readString(conn.from); auto narHash = Hash::parseAny(readString(conn.from), htSHA256); - ValidPathInfo info(path, narHash); + UnkeyedValidPathInfo info(narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, conn); conn.from >> info.registrationTime >> info.narSize; @@ -164,14 +174,8 @@ ValidPathInfo WorkerProto::Serialise::read(const Store & store, R return info; } -void WorkerProto::Serialise::write( - const Store & store, - WriteConn conn, - const ValidPathInfo & pathInfo, - bool includePath) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { - if (includePath) - conn.to << store.printStorePath(pathInfo.path); conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") << pathInfo.narHash.to_string(HashFormat::Base16, false); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d35e3c678..dcd54ad16 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -32,6 +32,7 @@ struct DerivedPath; struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; +struct UnkeyedValidPathInfo; enum TrustedFlag : bool; @@ -207,6 +208,10 @@ DECLARE_WORKER_SERIALISER(BuildResult); template<> DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> +DECLARE_WORKER_SERIALISER(ValidPathInfo); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); +template<> DECLARE_WORKER_SERIALISER(std::optional); template @@ -221,15 +226,4 @@ template DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ -/* These are a non-standard form for historical reasons. */ - -template<> -struct WorkerProto::Serialise -{ - static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn); - static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn, StorePath && path); - - static void write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo, bool includePath = true); -}; - } diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin new file mode 100644 index 0000000000000000000000000000000000000000..e69ccbe83862b29e3f79810c0c2ff4561a42f86f GIT binary patch literal 328 zcmbu4%?`pK5QOo8D!&WM#q>Qq0R{CUX=_wYUVUsEAs(9a)(w2^a=MH48Q8;TGRcmlTf(6rfXPFjSJ46(yf&k8}#9A2z znmI2!XwnU&AmvqJYIhQk7+x#{!gQkU@$Il_5ceP*O@N zxu!r+#$?P>4g#ry<$$?w`2MnGPahlJ`HLM!?tH6m@;lyRZZ9kI;P`*_d+&94ym9_l z{ZG@h-`{zOWuKw$x?n$F_k7^>Jh*~Rnvu&5}L}Qp+gUA>+#+bF!C(UNw&;S4c literal 0 HcmV?d00001 From 201a4af9a4447a151682de6705a4a74783bf0e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=A2=D1=80=D0=BE?= =?UTF-8?q?=D1=84=D0=B8=D0=BC=D0=BE=D0=B2?= <62882157+trofkm@users.noreply.github.com> Date: Mon, 23 Oct 2023 01:56:46 +0300 Subject: [PATCH 210/402] Clean up `app.cc` (#9201) - Rename `expected` to `expectedType` - Use early `return` and `continue` to reduce nesting --- .gitignore | 2 ++ src/nix/app.cc | 40 ++++++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 2d3314015..04d96ca2c 100644 --- a/.gitignore +++ b/.gitignore @@ -138,7 +138,9 @@ nix-rust/target result +# IDE .vscode/ +.idea/ # clangd and possibly more .cache/ diff --git a/src/nix/app.cc b/src/nix/app.cc index 34fac9935..935ed18ec 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -20,20 +20,24 @@ StringPairs resolveRewrites( const std::vector & dependencies) { StringPairs res; - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - for (auto & dep : dependencies) { - if (auto drvDep = std::get_if(&dep.path)) { - for (auto & [ outputName, outputPath ] : drvDep->outputs) { - res.emplace( - DownstreamPlaceholder::fromSingleDerivedPathBuilt( - SingleDerivedPath::Built { - .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), - .output = outputName, - }).render(), - store.printStorePath(outputPath) - ); - } - } + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + return res; + } + for (auto &dep: dependencies) { + auto drvDep = std::get_if(&dep.path); + if (!drvDep) { + continue; + } + + for (const auto & [ outputName, outputPath ] : drvDep->outputs) { + res.emplace( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), + .output = outputName, + }).render(), + store.printStorePath(outputPath) + ); } } return res; @@ -58,11 +62,11 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto type = cursor->getAttr("type")->getString(); - std::string expected = !attrPath.empty() && + std::string expectedType = !attrPath.empty() && (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp") ? "app" : "derivation"; - if (type != expected) - throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type != expectedType) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expectedType); if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); @@ -91,7 +95,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) }, c.raw)); } - return UnresolvedApp{App { + return UnresolvedApp { App { .context = std::move(context2), .program = program, }}; From 256dfb98e864e474921ac5a8264e33de6978f47d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 23 Oct 2023 04:05:02 +0200 Subject: [PATCH 211/402] remove Basic Package Management section (#7974) this is the first thing most beginners see, and it misleads them into assuming `nix-env` is appropriate for doing anything but setting and reverting profile generations. this chapter is the root of most evil around the ecosystem, and today we finally close it for good. --- doc/manual/src/SUMMARY.md.in | 1 - .../package-management/basic-package-mgmt.md | 179 ------------------ 2 files changed, 180 deletions(-) delete mode 100644 doc/manual/src/package-management/basic-package-mgmt.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index e49e77cf5..60ebeb138 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,7 +17,6 @@ - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) - [Package Management](package-management/package-management.md) - - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md deleted file mode 100644 index 07b92fb76..000000000 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ /dev/null @@ -1,179 +0,0 @@ -# Basic Package Management - -The main command for package management is -[`nix-env`](../command-ref/nix-env.md). You can use it to install, -upgrade, and erase packages, and to query what packages are installed -or are available for installation. - -In Nix, different users can have different “views” on the set of -installed applications. That is, there might be lots of applications -present on the system (possibly in many different versions), but users -can have a specific selection of those active — where “active” just -means that it appears in a directory in the user’s `PATH`. Such a view -on the set of installed applications is called a *user environment*, -which is just a directory tree consisting of symlinks to the files of -the active applications. - -Components are installed from a set of *Nix expressions* that tell Nix -how to build those packages, including, if necessary, their -dependencies. There is a collection of Nix expressions called the -Nixpkgs package collection that contains packages ranging from basic -development stuff such as GCC and Glibc, to end-user applications like -Mozilla Firefox. (Nix is however not tied to the Nixpkgs package -collection; you could write your own Nix expressions based on Nixpkgs, -or completely new ones.) - -You can manually download the latest version of Nixpkgs from -. However, it’s much more -convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes -it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is -automatically added to your list of “subscribed” channels when you -install Nix. If this is not the case for some reason, you can add it -as follows: - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -$ nix-channel --update -``` - -> **Note** -> -> On NixOS, you’re automatically subscribed to a NixOS channel -> corresponding to your NixOS major release (e.g. -> ). A NixOS channel is identical -> to the Nixpkgs channel, except that it contains only Linux binaries -> and is updated only if a set of regression tests succeed. - -You can view the set of available packages in Nixpkgs: - -```console -$ nix-env --query --available --attr-path -nixpkgs.aterm aterm-2.2 -nixpkgs.bash bash-3.0 -nixpkgs.binutils binutils-2.15 -nixpkgs.bison bison-1.875d -nixpkgs.blackdown blackdown-1.4.2 -nixpkgs.bzip2 bzip2-1.0.2 -… -``` - -The flag `-q` specifies a query operation, `-a` means that you want -to show the “available” (i.e., installable) packages, as opposed to the -installed packages, and `-P` prints the attribute paths that can be used -to unambiguously select a package for installation (listed in the first column). -If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, -then you need to pass the path to your Nixpkgs tree using the `-f` flag: - -```console -$ nix-env --query --available --attr-path --file /path/to/nixpkgs -aterm aterm-2.2 -bash bash-3.0 -… -``` - -where */path/to/nixpkgs* is where you’ve unpacked or checked out -Nixpkgs. - -You can filter the packages by name: - -```console -$ nix-env --query --available --attr-path firefox -nixpkgs.firefox-esr firefox-91.3.0esr -nixpkgs.firefox firefox-94.0.1 -``` - -and using regular expressions: - -```console -$ nix-env --query --available --attr-path 'firefox.*' -``` - -It is also possible to see the *status* of available packages, i.e., -whether they are installed into the user environment and/or present in -the system: - -```console -$ nix-env --query --available --attr-path --status -… --PS nixpkgs.bash bash-3.0 ---S nixpkgs.binutils binutils-2.15 -IPS nixpkgs.bison bison-1.875d -… -``` - -The first character (`I`) indicates whether the package is installed in -your current user environment. The second (`P`) indicates whether it is -present on your system (in which case installing it into your user -environment would be a very quick operation). The last one (`S`) -indicates whether there is a so-called *substitute* for the package, -which is Nix’s mechanism for doing binary deployment. It just means that -Nix knows that it can fetch a pre-built package from somewhere -(typically a network server) instead of building it locally. - -You can install a package using `nix-env --install --attr `. For instance, - -```console -$ nix-env --install --attr nixpkgs.subversion -``` - -will install the package called `subversion` from `nixpkgs` channel (which is, of course, the -[Subversion version management system](http://subversion.tigris.org/)). - -> **Note** -> -> When you ask Nix to install a package, it will first try to get it in -> pre-compiled form from a *binary cache*. By default, Nix will use the -> binary cache ; it contains binaries for most -> packages in Nixpkgs. Only if no binary is available in the binary -> cache, Nix will build the package from source. So if `nix-env -> -iA nixpkgs.subversion` results in Nix building stuff from source, then either -> the package is not built for your platform by the Nixpkgs build -> servers, or your version of Nixpkgs is too old or too new. For -> instance, if you have a very recent checkout of Nixpkgs, then the -> Nixpkgs build servers may not have had a chance to build everything -> and upload the resulting binaries to . The -> Nixpkgs channel is only updated after all binaries have been uploaded -> to the cache, so if you stick to the Nixpkgs channel (rather than -> using a Git checkout of the Nixpkgs tree), you will get binaries for -> most packages. - -Naturally, packages can also be uninstalled. Unlike when installing, you will -need to use the derivation name (though the version part can be omitted), -instead of the attribute path, as `nix-env` does not record which attribute -was used for installing: - -```console -$ nix-env --uninstall subversion -``` - -Upgrading to a new version is just as easy. If you have a new release of -Nix Packages, you can do: - -```console -$ nix-env --upgrade --attr nixpkgs.subversion -``` - -This will *only* upgrade Subversion if there is a “newer” version in the -new set of Nix expressions, as defined by some pretty arbitrary rules -regarding ordering of version numbers (which generally do what you’d -expect of them). To just unconditionally replace Subversion with -whatever version is in the Nix expressions, use `-i` instead of `-u`; -`-i` will remove whatever version is already installed. - -You can also upgrade all packages for which there are newer versions: - -```console -$ nix-env --upgrade -``` - -Sometimes it’s useful to be able to ask what `nix-env` would do, without -actually doing it. For instance, to find out what packages would be -upgraded by `nix-env --upgrade `, you can do - -```console -$ nix-env --upgrade --dry-run -(dry run; not doing anything) -upgrading `libxslt-1.1.0' to `libxslt-1.1.10' -upgrading `graphviz-1.10' to `graphviz-1.12' -upgrading `coreutils-5.0' to `coreutils-5.2.1' -``` From 34a42f0d0ac20a70fdcc3dbf99a212b523c5f2d4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Oct 2023 11:05:50 +0200 Subject: [PATCH 212/402] Move PosixSourceAccessor into its own file --- src/libfetchers/fs-input-accessor.cc | 1 + src/libutil/archive.cc | 2 +- src/libutil/posix-source-accessor.cc | 86 ++++++++++++++++++++++++++++ src/libutil/posix-source-accessor.hh | 34 +++++++++++ src/libutil/source-accessor.cc | 81 -------------------------- src/libutil/source-accessor.hh | 27 --------- 6 files changed, 122 insertions(+), 109 deletions(-) create mode 100644 src/libutil/posix-source-accessor.cc create mode 100644 src/libutil/posix-source-accessor.hh diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 3444c4643..7638d2d82 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -1,4 +1,5 @@ #include "fs-input-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" namespace nix { diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0cd54e5db..3b1a1e0ef 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -14,7 +14,7 @@ #include "archive.hh" #include "util.hh" #include "config.hh" -#include "source-accessor.hh" +#include "posix-source-accessor.hh" namespace nix { diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc new file mode 100644 index 000000000..48b4fe626 --- /dev/null +++ b/src/libutil/posix-source-accessor.cc @@ -0,0 +1,86 @@ +#include "posix-source-accessor.hh" + +namespace nix { + +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + // FIXME: add O_NOFOLLOW since symlinks should be resolved by the + // caller? + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::vector buf(64 * 1024); + while (left) { + checkInterrupt(); + ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + return nix::pathExists(path.abs()); +} + +SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +{ + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + return nix::readLink(path.abs()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return path; +} + +} diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh new file mode 100644 index 000000000..608f96ee2 --- /dev/null +++ b/src/libutil/posix-source-accessor.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : SourceAccessor +{ + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; + + bool pathExists(const CanonPath & path) override; + + Stat lstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; +}; + +} diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 2d03d3d7a..d168a9667 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -55,85 +55,4 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } -void PosixSourceAccessor::readFile( - const CanonPath & path, - Sink & sink, - std::function sizeCallback) -{ - // FIXME: add O_NOFOLLOW since symlinks should be resolved by the - // caller? - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("statting file"); - - sizeCallback(st.st_size); - - off_t left = st.st_size; - - std::vector buf(64 * 1024); - while (left) { - checkInterrupt(); - ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading from file '%s'", showPath(path)); - } - else if (rd == 0) - throw SysError("unexpected end-of-file reading '%s'", showPath(path)); - else { - assert(rd <= left); - sink({(char *) buf.data(), (size_t) rd}); - left -= rd; - } - } -} - -bool PosixSourceAccessor::pathExists(const CanonPath & path) -{ - return nix::pathExists(path.abs()); -} - -SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) -{ - auto st = nix::lstat(path.abs()); - mtime = std::max(mtime, st.st_mtime); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; -} - -SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) -{ - DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; -} - -std::string PosixSourceAccessor::readLink(const CanonPath & path) -{ - return nix::readLink(path.abs()); -} - -std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) -{ - return path; -} - } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index f3504c9bb..fd823aa39 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -104,31 +104,4 @@ struct SourceAccessor virtual std::string showPath(const CanonPath & path); }; -/** - * A source accessor that uses the Unix filesystem. - */ -struct PosixSourceAccessor : SourceAccessor -{ - /** - * The most recent mtime seen by lstat(). This is a hack to - * support dumpPathAndGetMtime(). Should remove this eventually. - */ - time_t mtime = 0; - - void readFile( - const CanonPath & path, - Sink & sink, - std::function sizeCallback) override; - - bool pathExists(const CanonPath & path) override; - - Stat lstat(const CanonPath & path) override; - - DirEntries readDirectory(const CanonPath & path) override; - - std::string readLink(const CanonPath & path) override; - - std::optional getPhysicalPath(const CanonPath & path) override; -}; - } From b461cac21aa5b9127d230ed3f1a1ca8e8266fb60 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Oct 2023 09:03:11 -0400 Subject: [PATCH 213/402] Overhaul completions, redo #6693 (#8131) As I complained in https://github.com/NixOS/nix/pull/6784#issuecomment-1421777030 (a comment on the wrong PR, sorry again!), #6693 introduced a second completions mechanism to fix a bug. Having two completion mechanisms isn't so nice. As @thufschmitt also pointed out, it was a bummer to go from `FlakeRef` to `std::string` when collecting flake refs. Now it is `FlakeRefs` again. The underlying issue that sought to work around was that completion of arguments not at the end can still benefit from the information from latter arguments. To fix this better, we rip out that change and simply defer all completion processing until after all the (regular, already-complete) arguments have been passed. In addition, I noticed the original completion logic used some global variables. I do not like global variables, because even if they save lines of code, they also obfuscate the architecture of the code. I got rid of them moved them to a new `RootArgs` class, which now has `parseCmdline` instead of `Args`. The idea is that we have many argument parsers from subcommands and what-not, but only one root args that owns the other per actual parsing invocation. The state that was global is now part of the root args instead. This did, admittedly, add a bunch of new code. And I do feel bad about that. So I went and added a lot of API docs to try to at least make the current state of things clear to the next person. -- This is needed for RFC 134 (tracking issue #7868). It was very hard to modularize `Installable` parsing when there were two completion arguments. I wouldn't go as far as to say it is *easy* now, but at least it is less hard (and the completions test finally passed). Co-authored-by: Valentin Gagarin --- src/libcmd/command.hh | 26 ++++---- src/libcmd/common-eval-args.cc | 4 +- src/libcmd/installables.cc | 114 +++++++++++++++++++-------------- src/libmain/common-args.cc | 9 +-- src/libmain/shared.hh | 3 +- src/libutil/args.cc | 111 ++++++++++++++++++++------------ src/libutil/args.hh | 109 ++++++++++++++++++++----------- src/libutil/args/root.hh | 72 +++++++++++++++++++++ src/libutil/local.mk | 5 ++ src/nix/bundle.cc | 4 +- src/nix/flake.cc | 14 ++-- src/nix/main.cc | 27 ++++---- src/nix/registry.cc | 4 +- src/nix/why-depends.cc | 8 +-- 14 files changed, 329 insertions(+), 181 deletions(-) create mode 100644 src/libutil/args/root.hh diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 5c4569001..dafc0db3b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -97,8 +97,6 @@ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; - std::optional needsFlakeInputCompletion = {}; - MixFlakeOptions(); /** @@ -109,12 +107,8 @@ struct MixFlakeOptions : virtual Args, EvalCommand * command is operating with (presumably specified via some other * arguments) so that the completions for these flags can use them. */ - virtual std::vector getFlakesForCompletion() + virtual std::vector getFlakeRefsForCompletion() { return {}; } - - void completeFlakeInput(std::string_view prefix); - - void completionHook() override; }; struct SourceExprCommand : virtual Args, MixFlakeOptions @@ -137,7 +131,13 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions /** * Complete an installable from the given prefix. */ - void completeInstallable(std::string_view prefix); + void completeInstallable(AddCompletions & completions, std::string_view prefix); + + /** + * Convenience wrapper around the underlying function to make setting the + * callback easier. + */ + CompleterClosure getCompleteInstallable(); }; /** @@ -170,7 +170,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand bool readFromStdIn = false; - std::vector getFlakesForCompletion() override; + std::vector getFlakeRefsForCompletion() override; private: @@ -199,10 +199,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand void run(ref store) override; - std::vector getFlakesForCompletion() override - { - return {_installable}; - } + std::vector getFlakeRefsForCompletion() override; private: @@ -329,9 +326,10 @@ struct MixEnvironment : virtual Args { void setEnviron(); }; -void completeFlakeRef(ref store, std::string_view prefix); +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e6df49a7a..e53bc4c01 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -133,8 +133,8 @@ MixEvalArgs::MixEvalArgs() if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(openStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, openStore(), prefix); }} }); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 01c01441c..eff18bbf6 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -28,6 +28,20 @@ namespace nix { +static void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix) +{ + for (auto & flakeRef : flakeRefs) { + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions.add(input.first); + } +} + MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; @@ -79,8 +93,8 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, - .completer = {[&](size_t, std::string_view prefix) { - needsFlakeInputCompletion = {std::string(prefix)}; + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); }} }); @@ -95,11 +109,12 @@ MixFlakeOptions::MixFlakeOptions() flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."), true)); }}, - .completer = {[&](size_t n, std::string_view prefix) { - if (n == 0) - needsFlakeInputCompletion = {std::string(prefix)}; - else if (n == 1) - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { + if (n == 0) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + } else if (n == 1) { + completeFlakeRef(completions, getEvalState()->store, prefix); + } }} }); @@ -146,30 +161,12 @@ MixFlakeOptions::MixFlakeOptions() } } }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getEvalState()->store, prefix); }} }); } -void MixFlakeOptions::completeFlakeInput(std::string_view prefix) -{ - auto evalState = getEvalState(); - for (auto & flakeRefS : getFlakesForCompletion()) { - auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); - } -} - -void MixFlakeOptions::completionHook() -{ - if (auto & prefix = needsFlakeInputCompletion) - completeFlakeInput(*prefix); -} - SourceExprCommand::SourceExprCommand() { addFlag({ @@ -226,11 +223,18 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() }; } -void SourceExprCommand::completeInstallable(std::string_view prefix) +Args::CompleterClosure SourceExprCommand::getCompleteInstallable() +{ + return [this](AddCompletions & completions, size_t, std::string_view prefix) { + completeInstallable(completions, prefix); + }; +} + +void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix) { try { if (file) { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); evalSettings.pureEval = false; auto state = getEvalState(); @@ -265,14 +269,15 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) std::string name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions->add(name); + completions.add(name); else - completions->add(prefix_ + "." + name); + completions.add(prefix_ + "." + name); } } } } else { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, getDefaultFlakeAttrPathPrefixes(), @@ -285,6 +290,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) } void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, @@ -296,9 +302,9 @@ void completeFlakeRefWithFragment( try { auto hash = prefix.find('#'); if (hash == std::string::npos) { - completeFlakeRef(evalState->store, prefix); + completeFlakeRef(completions, evalState->store, prefix); } else { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); auto fragment = prefix.substr(hash + 1); std::string prefixRoot = ""; @@ -341,7 +347,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -352,7 +358,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->add(flakeRefS + "#" + prefixRoot); + completions.add(flakeRefS + "#" + prefixRoot); } } } @@ -361,15 +367,15 @@ void completeFlakeRefWithFragment( } } -void completeFlakeRef(ref store, std::string_view prefix) +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") - completions->add("."); + completions.add("."); - completeDir(0, prefix); + Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ for (auto & registry : fetchers::getRegistries(store)) { @@ -378,10 +384,10 @@ void completeFlakeRef(ref store, std::string_view prefix) if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) - completions->add(from2); + completions.add(from2); } else { if (hasPrefix(from, prefix)) - completions->add(from); + completions.add(from); } } } @@ -747,9 +753,7 @@ RawInstallablesCommand::RawInstallablesCommand() expectArgs({ .label = "installables", .handler = {&rawInstallables}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } @@ -762,6 +766,17 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector & } } +std::vector RawInstallablesCommand::getFlakeRefsForCompletion() +{ + applyDefaultInstallables(rawInstallables); + std::vector res; + for (auto i : rawInstallables) + res.push_back(parseFlakeRefWithFragment( + expandTilde(i), + absPath(".")).first); + return res; +} + void RawInstallablesCommand::run(ref store) { if (readFromStdIn && !isatty(STDIN_FILENO)) { @@ -775,10 +790,13 @@ void RawInstallablesCommand::run(ref store) run(store, std::move(rawInstallables)); } -std::vector RawInstallablesCommand::getFlakesForCompletion() +std::vector InstallableCommand::getFlakeRefsForCompletion() { - applyDefaultInstallables(rawInstallables); - return rawInstallables; + return { + parseFlakeRefWithFragment( + expandTilde(_installable), + absPath(".")).first + }; } void InstallablesCommand::run(ref store, std::vector && rawInstallables) @@ -794,9 +812,7 @@ InstallableCommand::InstallableCommand() .label = "installable", .optional = true, .handler = {&_installable}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index f92920d18..205b77808 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,4 +1,5 @@ #include "common-args.hh" +#include "args/root.hh" #include "globals.hh" #include "loggers.hh" @@ -34,21 +35,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .category = miscCategory, .labels = {"name", "value"}, - .handler = {[](std::string name, std::string value) { + .handler = {[this](std::string name, std::string value) { try { globalConfig.set(name, value); } catch (UsageError & e) { - if (!completions) + if (!getRoot().completions) warn(e.what()); } }}, - .completer = [](size_t index, std::string_view prefix) { + .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) { if (index == 0) { std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->add(s.first, fmt("Set the `%s` setting.", s.first)); + completions.add(s.first, fmt("Set the `%s` setting.", s.first)); } } }); diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 9415be78a..3159fe479 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -3,6 +3,7 @@ #include "util.hh" #include "args.hh" +#include "args/root.hh" #include "common-args.hh" #include "path.hh" #include "derived-path.hh" @@ -66,7 +67,7 @@ template N getIntArg(const std::string & opt, } -struct LegacyArgs : public MixCommonArgs +struct LegacyArgs : public MixCommonArgs, public RootArgs { std::function parseArg; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index e410c7eec..6bc3cae07 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,4 +1,5 @@ #include "args.hh" +#include "args/root.hh" #include "hash.hh" #include "json-utils.hh" @@ -26,6 +27,11 @@ void Args::removeFlag(const std::string & longName) longFlags.erase(flag); } +void Completions::setType(AddCompletions::Type t) +{ + type = t; +} + void Completions::add(std::string completion, std::string description) { description = trim(description); @@ -37,7 +43,7 @@ void Completions::add(std::string completion, std::string description) if (needs_ellipsis) description.append(" [...]"); } - insert(Completion { + completions.insert(Completion { .completion = completion, .description = description }); @@ -46,12 +52,20 @@ void Completions::add(std::string completion, std::string description) bool Completion::operator<(const Completion & other) const { return completion < other.completion || (completion == other.completion && description < other.description); } -CompletionType completionType = ctNormal; -std::shared_ptr completions; - std::string completionMarker = "___COMPLETE___"; -static std::optional needsCompletion(std::string_view s) +RootArgs & Args::getRoot() +{ + Args * p = this; + while (p->parent) + p = p->parent; + + auto * res = dynamic_cast(p); + assert(res); + return *res; +} + +std::optional RootArgs::needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -60,7 +74,7 @@ static std::optional needsCompletion(std::string_view s) return {}; } -void Args::parseCmdline(const Strings & _cmdline) +void RootArgs::parseCmdline(const Strings & _cmdline) { Strings pendingArgs; bool dashDash = false; @@ -71,7 +85,7 @@ void Args::parseCmdline(const Strings & _cmdline) size_t n = std::stoi(*s); assert(n > 0 && n <= cmdline.size()); *std::next(cmdline.begin(), n - 1) += completionMarker; - completions = std::make_shared(); + completions = std::make_shared(); verbosity = lvlError; } @@ -125,17 +139,23 @@ void Args::parseCmdline(const Strings & _cmdline) for (auto & f : flagExperimentalFeatures) experimentalFeatureSettings.require(f); + /* Now that all the other args are processed, run the deferred completions. + */ + for (auto d : deferredCompletions) + d.completer(*completions, d.n, d.prefix); } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); + auto & rootArgs = getRoot(); + auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; if (auto & f = flag.experimentalFeature) - flagExperimentalFeatures.insert(*f); + rootArgs.flagExperimentalFeatures.insert(*f); std::vector args; bool anyCompleted = false; @@ -146,10 +166,15 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) "flag '%s' requires %d argument(s), but only %d were given", name, flag.handler.arity, n); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { anyCompleted = true; - if (flag.completer) - flag.completer(n, *prefix); + if (flag.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = flag.completer, + .n = n, + .prefix = *prefix, + }); + } } args.push_back(*pos++); } @@ -159,14 +184,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) }; if (std::string(*pos, 0, 2) == "--") { - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) { if (auto & f = flag->experimentalFeature) - flagExperimentalFeatures.insert(*f); - completions->add("--" + name, flag->description); + rootArgs.flagExperimentalFeatures.insert(*f); + rootArgs.completions->add("--" + name, flag->description); } } return false; @@ -183,12 +208,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return process(std::string("-") + c, *i->second); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { if (prefix == "-") { - completions->add("--"); + rootArgs.completions->add("--"); for (auto & [flagName, flag] : shortFlags) if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) - completions->add(std::string("-") + flagName, flag->description); + rootArgs.completions->add(std::string("-") + flagName, flag->description); } } @@ -203,6 +228,8 @@ bool Args::processArgs(const Strings & args, bool finish) return true; } + auto & rootArgs = getRoot(); + auto & exp = expectedArgs.front(); bool res = false; @@ -211,15 +238,23 @@ bool Args::processArgs(const Strings & args, bool finish) (exp.handler.arity != ArityAny && args.size() == exp.handler.arity)) { std::vector ss; + bool anyCompleted = false; for (const auto &[n, s] : enumerate(args)) { - if (auto prefix = needsCompletion(s)) { + if (auto prefix = rootArgs.needsCompletion(s)) { + anyCompleted = true; ss.push_back(*prefix); - if (exp.completer) - exp.completer(n, *prefix); + if (exp.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = exp.completer, + .n = n, + .prefix = *prefix, + }); + } } else ss.push_back(s); } - exp.handler.fun(ss); + if (!anyCompleted) + exp.handler.fun(ss); expectedArgs.pop_front(); res = true; } @@ -271,11 +306,11 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) - completions->add(type); + completions.add(type); } Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) @@ -287,7 +322,7 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) .handler = {[ht](std::string s) { *ht = parseHashType(s); }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } @@ -300,13 +335,13 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< .handler = {[oht](std::string s) { *oht = std::optional { parseHashType(s) }; }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } -static void _completePath(std::string_view prefix, bool onlyDirs) +static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs) { - completionType = ctFilenames; + completions.setType(Completions::Type::Filenames); glob_t globbuf; int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR @@ -320,20 +355,20 @@ static void _completePath(std::string_view prefix, bool onlyDirs) auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } - completions->add(globbuf.gl_pathv[i]); + completions.add(globbuf.gl_pathv[i]); } } globfree(&globbuf); } -void completePath(size_t, std::string_view prefix) +void Args::completePath(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, false); + _completePath(completions, prefix, false); } -void completeDir(size_t, std::string_view prefix) +void Args::completeDir(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, true); + _completePath(completions, prefix, true); } Strings argvToStrings(int argc, char * * argv) @@ -368,10 +403,10 @@ MultiCommand::MultiCommand(const Commands & commands_) command = {s, i->second()}; command->second->parent = this; }}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { for (auto & [name, command] : commands) if (hasPrefix(name, prefix)) - completions->add(name); + completions.add(name); }} }); @@ -393,14 +428,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } -void MultiCommand::completionHook() -{ - if (command) - return command->second->completionHook(); - else - return Args::completionHook(); -} - nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 6457cceed..a5d7cbe4a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -15,16 +15,14 @@ enum HashType : char; class MultiCommand; +class RootArgs; + +class AddCompletions; + class Args { public: - /** - * Parse the command line, throwing a UsageError if something goes - * wrong. - */ - void parseCmdline(const Strings & cmdline); - /** * Return a short one-line description of the command. */ @@ -123,6 +121,25 @@ protected: { } }; + /** + * The basic function type of the completion callback. + * + * Used to define `CompleterClosure` and some common case completers + * that individual flags/arguments can use. + * + * The `AddCompletions` that is passed is an interface to the state + * stored as part of the root command + */ + typedef void CompleterFun(AddCompletions &, size_t, std::string_view); + + /** + * The closure type of the completion callback. + * + * This is what is actually stored as part of each Flag / Expected + * Arg. + */ + typedef std::function CompleterClosure; + /** * Description of flags / options * @@ -140,7 +157,7 @@ protected: std::string category; Strings labels; Handler handler; - std::function completer; + CompleterClosure completer; std::optional experimentalFeature; @@ -177,7 +194,7 @@ protected: std::string label; bool optional = false; Handler handler; - std::function completer; + CompleterClosure completer; }; /** @@ -211,13 +228,6 @@ protected: */ virtual void initialFlagsProcessed() {} - /** - * Called after the command line has been processed if we need to generate - * completions. Useful for commands that need to know the whole command line - * in order to know what completions to generate. - */ - virtual void completionHook() { } - public: void addFlag(Flag && flag); @@ -252,24 +262,30 @@ public: }); } + static CompleterFun completePath; + + static CompleterFun completeDir; + virtual nlohmann::json toJSON(); friend class MultiCommand; /** * The parent command, used if this is a subcommand. + * + * Invariant: An Args with a null parent must also be a RootArgs + * + * \todo this would probably be better in the CommandClass. + * getRoot() could be an abstract method that peels off at most one + * layer before recuring. */ MultiCommand * parent = nullptr; -private: - /** - * Experimental features needed when parsing args. These are checked - * after flag parsing is completed in order to support enabling - * experimental features coming after the flag that needs the - * experimental feature. + * Traverse parent pointers until we find the \ref RootArgs "root + * arguments" object. */ - std::set flagExperimentalFeatures; + RootArgs & getRoot(); }; /** @@ -320,8 +336,6 @@ public: bool processArgs(const Strings & args, bool finish) override; - void completionHook() override; - nlohmann::json toJSON() override; }; @@ -333,21 +347,40 @@ struct Completion { bool operator<(const Completion & other) const; }; -class Completions : public std::set { + +/** + * The abstract interface for completions callbacks + * + * The idea is to restrict the callback so it can only add additional + * completions to the collection, or set the completion type. By making + * it go through this interface, the callback cannot make any other + * changes, or even view the completions / completion type that have + * been set so far. + */ +class AddCompletions +{ public: - void add(std::string completion, std::string description = ""); + + /** + * The type of completion we are collecting. + */ + enum class Type { + Normal, + Filenames, + Attrs, + }; + + /** + * Set the type of the completions being collected + * + * \todo it should not be possible to change the type after it has been set. + */ + virtual void setType(Type type) = 0; + + /** + * Add a single completion to the collection + */ + virtual void add(std::string completion, std::string description = "") = 0; }; -extern std::shared_ptr completions; - -enum CompletionType { - ctNormal, - ctFilenames, - ctAttrs -}; -extern CompletionType completionType; - -void completePath(size_t, std::string_view prefix); - -void completeDir(size_t, std::string_view prefix); } diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh new file mode 100644 index 000000000..bb98732a1 --- /dev/null +++ b/src/libutil/args/root.hh @@ -0,0 +1,72 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/** + * The concrete implementation of a collection of completions. + * + * This is exposed so that the main entry point can print out the + * collected completions. + */ +struct Completions final : AddCompletions +{ + std::set completions; + Type type = Type::Normal; + + void setType(Type type) override; + void add(std::string completion, std::string description = "") override; +}; + +/** + * The outermost Args object. This is the one we will actually parse a command + * line with, whereas the inner ones (if they exists) are subcommands (and this + * is also a MultiCommand or something like it). + * + * This Args contains completions state shared between it and all of its + * descendent Args. + */ +class RootArgs : virtual public Args +{ +public: + /** Parse the command line, throwing a UsageError if something goes + * wrong. + */ + void parseCmdline(const Strings & cmdline); + + std::shared_ptr completions; + +protected: + + friend class Args; + + /** + * A pointer to the completion and its two arguments; a thunk; + */ + struct DeferredCompletion { + const CompleterClosure & completer; + size_t n; + std::string prefix; + }; + + /** + * Completions are run after all args and flags are parsed, so completions + * of earlier arguments can benefit from later arguments. + */ + std::vector deferredCompletions; + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set flagExperimentalFeatures; + +private: + + std::optional needsCompletion(std::string_view s); +}; + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc5..81efaafec 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,8 +6,13 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) +libutil_CXXFLAGS += -I src/libutil + libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +$(foreach i, $(wildcard $(d)/args/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) + ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fbc83b08e..504e35c81 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -21,8 +21,8 @@ struct CmdBundle : InstallableValueCommand .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b4e4156c0..0116eff2e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -36,8 +36,8 @@ public: .label = "flake-url", .optional = true, .handler = {&flakeUrl}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } @@ -52,9 +52,12 @@ public: return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); } - std::vector getFlakesForCompletion() override + std::vector getFlakeRefsForCompletion() override { - return {flakeUrl}; + return { + // Like getFlakeRef but with expandTilde calld first + parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + }; } }; @@ -762,8 +765,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, defaultTemplateAttrPathsPrefixes, diff --git a/src/nix/main.cc b/src/nix/main.cc index ee3878cc7..ffba10099 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,5 +1,6 @@ #include +#include "args/root.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" @@ -56,7 +57,7 @@ static bool haveInternet() std::string programPath; -struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs { bool useNet = true; bool refresh = false; @@ -241,10 +242,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) static NixArgs & getNixArgs(Command & cmd) { - assert(cmd.parent); - MultiCommand * toplevel = cmd.parent; - while (toplevel->parent) toplevel = toplevel->parent; - return dynamic_cast(*toplevel); + return dynamic_cast(cmd.getRoot()); } struct CmdHelp : Command @@ -412,16 +410,16 @@ void mainWrapped(int argc, char * * argv) Finally printCompletions([&]() { - if (completions) { - switch (completionType) { - case ctNormal: + if (args.completions) { + switch (args.completions->type) { + case Completions::Type::Normal: logger->cout("normal"); break; - case ctFilenames: + case Completions::Type::Filenames: logger->cout("filenames"); break; - case ctAttrs: + case Completions::Type::Attrs: logger->cout("attrs"); break; } - for (auto & s : *completions) + for (auto & s : args.completions->completions) logger->cout(s.completion + "\t" + trim(s.description)); } }); @@ -429,7 +427,7 @@ void mainWrapped(int argc, char * * argv) try { args.parseCmdline(argvToStrings(argc, argv)); } catch (UsageError &) { - if (!args.helpRequested && !completions) throw; + if (!args.helpRequested && !args.completions) throw; } if (args.helpRequested) { @@ -446,10 +444,7 @@ void mainWrapped(int argc, char * * argv) return; } - if (completions) { - args.completionHook(); - return; - } + if (args.completions) return; if (args.showVersion) { printVersion(programName); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index cb94bbd31..f509ccae8 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -175,8 +175,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand .label = "locked", .optional = true, .handler = {&locked}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 592de773c..055cf6d0d 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -38,17 +38,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions expectArgs({ .label = "package", .handler = {&_package}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); expectArgs({ .label = "dependency", .handler = {&_dependency}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); addFlag({ From fa9642ec459859557ce29e5f68413cf3b0846fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 7 Jun 2023 18:12:35 +0200 Subject: [PATCH 214/402] nix-shell: support single quotes in shebangs Single quotes are a basic feature of shell syntax that people expect to work. They are also more convenient for writing literal code expressions with less escaping. --- doc/manual/src/command-ref/nix-shell.md | 6 +++--- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix-build/nix-build.cc | 26 ++++++++++++++++++------- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 195b72be5..1eaf3c36a 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -235,14 +235,14 @@ package like Terraform: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])' terraform apply ``` > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use single or double quotes (`'`, `"`) when passing a simple Nix expression > in a nix-shell shebang. Finally, using the merging of multiple nix-shell shebangs the following @@ -251,7 +251,7 @@ branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])' #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index d0b516dfb..276252c37 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -12,4 +12,6 @@ - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). +- `nix-shell` shebang lines now support single-quoted arguments. + - `builtins.fetchTree` is now marked as stable. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 2895c5e3c..ae05444b7 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -40,7 +40,8 @@ static std::vector shellwords(const std::string & s) std::string cur; enum state { sBegin, - sQuote + sSingleQuote, + sDoubleQuote }; state st = sBegin; auto it = begin; @@ -56,15 +57,26 @@ static std::vector shellwords(const std::string & s) } } switch (*it) { + case '\'': + if (st != sDoubleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sSingleQuote : sBegin; + } + break; case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; + if (st != sSingleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sDoubleQuote : sBegin; + } break; case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; + if (st != sSingleQuote) { + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + } break; } } From 595010b631b0f92ee6eb4afb8494bc9c73e8513b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Thu, 8 Jun 2023 03:11:04 +0200 Subject: [PATCH 215/402] nix-shell: fix shebang whitespace parsing Leading whitespace after `nix-shell` used to produce an empty argument, while an empty argument at the end of the line was ignored. Fix the first issue by consuming the initial whitespace before calling shellwords; fix the second issue by returning immediately if whitespace is found at the end of the string instead of checking for an empty string. Also throw if quotes aren't terminated. --- src/nix-build/nix-build.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index ae05444b7..e62c4f6b1 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -34,7 +34,7 @@ extern char * * environ __attribute__((weak)); */ static std::vector shellwords(const std::string & s) { - std::regex whitespace("^(\\s+).*"); + std::regex whitespace("^\\s+"); auto begin = s.cbegin(); std::vector res; std::string cur; @@ -51,9 +51,10 @@ static std::vector shellwords(const std::string & s) if (regex_search(it, s.cend(), match, whitespace)) { cur.append(begin, it); res.push_back(cur); - cur.clear(); - it = match[1].second; + it = match[0].second; + if (it == s.cend()) return res; begin = it; + cur.clear(); } } switch (*it) { @@ -80,8 +81,9 @@ static std::vector shellwords(const std::string & s) break; } } + if (st != sBegin) throw Error("unterminated quote in shebang line"); cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); + res.push_back(cur); return res; } @@ -140,7 +142,7 @@ static void main_nix_build(int argc, char * * argv) for (auto line : lines) { line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) + if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$"))) for (const auto & word : shellwords(match[1].str())) args.push_back(word); } From c82066cf73c1b6d8f910ea68767035d0ead7c7d9 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 16:56:30 +0300 Subject: [PATCH 216/402] fix: Declare constructor as default --- src/libutil/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index a5d7cbe4a..ebd6fb67b 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -57,7 +57,7 @@ protected: std::function)> fun; size_t arity; - Handler() {} + Handler() = default; Handler(std::function)> && fun) : fun(std::move(fun)) From b205da16ef94d02e08fbe751237e333d8f53aca8 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:06:15 +0300 Subject: [PATCH 217/402] fix: Explicitly pass lambda scope variables. Default capture implicitly also capture *this, which would automatically be used if for example you referenced a method from the enclosing scope. --- src/libutil/args.hh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ebd6fb67b..021c2a937 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -84,29 +84,29 @@ protected: { } Handler(std::vector * dest) - : fun([=](std::vector ss) { *dest = ss; }) + : fun([dest](std::vector ss) { *dest = ss; }) , arity(ArityAny) { } Handler(std::string * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } Handler(std::optional * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } template Handler(T * dest, const T & val) - : fun([=](std::vector ss) { *dest = val; }) + : fun([dest, val](std::vector ss) { *dest = val; }) , arity(0) { } template Handler(I * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) @@ -114,7 +114,7 @@ protected: template Handler(std::optional * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) From a31fc5cc8643beae13b8f071fbeac50aac1a94e2 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:07:17 +0300 Subject: [PATCH 218/402] fix: Use `using` instead of `typedef` for type aliasing. Since C++ 11 we shouldn't use c-style `typedefs`. In addition, `using` can be templated. --- src/libutil/args.hh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 021c2a937..cee672d80 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -130,7 +130,7 @@ protected: * The `AddCompletions` that is passed is an interface to the state * stored as part of the root command */ - typedef void CompleterFun(AddCompletions &, size_t, std::string_view); + using CompleterFun = void(AddCompletions &, size_t, std::string_view); /** * The closure type of the completion callback. @@ -138,7 +138,7 @@ protected: * This is what is actually stored as part of each Flag / Expected * Arg. */ - typedef std::function CompleterClosure; + using CompleterClosure = std::function; /** * Description of flags / options @@ -148,7 +148,7 @@ protected: */ struct Flag { - typedef std::shared_ptr ptr; + using ptr = std::shared_ptr; std::string longName; std::set aliases; @@ -303,7 +303,7 @@ struct Command : virtual public Args */ virtual void run() = 0; - typedef int Category; + using Category = int; static constexpr Category catDefault = 0; @@ -312,7 +312,7 @@ struct Command : virtual public Args virtual Category category() { return catDefault; } }; -typedef std::map()>> Commands; +using Commands = std::map()>>; /** * An argument parser that supports multiple subcommands, From 90e3ed06f84d0f184de9ee60b8219ba40607387e Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:07:57 +0300 Subject: [PATCH 219/402] fix: Use default destructor. --- src/libutil/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index cee672d80..ff2bf3cab 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -296,7 +296,7 @@ struct Command : virtual public Args { friend class MultiCommand; - virtual ~Command() { } + virtual ~Command() = default; /** * Entry point to the command From e053eeb272d2a9115afe7f42077002a3de6b8633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 17 Jun 2023 14:54:34 +0200 Subject: [PATCH 220/402] tests: test nix-shell shebang quoting --- tests/functional/nix-shell.sh | 5 +++++ tests/functional/shell.shebang.nix | 10 ++++++++++ 2 files changed, 15 insertions(+) create mode 100755 tests/functional/shell.shebang.nix diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index edaa1249b..13403fadb 100644 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -84,6 +84,11 @@ chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] +# Test nix-shell shebang quoting +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.shebang.nix +chmod a+rx $TEST_ROOT/shell.shebang.nix +$TEST_ROOT/shell.shebang.nix + # Test 'nix develop'. nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' diff --git a/tests/functional/shell.shebang.nix b/tests/functional/shell.shebang.nix new file mode 100755 index 000000000..08e43d53c --- /dev/null +++ b/tests/functional/shell.shebang.nix @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --argstr s1 'foo "bar" \baz'"'"'qux' --argstr s2 "foo 'bar' \"\baz" --argstr s3 \foo\ bar\'baz --argstr s4 '' +#! nix-shell shell.shebang.nix --command true +{ s1, s2, s3, s4 }: +assert s1 == ''foo "bar" \baz'qux''; +assert s2 == "foo 'bar' \"baz"; +assert s3 == "foo bar'baz"; +assert s4 == ""; +(import {}).runCommand "nix-shell" {} "" From 765436e300b855922d5793092cc2a533c81d521d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Oct 2023 11:15:52 -0400 Subject: [PATCH 221/402] Add `builtins.addDrvOutputDependencies` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End goal: make `(mkDerivation x).drvPath` behave like a non-DrvDeep context. Problem: users won't be able to recover the DrvDeep behavior when nixpkgs makes this change. Solution: add this primop. The new primop is fairly simple, and is supposed to complement other existing ones (`builtins.storePath`, `builtins.outputOf`) so there are simple ways to construct strings with every type of string context element. (It allows nothing we couldn't already do with `builtins.getContext` and `builtins.appendContext`, which is also true of those other two primops.) This was originally in #8595, but then it was proposed to land some doc changes separately. So now the code changes proper is just moved to this, and the doc will be done in that. Co-authored-by: Robert Hensing Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.nore github.com> Co-authored-by: Valentin Gagarin **Example** + > + > Many operations require a string context to be empty because they are intended only to work with "regular" strings, and also to help users avoid unintentionally loosing track of string context elements. + > `builtins.hasContext` can help create better domain-specific errors in those case. + > + > ```nix + > name: meta: + > + > if builtins.hasContext name + > then throw "package name cannot contain string context" + > else { ${name} = meta; } + > ``` )", .fun = prim_hasContext }); -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; @@ -66,11 +73,83 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p static RegisterPrimOp primop_unsafeDiscardOutputDependency({ .name = "__unsafeDiscardOutputDependency", - .arity = 1, + .args = {"s"}, + .doc = R"( + Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element. + + This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). + + This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context, + whereas Nix normally tracks all dependencies consistently. + Safe operations "grow" but never "shrink" string contexts. + [`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things). + Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything. + + [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies + )", .fun = prim_unsafeDiscardOutputDependency }); +static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies"); + + auto contextSize = context.size(); + if (contextSize != 1) { + throw EvalError({ + .msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize), + .errPos = state.positions[pos] + }); + } + NixStringContext context2 { + (NixStringContextElem { std::visit(overloaded { + [&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep { + if (!c.path.isDerivation()) { + throw EvalError({ + .msg = hintfmt("path '%s' is not a derivation", + state.store->printStorePath(c.path)), + .errPos = state.positions[pos], + }); + } + return NixStringContextElem::DrvDeep { + .drvPath = c.path, + }; + }, + [&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep { + throw EvalError({ + .msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output), + .errPos = state.positions[pos], + }); + }, + [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { + /* Reuse original item because we want this to be idempotent. */ + return std::move(c); + }, + }, context.begin()->raw) }), + }; + + v.mkString(*s, context2); +} + +static RegisterPrimOp primop_addDrvOutputDependencies({ + .name = "__addDrvOutputDependencies", + .args = {"s"}, + .doc = R"( + Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element. + + The store path that is the constant string context element should point to a valid derivation, and end in `.drv`. + + The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element. + The latter is supported so this function is idempotent. + + This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies). + )", + .fun = prim_addDrvOutputDependencies +}); + + /* Extract the context of a string as a structured Nix value. The context is represented as an attribute set whose keys are the diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp new file mode 100644 index 000000000..ad91a22aa --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1: + + 1| builtins.addDrvOutputDependencies "" + | ^ + 2| + + error: context of string '' must have exactly one element, but has 0 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix new file mode 100644 index 000000000..dc9ee3ba2 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix @@ -0,0 +1 @@ +builtins.addDrvOutputDependencies "" diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp new file mode 100644 index 000000000..bb389db4e --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4: + + 17| + 18| in builtins.addDrvOutputDependencies combo-path + | ^ + 19| + + error: context of string '/nix/store/pg9yqs4yd85yhdm3f4i5dyaqp5jahrsz-fail.drv/nix/store/2dxd5frb715z451vbf7s8birlf3argbk-fail-2.drv' must have exactly one element, but has 2 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix new file mode 100644 index 000000000..dbde264df --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix @@ -0,0 +1,18 @@ +let + drv0 = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + drv1 = derivation { + name = "fail-2"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + combo-path = "${drv0.drvPath}${drv1.drvPath}"; + +in builtins.addDrvOutputDependencies combo-path diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp new file mode 100644 index 000000000..070381118 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4: + + 8| + 9| in builtins.addDrvOutputDependencies drv.outPath + | ^ + 10| + + error: `addDrvOutputDependencies` can only act on derivations, not on a derivation output such as 'out' diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix new file mode 100644 index 000000000..e379e1d95 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix @@ -0,0 +1,9 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + +in builtins.addDrvOutputDependencies drv.outPath diff --git a/tests/functional/lang/eval-okay-context-introspection.exp b/tests/functional/lang/eval-okay-context-introspection.exp index 03b400cc8..a136b0035 100644 --- a/tests/functional/lang/eval-okay-context-introspection.exp +++ b/tests/functional/lang/eval-okay-context-introspection.exp @@ -1 +1 @@ -[ true true true true true true ] +[ true true true true true true true true true true true true true ] diff --git a/tests/functional/lang/eval-okay-context-introspection.nix b/tests/functional/lang/eval-okay-context-introspection.nix index 50a78d946..8886cf32e 100644 --- a/tests/functional/lang/eval-okay-context-introspection.nix +++ b/tests/functional/lang/eval-okay-context-introspection.nix @@ -31,11 +31,29 @@ let (builtins.unsafeDiscardStringContext str) (builtins.getContext str); + # Only holds true if string context contains both a `DrvDeep` and + # `Opaque` element. + almostEtaRule = str: + str == builtins.addDrvOutputDependencies + (builtins.unsafeDiscardOutputDependency str); + + addDrvOutputDependencies_idempotent = str: + builtins.addDrvOutputDependencies str == + builtins.addDrvOutputDependencies (builtins.addDrvOutputDependencies str); + + rules = str: [ + (etaRule str) + (almostEtaRule str) + (addDrvOutputDependencies_idempotent str) + ]; + in [ (legit-context == desired-context) (reconstructed-path == combo-path) (etaRule "foo") - (etaRule drv.drvPath) (etaRule drv.foo.outPath) - (etaRule (builtins.unsafeDiscardOutputDependency drv.drvPath)) +] ++ builtins.concatMap rules [ + drv.drvPath + (builtins.addDrvOutputDependencies drv.drvPath) + (builtins.unsafeDiscardOutputDependency drv.drvPath) ] From c9528d20810c577fd394a8668bec645799d5eded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=A2=D1=80=D0=BE?= =?UTF-8?q?=D1=84=D0=B8=D0=BC=D0=BE=D0=B2?= <62882157+trofkm@users.noreply.github.com> Date: Mon, 23 Oct 2023 20:20:23 +0300 Subject: [PATCH 222/402] fix: Remove extra to from README.md (#9213) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 623a9722c..e1cace3b4 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Full reference documentation can be found in the [Nix manual](https://nixos.org/ ## Building And Developing See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to -to set up a development environment and build Nix from source. + set up a development environment and build Nix from source. ## Contributing From cd680bd53d90971211fa8926940bfe6d987ca6cf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 23 Oct 2023 19:22:33 +0200 Subject: [PATCH 223/402] Merge how-to section on S3 buckets into S3 store docs (#7972) Rather than having a misc tutorial page in the grab-bag "package management" section, this information should just be part of the S3 store docs. --------- Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 - .../src/advanced-topics/post-build-hook.md | 5 +- .../src/package-management/s3-substituter.md | 115 ------------------ src/libstore/s3-binary-cache-store.md | 100 ++++++++++++++- 4 files changed, 100 insertions(+), 121 deletions(-) delete mode 100644 doc/manual/src/package-management/s3-substituter.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 60ebeb138..f3233f8c9 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - - [Serving a Nix store via S3](package-management/s3-substituter.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index e4475bd9b..3c1cc9b36 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -17,9 +17,8 @@ the build loop. # Prerequisites -This tutorial assumes you have [configured an S3-compatible binary -cache](../package-management/s3-substituter.md), and that the `root` -user's default AWS profile can upload to the bucket. +This tutorial assumes you have configured an [S3-compatible binary cache](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) as a [substituter](../command-ref/conf-file.md#conf-substituters), +and that the `root` user's default AWS profile can upload to the bucket. # Set up a Signing Key diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md deleted file mode 100644 index d8a1d9105..000000000 --- a/doc/manual/src/package-management/s3-substituter.md +++ /dev/null @@ -1,115 +0,0 @@ -# Serving a Nix store via S3 - -Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) -for storing and fetching store paths from -Amazon S3 and S3-compatible services. This uses the same *binary* -cache mechanism that Nix usually uses to fetch prebuilt binaries from -[cache.nixos.org](https://cache.nixos.org/). - -In this example we will use the bucket named `example-nix-cache`. - -## Anonymous Reads to your S3-compatible binary cache - -If your binary cache is publicly accessible and does not require -authentication, the simplest and easiest way to use Nix with your S3 -compatible binary cache is to use the HTTP URL for that cache. - -For AWS S3 the binary cache URL for example bucket will be exactly - or -. For S3 compatible binary caches, consult that -cache's documentation. - -Your bucket will need the following bucket policy: - -```json -{ - "Id": "DirectReads", - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowDirectReads", - "Action": [ - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ], - "Principal": "*" - } - ] -} -``` - -## Authenticated Reads to your S3 binary cache - -For AWS S3 the binary cache URL for example bucket will be exactly -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Nix supports authenticated reads from Amazon S3 and S3 compatible binary -caches. - -Your bucket will need a bucket policy allowing the desired users to -perform the `s3:GetObject` and `s3:GetBucketLocation` action on all -objects in the bucket. The [anonymous policy given -above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be -updated to have a restricted `Principal` to support this. - -## Authenticated Writes to your S3-compatible binary cache - -Nix support fully supports writing to Amazon S3 and S3 compatible -buckets. The binary cache URL for our example bucket will be -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Your account will need the following IAM policy to upload to the cache: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "UploadToCache", - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts", - "s3:PutObject" - ], - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ] - } - ] -} -``` - -## Examples - -To upload with a specific credential profile for Amazon S3: - -```console -$ nix copy nixpkgs.hello \ - --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' -``` - -To upload to an S3-compatible binary cache: - -```console -$ nix copy nixpkgs.hello --to \ - 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' -``` diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md index 70fe0eb09..675470261 100644 --- a/src/libstore/s3-binary-cache-store.md +++ b/src/libstore/s3-binary-cache-store.md @@ -2,7 +2,103 @@ R"( **Store URL format**: `s3://`*bucket-name* -This store allows reading and writing a binary cache stored in an AWS -S3 bucket. +This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket. +This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store). + +For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly . +For S3 compatible binary caches, consult that cache's documentation. + +### Anonymous reads to your S3-compatible binary cache + +> If your binary cache is publicly accessible and does not require authentication, +> it is simplest to use the [HTTP Binary Cache Store] rather than S3 Binary Cache Store with +> instead of . + +Your bucket will need a +[bucket policy](https://docs.aws.amazon.com/AmazonS3/v1/userguide/bucket-policies.html) +like the following to be accessible: + +```json +{ + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] +} +``` + +### Authentication + +Nix will use the +[default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) +for authenticating requests to Amazon S3. + +Note that this means Nix will read environment variables and files with different idioms than with Nix's own settings, as implemented by the AWS SDK. +Consult the documentation linked above for further details. + +### Authenticated reads to your S3 binary cache + +Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket. +The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this. + +### Authenticated writes to your S3-compatible binary cache + +Your account will need an IAM policy to support uploading to the bucket: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +``` + +### Examples + +With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental). + +- To upload with a specific credential profile for Amazon S3: + + ```console + $ nix copy nixpkgs.hello \ + --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' + ``` + +- To upload to an S3-compatible binary cache: + + ```console + $ nix copy nixpkgs.hello --to \ + 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' + ``` )" From cde3c63617137a795f9d3e6fa38714d1a2c54bfa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 19:30:00 +0200 Subject: [PATCH 224/402] system-features: Typo There I was, thinking all of Apple's OSes started with lower case. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e90f70f5f..12fb48d93 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -716,7 +716,7 @@ public: - `apple-virt` - Included on darwin if virtualization is available. + Included on Darwin if virtualization is available. - `kvm` From abb1c829c8ced0680ffa7ef9e45c1844fb24bcb5 Mon Sep 17 00:00:00 2001 From: Vignesh Date: Tue, 24 Oct 2023 14:48:00 +0530 Subject: [PATCH 225/402] Release notes updated for #9150 reverted (#9227) --- doc/manual/src/release-notes/rl-2.15.md | 2 +- doc/manual/src/release-notes/rl-2.7.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 4faf0b143..133121999 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -44,7 +44,7 @@ (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. - `nix store info` and `nix doctor` now display this information. + `nix store ping` and `nix doctor` now display this information. * The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md index dd649e166..2f3879422 100644 --- a/doc/manual/src/release-notes/rl-2.7.md +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -24,7 +24,7 @@ [repository](https://github.com/NixOS/bundlers) has various bundlers implemented. -* `nix store info` now reports the version of the remote Nix daemon. +* `nix store ping` now reports the version of the remote Nix daemon. * `nix flake {init,new}` now display information about which files have been created. From f269911641fcc6cef836a07393feae9d067096a8 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 24 Oct 2023 11:22:02 +0200 Subject: [PATCH 226/402] Document builtins.substring negative length behavior (#9226) --- src/libexpr/primops.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5033d4e2d..704e7007b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3720,10 +3720,11 @@ static RegisterPrimOp primop_substring({ .doc = R"( Return the substring of *s* from character position *start* (zero-based) up to but not including *start + len*. If *start* is - greater than the length of the string, an empty string is returned, - and if *start + len* lies beyond the end of the string, only the - substring up to the end of the string is returned. *start* must be - non-negative. For example, + greater than the length of the string, an empty string is returned. + If *start + len* lies beyond the end of the string or *len* is `-1`, + only the substring up to the end of the string is returned. + *start* must be non-negative. + For example, ```nix builtins.substring 0 3 "nixos" From eaced12c94e13f0fedd8324f4f9bc8684ea351ae Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:01 +0200 Subject: [PATCH 227/402] Fix signed vs. unsigned comparison warning and improve code --- src/libstore/builtins/buildenv.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 7bba33fb9..c8911d153 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -174,15 +174,19 @@ void builtinBuildenv(const BasicDerivation & drv) /* Convert the stuff we get from the environment back into a * coherent data type. */ Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + { + auto derivations = tokenizeString(getAttr("derivations")); + + auto itemIt = derivations.begin(); + while (itemIt != derivations.end()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + const bool active = "false" != *itemIt++; + const int priority = stoi(*itemIt++); + const size_t outputs = stoul(*itemIt++); + + for (size_t n {0}; n < outputs; n++) { + pkgs.emplace_back(std::move(*itemIt++), active, priority); + } } } From b113d925de50c5f40c4fd694167adde3eeb2b060 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:23 +0200 Subject: [PATCH 228/402] Fix warning --- src/libstore/content-address.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 52c60154c..a5f7cdf81 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -29,12 +29,13 @@ std::string ContentAddressMethod::renderPrefix() const ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { - ContentAddressMethod method = FileIngestionMethod::Flat; - if (splitPrefix(m, "r:")) - method = FileIngestionMethod::Recursive; - else if (splitPrefix(m, "text:")) - method = TextIngestionMethod {}; - return method; + if (splitPrefix(m, "r:")) { + return FileIngestionMethod::Recursive; + } + else if (splitPrefix(m, "text:")) { + return TextIngestionMethod {}; + } + return FileIngestionMethod::Flat; } std::string ContentAddressMethod::render(HashType ht) const From 7f71fc7540502295202b075f82e89fcf993e7e3a Mon Sep 17 00:00:00 2001 From: Pol Dellaiera Date: Mon, 23 Oct 2023 23:46:06 +0200 Subject: [PATCH 229/402] fix: make sure `tar` reproducibility flags are set --- tests/functional/tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 6e621a28c..e59ee400e 100644 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -18,7 +18,7 @@ test_tarball() { local compressor="$2" tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball + (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies From 8d9e0b7aed2e3bd218c5344d1dcf4a8632e0d63a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 04:28:35 +0200 Subject: [PATCH 230/402] document the store concept (#9206) * document the store concept and its purpose reword the glossary to link to more existing information instead of repeating it. move the store documentation to the top of the table of contents, in front of the Nix language. this will provide a natural place to document other aspects of the store as well as the various store types. move the package management section after the Nix language and before Advanced Topics to follow the pattern to layer more complex concepts on top of each other. this structure of the manual will also nudge beginners to learn Nix bottom-up and hopefully make more likely that they understand underlying concepts first before delving into complex use cases that may or may not be easy to implement with what's currently there. [John adds this note] The sort of beginner who likes to dive straight into reference documentation should prefer this approach. Conversely, the sort of beginner who would prefer the opposite top-down approach of trying to solve problems before they understand everything that is going on is better off reading other tutorial/guide material anyways, and will just "random-access" the reference manual as a last resort. For such random-access the order doesn't matter, so this restructure doesn't make them any worse off. Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 20 +++++++------ doc/manual/src/architecture/architecture.md | 1 + doc/manual/src/glossary.md | 30 +++++++++---------- .../file-system-object.md | 0 doc/manual/src/store/index.md | 4 +++ 5 files changed, 30 insertions(+), 25 deletions(-) rename doc/manual/src/{architecture => store}/file-system-object.md (100%) create mode 100644 doc/manual/src/store/index.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f3233f8c9..2fe77d2c6 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,14 +16,8 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) -- [Package Management](package-management/package-management.md) - - [Profiles](package-management/profiles.md) - - [Garbage Collection](package-management/garbage-collection.md) - - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Sharing Packages Between Machines](package-management/sharing-packages.md) - - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - - [Copying Closures via SSH](package-management/copy-closure.md) - - [Serving a Nix store via SSH](package-management/ssh-substituter.md) +- [Nix Store](store/index.md) + - [File System Object](store/file-system-object.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) @@ -35,7 +29,16 @@ - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) +- [Package Management](package-management/package-management.md) + - [Profiles](package-management/profiles.md) + - [Garbage Collection](package-management/garbage-collection.md) + - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - [Advanced Topics](advanced-topics/advanced-topics.md) + - [Sharing Packages Between Machines](package-management/sharing-packages.md) + - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) + - [Copying Closures via SSH](package-management/copy-closure.md) + - [Serving a Nix store via SSH](package-management/ssh-substituter.md) + - [Serving a Nix store via S3](package-management/s3-substituter.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) @@ -97,7 +100,6 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) - - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 9e969972e..6e832e1f9 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -59,6 +59,7 @@ The [Nix language](../language/index.md) evaluator transforms Nix expressions in The command line interface and Nix expressions are what users deal with most. > **Note** +> > The Nix language itself does not have a notion of *packages* or *configurations*. > As far as we are concerned here, the inputs and results of a build plan are just data. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index d49d5e52e..ad3cc147b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -58,22 +58,16 @@ - [store]{#gloss-store} - The location in the file system where store objects live. Typically - `/nix/store`. + A collection of store objects, with operations to manipulate that collection. + See [Nix Store] for details. - From the perspective of the location where Nix is - invoked, the Nix store can be referred to - as a "_local_" or a "_remote_" one: + There are many types of stores. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. - + A [local store]{#gloss-local-store} exists on the filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. Local stores can be used for building derivations. - - + A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. + From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_. + Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`. + Local stores can be used for building [derivations](#derivation). + See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details. [store]: #gloss-store [local store]: #gloss-local-store @@ -103,15 +97,19 @@ The Nix data model for representing simplified file system data. - See [File System Object](@docroot@/architecture/file-system-object.md) for details. + See [File System Object](@docroot@/store/file-system-object.md) for details. [file system object]: #gloss-file-system-object - [store object]{#gloss-store-object} - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. + Part of the contents of a [store]. + + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. It can be referred to by a [store path]. + See [Store Object](@docroot@/store/index.md#store-object) for details. + [store object]: #gloss-store-object - [IFD]{#gloss-ifd} diff --git a/doc/manual/src/architecture/file-system-object.md b/doc/manual/src/store/file-system-object.md similarity index 100% rename from doc/manual/src/architecture/file-system-object.md rename to doc/manual/src/store/file-system-object.md diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md new file mode 100644 index 000000000..316e04179 --- /dev/null +++ b/doc/manual/src/store/index.md @@ -0,0 +1,4 @@ +# Nix Store + +The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them. +There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches. From 78278f2b3f1c1c55aec7c147d2e266b805863985 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 12:00:56 +0200 Subject: [PATCH 231/402] add notes on comments in code samples --- doc/manual/src/contributing/documentation.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index f73ab2149..190d367db 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -73,6 +73,17 @@ It should therefore aim to be correct, consistent, complete, and easy to navigat Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. +- Always explain code examples in the text. + + Use comments in code samples very sparingly, for instance to highlight a particular aspect. + Readers tend to glance over large amounts of code when scanning for information. + + Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether. + + If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one. + If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end. + This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject. + - Use British English. This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. From 00c90eae95c5987a8352dd786d9687f3d213f54a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 12:01:17 +0200 Subject: [PATCH 232/402] add note on highlighting examples and syntax definitions --- doc/manual/src/contributing/documentation.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index f73ab2149..e35a29d93 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -151,6 +151,24 @@ Please observe these guidelines to ease reviews: > This is a note. ``` + Highlight examples as such: + + ```` + > **Example** + > + > ```console + > $ nix --version + > ``` + ```` + + Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation: + + ```` + > **Syntax** + > + > *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}` + ```` + ### The `@docroot@` variable `@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. From 7bc45c6136af4bb922285ee786f58e54753a2b0d Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Wed, 25 Oct 2023 14:49:30 +0200 Subject: [PATCH 233/402] docs: clarify flake types and implied defaults --- src/nix/flake.md | 77 +++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index f08648417..d8b5bf435 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -98,7 +98,15 @@ Here are some examples of flake references in their URL-like representation: ## Path-like syntax -Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity). +Flakes corresponding to a local path can also be referred to by a direct +path reference, either `/absolute/path/to/the/flake` or`./relative/path/to/the/flake`. +Note that the leading `./` is mandatory for relative paths. If it is +omitted, the path will be interpreted as [URL-like syntax](#url-like-syntax), +which will cause error messages like this: + +```console +error: cannot find flake 'flake:relative/path/to/the/flake' in the flake registries +``` The semantic of such a path is as follows: @@ -153,18 +161,39 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `path`: arbitrary local directories, or local Git trees. The - required attribute `path` specifies the path of the flake. The URL - form is +* `indirect`: *The default*. Indirection through the flake registry. + These have the form ``` - [path:](\?(/(/rev)?)? ``` - where *path* is an absolute path. + These perform a lookup of `` in the flake registry. For + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. - *path* must be a directory in the file system containing a file - named `flake.nix`. + For example, these are valid indirect flake references: + + * `nixpkgs` + * `nixpkgs/nixos-unstable` + * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `sub/dir` (if a flake named `sub` is in the registry) + +* `path`: arbitrary local directories. The required attribute `path` + specifies the path of the flake. The URL form is + + ``` + path:(\?)? + ``` + + where *path* is an absolute path to a directory in the file system + containing a file named `flake.nix`. + + If the flake at *path* is not inside a git repository, the `path:` + prefix is implied and can be omitted. *path* generally must be an absolute path. However, on the command line, it can be a relative path (e.g. `.` or `./foo`) which is @@ -173,20 +202,24 @@ Currently the `type` attribute can be one of the following: (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). + For example, these are valid path flake references: + + * `path:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `git`: Git repositories. The location of the repository is specified by the attribute `url`. They have the URL form ``` - git(+http|+https|+ssh|+git|+file|):(//)?(\?)? + git(+http|+https|+ssh|+git|+file):(//)?(\?)? ``` - or - - ``` - @: - ``` + If *path* starts with `/` (or `./` when used as an argument on the + command line) and is a local path to a git repository, the leading + `git:` or `+file` prefixes are implied and can be omitted. The `ref` attribute defaults to resolving the `HEAD` reference. @@ -203,6 +236,9 @@ Currently the `type` attribute can be one of the following: For example, the following are valid Git flake references: + * `git:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` @@ -324,19 +360,6 @@ Currently the `type` attribute can be one of the following: * `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c` * `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht` -* `indirect`: Indirections through the flake registry. These have the - form - - ``` - [flake:](/(/rev)?)? - ``` - - These perform a lookup of `` in the flake registry. For - example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake - references. The specified `rev` and/or `ref` are merged with the - entry in the registry; see [nix registry](./nix3-registry.md) for - details. - # Flake format As an example, here is a simple `flake.nix` that depends on the From f555c98a343fbed98b0c01b1b19e6796feaea936 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:37 +0200 Subject: [PATCH 234/402] Improve loop over gid container --- src/libstore/lock.cc | 60 +++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 7202a64b3..165e4969f 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -7,6 +7,31 @@ namespace nix { +#if __linux__ + +static std::vector get_group_list(const char *username, gid_t group_id) +{ + std::vector gids; + gids.resize(32); // Initial guess + + auto getgroupl_failed {[&] { + int ngroups = gids.size(); + int err = getgrouplist(username, group_id, gids.data(), &ngroups); + gids.resize(ngroups); + return err == -1; + }}; + + // The first error means that the vector was not big enough. + // If it happens again, there is some different problem. + if (getgroupl_failed() && getgroupl_failed()) { + throw SysError("failed to get list of supplementary groups for '%s'", username); + } + + return gids; +} +#endif + + struct SimpleUserLock : UserLock { AutoCloseFD fdUserLock; @@ -67,37 +92,14 @@ struct SimpleUserLock : UserLock throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build - user. This is usually either empty or contains a - group such as "kvm". */ - int ngroups = 32; // arbitrary initial guess - std::vector gids; - gids.resize(ngroups); - - int err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - - /* Our initial size of 32 wasn't sufficient, the - correct size has been stored in ngroups, so we try - again. */ - if (err == -1) { - gids.resize(ngroups); - err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - } - - // If it failed once more, then something must be broken. - if (err == -1) - throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + /* Get the list of supplementary groups of this user. This is + * usually either empty or contains a group such as "kvm". */ // Finally, trim back the GID list to its real size. - for (auto i = 0; i < ngroups; i++) - if (gids[i] != lock->gid) - lock->supplementaryGIDs.push_back(gids[i]); + for (auto gid : get_group_list(pw->pw_name, pw->pw_gid)) { + if (gid != lock->gid) + lock->supplementaryGIDs.push_back(gid); + } #endif return lock; From 95d657c8b3ae4282e24628ba7426edb90c8f3942 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Oct 2023 18:18:15 +0200 Subject: [PATCH 235/402] Input: Replace markFileChanged() by putFile() Committing a lock file using markFileChanged() required the input to be writable by the caller in the local filesystem (using the path returned by getSourcePath()). putFile() abstracts over this. --- src/libexpr/flake/flake.cc | 67 +++++++++++++++++++----------------- src/libfetchers/cache.hh | 1 + src/libfetchers/fetchers.cc | 17 +++++---- src/libfetchers/fetchers.hh | 21 +++++++---- src/libfetchers/git.cc | 20 +++++++---- src/libfetchers/indirect.cc | 1 + src/libfetchers/mercurial.cc | 21 +++++++---- src/libfetchers/path.cc | 20 +++++++++-- 8 files changed, 109 insertions(+), 59 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index be2cf014c..5c2a1623a 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -623,12 +623,7 @@ LockedFlake lockFlake( debug("new lock file: %s", newLockFile); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto sourcePath = topRef.input.getSourcePath(); - auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; - if (lockFlags.outputLockFilePath) { - outputLockFilePath = lockFlags.outputLockFilePath; - } /* Check whether we need to / can write the new lock file. */ if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { @@ -636,7 +631,7 @@ LockedFlake lockFlake( auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (outputLockFilePath) { + if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -644,41 +639,49 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - bool lockFileExists = pathExists(*outputLockFilePath); + auto newLockFileS = fmt("%s\n", newLockFile); - if (lockFileExists) { - auto s = chomp(diff); - if (s.empty()) - warn("updating lock file '%s'", *outputLockFilePath); - else - warn("updating lock file '%s':\n%s", *outputLockFilePath, s); - } else - warn("creating lock file '%s'", *outputLockFilePath); + if (lockFlags.outputLockFilePath) + writeFile(*lockFlags.outputLockFilePath, newLockFileS); + else { + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; - newLockFile.write(*outputLockFilePath); + bool lockFileExists = pathExists(*outputLockFilePath); - std::optional commitMessage = std::nullopt; - if (lockFlags.commitLockFile) { - if (lockFlags.outputLockFilePath) { - throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); - } - std::string cm; + if (lockFileExists) { + auto s = chomp(diff); + if (s.empty()) + warn("updating lock file '%s'", *outputLockFilePath); + else + warn("updating lock file '%s':\n%s", *outputLockFilePath, s); + } else + warn("creating lock file '%s'", *outputLockFilePath); - cm = fetchSettings.commitLockFileSummary.get(); + std::optional commitMessage = std::nullopt; - if (cm == "") { - cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + if (lockFlags.commitLockFile) { + if (lockFlags.outputLockFilePath) { + throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); + } + std::string cm; + + cm = fetchSettings.commitLockFileSummary.get(); + + if (cm == "") { + cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + } + + cm += "\n\nFlake lock file updates:\n\n"; + cm += filterANSIEscapes(diff, true); + commitMessage = cm; } - cm += "\n\nFlake lock file updates:\n\n"; - cm += filterANSIEscapes(diff, true); - commitMessage = cm; + topRef.input.putFile( + CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), + newLockFileS, commitMessage); } - topRef.input.markChangedFile( - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - commitMessage); - /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index ae398d040..af34e66ce 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -2,6 +2,7 @@ ///@file #include "fetchers.hh" +#include "path.hh" namespace nix::fetchers { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5688c4dc1..c339c441b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -196,12 +196,13 @@ std::optional Input::getSourcePath() const return scheme->getSourcePath(*this); } -void Input::markChangedFile( - std::string_view file, +void Input::putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const { assert(scheme); - return scheme->markChangedFile(*this, file, commitMsg); + return scheme->putFile(*this, path, contents, commitMsg); } std::string Input::getName() const @@ -292,14 +293,18 @@ Input InputScheme::applyOverrides( return input; } -std::optional InputScheme::getSourcePath(const Input & input) +std::optional InputScheme::getSourcePath(const Input & input) const { return {}; } -void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +void InputScheme::putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const { - assert(false); + throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); } void InputScheme::clone(const Input & input, const Path & destDir) const diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index ac605ff8e..4212a3e1f 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,13 +3,13 @@ #include "types.hh" #include "hash.hh" -#include "path.hh" +#include "canon-path.hh" #include "attrs.hh" #include "url.hh" #include -namespace nix { class Store; } +namespace nix { class Store; class StorePath; } namespace nix::fetchers { @@ -90,8 +90,13 @@ public: std::optional getSourcePath() const; - void markChangedFile( - std::string_view file, + /** + * Write a file to this input, for input types that support + * writing. Optionally commit the change (for e.g. Git inputs). + */ + void putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const; std::string getName() const; @@ -135,9 +140,13 @@ struct InputScheme virtual void clone(const Input & input, const Path & destDir) const; - virtual std::optional getSourcePath(const Input & input); + virtual std::optional getSourcePath(const Input & input) const; - virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + virtual void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const; virtual std::pair fetch(ref store, const Input & input) = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 26b8987d6..4bfd53b0e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -354,7 +354,7 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -362,18 +362,26 @@ struct GitInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto root = getSourcePath(input); + if (!root) + throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); + + writeFile((CanonPath(*root) + path).abs(), contents); + auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); + { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); if (commitMsg) runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg }); + { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 9a71df3d4..b18411bdc 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "url-parts.hh" +#include "path.hh" namespace nix::fetchers { diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index f830a3271..97c48afc9 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -109,7 +109,7 @@ struct MercurialInputScheme : InputScheme return res; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -117,18 +117,27 @@ struct MercurialInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto [isLocal, repoPath] = getActualUrl(input); + if (!isLocal) + throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string()); + + auto absPath = CanonPath(repoPath) + path; + + writeFile(absPath.abs(), contents); // FIXME: shut up if file is already tracked. runHg( - { "add", *sourcePath + "/" + std::string(file) }); + { "add", absPath.abs() }); if (commitMsg) runHg( - { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + { "commit", absPath.abs(), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index d829609b5..22be0f1fe 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -66,14 +66,28 @@ struct PathInputScheme : InputScheme }; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { return getStrAttr(input.attrs, "path"); } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - // nothing to do + writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents); + } + + CanonPath getAbsPath(const Input & input) const + { + auto path = getStrAttr(input.attrs, "path"); + + if (path[0] == '/') + return CanonPath(path); + + throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } std::pair fetch(ref store, const Input & _input) override From 15c430f38971c2f852effec22392cbe1da511aec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Oct 2023 18:44:09 +0200 Subject: [PATCH 236/402] Remove unused LockFile::write() --- src/libexpr/flake/lockfile.cc | 6 ------ src/libexpr/flake/lockfile.hh | 2 -- 2 files changed, 8 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index f3ea9063f..3e99fb2d4 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -214,12 +214,6 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) return stream; } -void LockFile::write(const Path & path) const -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", *this)); -} - std::optional LockFile::isUnlocked() const { std::set> nodes; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index ba4c0c848..5a1493404 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -65,8 +65,6 @@ struct LockFile static LockFile read(const Path & path); - void write(const Path & path) const; - /** * Check whether this lock file has any unlocked inputs. */ From 46028ff76493439921a5a9200d14c015f0a0c025 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Oct 2023 07:05:48 +0200 Subject: [PATCH 237/402] doc: Fix fetchGit default name (#9241) --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a99b0e500..767f559be 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -392,7 +392,7 @@ static RegisterPrimOp primop_fetchGit({ The URL of the repo. - - `name` (default: *basename of the URL*) + - `name` (default: `source`) The name of the directory the repo should be exported to in the store. From b66381e8d8728c040197ec78ed47d5eff88e1d0e Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 20:16:34 +0200 Subject: [PATCH 238/402] Use using instead of typedef --- src/libstore/path-info.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a82e643ae..580e94189 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,7 +29,7 @@ struct SubstitutablePathInfo uint64_t narSize; }; -typedef std::map SubstitutablePathInfos; +using SubstitutablePathInfos = std::map; struct UnkeyedValidPathInfo @@ -136,6 +136,6 @@ struct ValidPathInfo : UnkeyedValidPathInfo { virtual ~ValidPathInfo() { } }; -typedef std::map ValidPathInfos; +using ValidPathInfos = std::map; } From 28c39c370c99148ccc6576d429b41a1f46b08174 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 20:16:46 +0200 Subject: [PATCH 239/402] Provide default value for id to fix warning --- src/libstore/path-info.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 580e94189..c4c4a6366 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -42,7 +42,7 @@ struct UnkeyedValidPathInfo StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only + uint64_t id = 0; // internal use only /** * Whether the path is ultimately trusted, that is, it's a From e69c764708a78ae2ea38b4c37e5c07119e7097d0 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Thu, 26 Oct 2023 20:53:03 +0100 Subject: [PATCH 240/402] local-derivation-goal.cc: slightly clarify waiting message Before the change builder ID exhaustion printed the following message: [0/1 built] waiting for UID to build '/nix/store/hiy9136x0iyib4ssh3w3r5m8pxjnad50-python3.11-breathe-4.35.0.drv' After the change it should be: [0/1 built] waiting for a free build user ID for '/nix/store/hiy9136x0iyib4ssh3w3r5m8pxjnad50-python3.11-breathe-4.35.0.drv' --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b5b060d95..738e7051e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -227,7 +227,7 @@ void LocalDerivationGoal::tryLocalBuild() if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } From a419b6149705bddff60215a0d21ef355857ef2c5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Oct 2023 17:58:45 -0400 Subject: [PATCH 241/402] Turn derivation unit tests into unit characterization tests The brings a number of advantages, including: - Easier to update test data if design changes (and I do think our derivation JSON is not yet complaint with the guidelines). - Easier to reuse test data in other implementations, inching closer to compliance tests for Nix *the concept* rather than any one implementation. --- src/libstore/tests/characterization.hh | 5 + src/libstore/tests/common-protocol.cc | 8 +- src/libstore/tests/derivation.cc | 273 ++++++++---------- src/libstore/tests/protocol.hh | 4 +- .../derivation/bad-old-version-dyn-deps.drv | 1 + .../libstore/derivation/bad-version.drv | 1 + .../libstore/derivation/dynDerivationDeps.drv | 1 + .../derivation/dynDerivationDeps.json | 38 +++ .../derivation/output-caFixedFlat.json | 5 + .../derivation/output-caFixedNAR.json | 5 + .../derivation/output-caFixedText.json | 5 + .../derivation/output-caFloating.json | 3 + .../libstore/derivation/output-deferred.json | 1 + .../libstore/derivation/output-impure.json | 4 + .../derivation/output-inputAddressed.json | 3 + unit-test-data/libstore/derivation/simple.drv | 1 + .../libstore/derivation/simple.json | 25 ++ 17 files changed, 228 insertions(+), 155 deletions(-) create mode 100644 unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv create mode 100644 unit-test-data/libstore/derivation/bad-version.drv create mode 100644 unit-test-data/libstore/derivation/dynDerivationDeps.drv create mode 100644 unit-test-data/libstore/derivation/dynDerivationDeps.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedFlat.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedNAR.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedText.json create mode 100644 unit-test-data/libstore/derivation/output-caFloating.json create mode 100644 unit-test-data/libstore/derivation/output-deferred.json create mode 100644 unit-test-data/libstore/derivation/output-impure.json create mode 100644 unit-test-data/libstore/derivation/output-inputAddressed.json create mode 100644 unit-test-data/libstore/derivation/simple.drv create mode 100644 unit-test-data/libstore/derivation/simple.json diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh index 5f366cb42..46bf4b2e5 100644 --- a/src/libstore/tests/characterization.hh +++ b/src/libstore/tests/characterization.hh @@ -20,4 +20,9 @@ static bool testAccept() { return getEnv("_NIX_TEST_ACCEPT") == "1"; } +constexpr std::string_view cannotReadGoldenMaster = + "Cannot read golden master because another test is also updating it"; + +constexpr std::string_view updatingGoldenMaster = + "Updating golden master"; } diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc index 61c2cb70c..b3f4977d2 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/src/libstore/tests/common-protocol.cc @@ -24,14 +24,14 @@ public: { if (testAccept()) { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + GTEST_SKIP() << cannotReadGoldenMaster; } else { - auto expected = readFile(goldenMaster(testStem)); + auto encoded = readFile(goldenMaster(testStem)); T got = ({ - StringSource from { expected }; + StringSource from { encoded }; CommonProto::Serialise::read( *store, CommonProto::ReadConn { .from = from }); @@ -59,7 +59,7 @@ public: { createDirs(dirOf(file)); writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; + GTEST_SKIP() << updatingGoldenMaster; } else { diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index c360c9707..ca0cdff71 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -5,9 +5,12 @@ #include "derivations.hh" #include "tests/libstore.hh" +#include "tests/characterization.hh" namespace nix { +using nlohmann::json; + class DerivationTest : public LibStoreTest { public: @@ -16,6 +19,12 @@ public: * to worry about race conditions if the tests run concurrently. */ ExperimentalFeatureSettings mockXpSettings; + + Path unitTestData = getUnitTestData() + "/libstore/derivation"; + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem; + } }; class CaDerivationTest : public DerivationTest @@ -46,7 +55,7 @@ TEST_F(DerivationTest, BadATerm_version) { ASSERT_THROW( parseDerivation( *store, - R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + readFile(goldenMaster("bad-version.drv")), "whatever", mockXpSettings), FormatError); @@ -56,50 +65,61 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { ASSERT_THROW( parseDerivation( *store, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + readFile(goldenMaster("bad-old-version-dyn-deps.drv")), "dyn-dep-derivation", mockXpSettings), FormatError); } -#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (DerivationOutput { VAL }).toJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME)); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - DerivationOutput { VAL }, \ - DerivationOutput::fromJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME, \ - STR ## _json, \ - mockXpSettings)); \ +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = json::parse( \ + readFile(goldenMaster("output-" #NAME ".json"))); \ + DerivationOutput got = DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + encoded, \ + mockXpSettings); \ + DerivationOutput expected { VAL }; \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ + auto file = goldenMaster("output-" #NAME ".json"); \ + \ + json got = DerivationOutput { VAL }.toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got.dump(2) + "\n"); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = json::parse(readFile(file)); \ + ASSERT_EQ(got, expected); \ + } \ } TEST_JSON(DerivationTest, inputAddressed, - R"({ - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", (DerivationOutput::InputAddressed { .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), }), "drv-name", "output-name") TEST_JSON(DerivationTest, caFixedFlat, - R"({ - "hashAlgo": "sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .method = FileIngestionMethod::Flat, @@ -109,11 +129,6 @@ TEST_JSON(DerivationTest, caFixedFlat, "drv-name", "output-name") TEST_JSON(DerivationTest, caFixedNAR, - R"({ - "hashAlgo": "r:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .method = FileIngestionMethod::Recursive, @@ -123,11 +138,6 @@ TEST_JSON(DerivationTest, caFixedNAR, "drv-name", "output-name") TEST_JSON(DynDerivationTest, caFixedText, - R"({ - "hashAlgo": "text:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), @@ -136,9 +146,6 @@ TEST_JSON(DynDerivationTest, caFixedText, "drv-name", "output-name") TEST_JSON(CaDerivationTest, caFloating, - R"({ - "hashAlgo": "r:sha256" - })", (DerivationOutput::CAFloating { .method = FileIngestionMethod::Recursive, .hashType = htSHA256, @@ -146,15 +153,10 @@ TEST_JSON(CaDerivationTest, caFloating, "drv-name", "output-name") TEST_JSON(DerivationTest, deferred, - R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") TEST_JSON(ImpureDerivationTest, impure, - R"({ - "hashAlgo": "r:sha256", - "impure": true - })", (DerivationOutput::Impure { .method = FileIngestionMethod::Recursive, .hashType = htSHA256, @@ -163,43 +165,79 @@ TEST_JSON(ImpureDerivationTest, impure, #undef TEST_JSON -#define TEST_JSON(FIXTURE, NAME, STR, VAL) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (VAL).toJSON(*store)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - (VAL), \ - Derivation::fromJSON( \ - *store, \ - STR ## _json, \ - mockXpSettings)); \ +#define TEST_JSON(FIXTURE, NAME, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = json::parse( \ + readFile(goldenMaster( #NAME ".json"))); \ + Derivation expected { VAL }; \ + Derivation got = Derivation::fromJSON( \ + *store, \ + encoded, \ + mockXpSettings); \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ + auto file = goldenMaster( #NAME ".json"); \ + \ + json got = Derivation { VAL }.toJSON(*store); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got.dump(2) + "\n"); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = json::parse(readFile(file)); \ + ASSERT_EQ(got, expected); \ + } \ } -#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ - ASSERT_EQ( \ - STR, \ - (VAL).unparse(*store, false)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ - auto parsed = parseDerivation( \ - *store, \ - STR, \ - DRV_NAME, \ - mockXpSettings); \ - ASSERT_EQ( \ - (VAL).toJSON(*store), \ - parsed.toJSON(*store)); \ - ASSERT_EQ( \ - (VAL), \ - parsed); \ +#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = readFile(goldenMaster( #NAME ".drv")); \ + Derivation expected { VAL }; \ + auto got = parseDerivation( \ + *store, \ + std::move(encoded), \ + DRV_NAME, \ + mockXpSettings); \ + ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ + auto file = goldenMaster( #NAME ".drv"); \ + \ + auto got = (VAL).unparse(*store, false); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = readFile(file); \ + ASSERT_EQ(got, expected); \ + } \ } Derivation makeSimpleDrv(const Store & store) { @@ -236,36 +274,9 @@ Derivation makeSimpleDrv(const Store & store) { return drv; } -TEST_JSON(DerivationTest, simple, - R"({ - "name": "simple-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": {}, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeSimpleDrv(*store)) +TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store)) TEST_ATERM(DerivationTest, simple, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", makeSimpleDrv(*store), "simple-derivation") @@ -321,45 +332,9 @@ Derivation makeDynDepDerivation(const Store & store) { return drv; } -TEST_JSON(DynDerivationTest, dynDerivationDeps, - R"({ - "name": "dyn-dep-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": { - "cat": { - "dynamicOutputs": {}, - "outputs": ["kitten"] - }, - "goose": { - "dynamicOutputs": {}, - "outputs": ["gosling"] - } - }, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeDynDepDerivation(*store)) +TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store)) TEST_ATERM(DynDerivationTest, dynDerivationDeps, - R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", makeDynDepDerivation(*store), "dyn-dep-derivation") diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 496915745..7fdd3e11c 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -29,7 +29,7 @@ public: { if (testAccept()) { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + GTEST_SKIP() << cannotReadGoldenMaster; } else { @@ -70,7 +70,7 @@ public: { createDirs(dirOf(file)); writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; + GTEST_SKIP() << updatingGoldenMaster; } else { diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv new file mode 100644 index 000000000..3cd1ded02 --- /dev/null +++ b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/unit-test-data/libstore/derivation/bad-version.drv new file mode 100644 index 000000000..bbf75c114 --- /dev/null +++ b/unit-test-data/libstore/derivation/bad-version.drv @@ -0,0 +1 @@ +DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/unit-test-data/libstore/derivation/dynDerivationDeps.drv new file mode 100644 index 000000000..cfffe48ec --- /dev/null +++ b/unit-test-data/libstore/derivation/dynDerivationDeps.drv @@ -0,0 +1 @@ +DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/unit-test-data/libstore/derivation/dynDerivationDeps.json new file mode 100644 index 000000000..9dbeb1f15 --- /dev/null +++ b/unit-test-data/libstore/derivation/dynDerivationDeps.json @@ -0,0 +1,38 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "dyn-dep-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/unit-test-data/libstore/derivation/output-caFixedFlat.json new file mode 100644 index 000000000..fe000ea36 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedFlat.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/unit-test-data/libstore/derivation/output-caFixedNAR.json new file mode 100644 index 000000000..1afd60223 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedNAR.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "r:sha256", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/unit-test-data/libstore/derivation/output-caFixedText.json new file mode 100644 index 000000000..0b2cc8bbc --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedText.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "text:sha256", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/unit-test-data/libstore/derivation/output-caFloating.json new file mode 100644 index 000000000..9115de851 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFloating.json @@ -0,0 +1,3 @@ +{ + "hashAlgo": "r:sha256" +} diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/unit-test-data/libstore/derivation/output-deferred.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-deferred.json @@ -0,0 +1 @@ +{} diff --git a/unit-test-data/libstore/derivation/output-impure.json b/unit-test-data/libstore/derivation/output-impure.json new file mode 100644 index 000000000..62b61cdca --- /dev/null +++ b/unit-test-data/libstore/derivation/output-impure.json @@ -0,0 +1,4 @@ +{ + "hashAlgo": "r:sha256", + "impure": true +} diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/unit-test-data/libstore/derivation/output-inputAddressed.json new file mode 100644 index 000000000..86c7f3a05 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-inputAddressed.json @@ -0,0 +1,3 @@ +{ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/simple.drv b/unit-test-data/libstore/derivation/simple.drv new file mode 100644 index 000000000..bda74ad25 --- /dev/null +++ b/unit-test-data/libstore/derivation/simple.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/simple.json b/unit-test-data/libstore/derivation/simple.json new file mode 100644 index 000000000..20d0f8933 --- /dev/null +++ b/unit-test-data/libstore/derivation/simple.json @@ -0,0 +1,25 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "simple-derivation", + "outputs": {}, + "system": "wasm-sel4" +} From 325db01d269ca8580fc05ca4b56f28232266ecb7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 27 Oct 2023 07:30:16 +0200 Subject: [PATCH 242/402] fix anchor in conf-file I inadvertently switched it to `opt-` when refactoring, but it should have been `conf` to begin with. --- doc/manual/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 8bf16e9dd..db3daf252 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -103,7 +103,7 @@ $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "opt-"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix From 8381eeda6fa858b74bc7b516b9af9eecbbddd594 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Oct 2023 10:14:27 -0400 Subject: [PATCH 243/402] Systematize fetcher input attribute validation We now have `schemeName` and `allowedAttrs` functions for this purpose. We look up the schema with the former; we restrict the set of input attributes with the latter. --- src/libfetchers/fetchers.cc | 66 ++++++++++++++++++++++++++---------- src/libfetchers/fetchers.hh | 20 ++++++++++- src/libfetchers/git.cc | 30 ++++++++++++---- src/libfetchers/github.cc | 35 +++++++++++-------- src/libfetchers/indirect.cc | 23 +++++++++---- src/libfetchers/mercurial.cc | 23 +++++++++---- src/libfetchers/path.cc | 35 +++++++++++-------- src/libfetchers/tarball.cc | 34 +++++++++++-------- 8 files changed, 184 insertions(+), 82 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5688c4dc1..7a5c97399 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,12 +5,18 @@ namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +using InputSchemeMap = std::map>; + +std::unique_ptr inputSchemes = nullptr; void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); - inputSchemes->push_back(std::move(inputScheme)); + if (!inputSchemes) + inputSchemes = std::make_unique(); + auto schemeName = inputScheme->schemeName(); + if (inputSchemes->count(schemeName) > 0) + throw Error("Input scheme with name %s already registered", schemeName); + inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } Input Input::fromURL(const std::string & url, bool requireTree) @@ -33,7 +39,7 @@ static void fixupInput(Input & input) Input Input::fromURL(const ParsedURL & url, bool requireTree) { - for (auto & inputScheme : *inputSchemes) { + for (auto & [_, inputScheme] : *inputSchemes) { auto res = inputScheme->inputFromURL(url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); @@ -48,20 +54,44 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) Input Input::fromAttrs(Attrs && attrs) { - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); - if (res) { - experimentalFeatureSettings.require(inputScheme->experimentalFeature()); - res->scheme = inputScheme; - fixupInput(*res); - return std::move(*res); - } - } + auto schemeName = ({ + auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); + if (!schemeNameOpt) + throw Error("'type' attribute to specify input scheme is required but not provided"); + *std::move(schemeNameOpt); + }); - Input input; - input.attrs = attrs; - fixupInput(input); - return input; + auto raw = [&]() { + // Return an input without a scheme; most operations will fail, + // but not all of them. Doing this is to support those other + // operations which are supposed to be robust on + // unknown/uninterpretable inputs. + Input input; + input.attrs = attrs; + fixupInput(input); + return input; + }; + + std::shared_ptr inputScheme = ({ + auto i = inputSchemes->find(schemeName); + i == inputSchemes->end() ? nullptr : i->second; + }); + + if (!inputScheme) return raw(); + + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); + + auto allowedAttrs = inputScheme->allowedAttrs(); + + for (auto & [name, _] : attrs) + if (name != "type" && allowedAttrs.count(name) == 0) + throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); + + auto res = inputScheme->inputFromAttrs(attrs); + if (!res) return raw(); + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } ParsedURL Input::toURL() const @@ -307,7 +337,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } -std::optional InputScheme::experimentalFeature() +std::optional InputScheme::experimentalFeature() const { return {}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index ac605ff8e..b35d87eeb 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -126,6 +126,24 @@ struct InputScheme virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; + /** + * What is the name of the scheme? + * + * The `type` attribute is used to select which input scheme is + * used, and then the other fields are forwarded to that input + * scheme. + */ + virtual std::string_view schemeName() const = 0; + + /** + * Allowed attributes in an attribute set that is converted to an + * input. + * + * `type` is not included from this set, because the `type` field is + parsed first to choose which scheme; `type` is always required. + */ + virtual StringSet allowedAttrs() const = 0; + virtual ParsedURL toURL(const Input & input) const; virtual Input applyOverrides( @@ -144,7 +162,7 @@ struct InputScheme /** * Is this `InputScheme` part of an experimental feature? */ - virtual std::optional experimentalFeature(); + virtual std::optional experimentalFeature() const; virtual bool isDirect(const Input & input) const { return true; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 26b8987d6..bf25434c8 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -285,14 +285,32 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } + + std::string_view schemeName() const override + { + return "git"; + } + + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "shallow", + "submodules", + "lastModified", + "revCount", + "narHash", + "allRefs", + "name", + "dirtyRev", + "dirtyShortRev", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") - throw Error("unsupported Git input attribute '%s'", name); - maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "allRefs"); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 617fc7468..6c9b29721 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -27,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() const = 0; - virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { - if (url.scheme != type()) return {}; + if (url.scheme != schemeName()) return {}; auto path = tokenizeString>(url.path, "/"); @@ -91,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); Input input; - input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -101,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme return input; } + StringSet allowedAttrs() const override + { + return { + "owner", + "repo", + "ref", + "rev", + "narHash", + "lastModified", + "host", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != type()) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") - throw Error("unsupported input attribute '%s'", name); - getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); @@ -128,7 +133,7 @@ struct GitArchiveInputScheme : InputScheme if (ref) path += "/" + *ref; if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); return ParsedURL { - .scheme = type(), + .scheme = std::string { schemeName() }, .path = path, }; } @@ -220,7 +225,7 @@ struct GitArchiveInputScheme : InputScheme return {result.storePath, input}; } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } @@ -228,7 +233,7 @@ struct GitArchiveInputScheme : InputScheme struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string_view schemeName() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -309,7 +314,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } + std::string_view schemeName() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -377,7 +382,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() const override { return "sourcehut"; } + std::string_view schemeName() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 9a71df3d4..06f7f908d 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -49,14 +49,23 @@ struct IndirectInputScheme : InputScheme return input; } + std::string_view schemeName() const override + { + return "indirect"; + } + + StringSet allowedAttrs() const override + { + return { + "id", + "ref", + "rev", + "narHash", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") - throw Error("unsupported indirect input attribute '%s'", name); - auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); @@ -92,7 +101,7 @@ struct IndirectInputScheme : InputScheme throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index f830a3271..99002a94f 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -69,14 +69,25 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } + std::string_view schemeName() const override + { + return "hg"; + } + + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "revCount", + "narHash", + "name", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") - throw Error("unsupported Mercurial input attribute '%s'", name); - parseURL(getStrAttr(attrs, "url")); if (auto ref = maybeGetStrAttr(attrs, "ref")) { diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index d829609b5..699efbc3b 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -32,23 +32,30 @@ struct PathInputScheme : InputScheme return input; } + std::string_view schemeName() const override + { + return "path"; + } + + StringSet allowedAttrs() const override + { + return { + "path", + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree work + the same as the repository from which is exported (e.g. + path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). + */ + "rev", + "revCount", + "lastModified", + "narHash", + }; + } std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - getStrAttr(attrs, "path"); - for (auto & [name, value] : attrs) - /* Allow the user to pass in "fake" tree info - attributes. This is useful for making a pinned tree - work the same as the repository from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") - // checked in Input::fromAttrs - ; - else - throw Error("unsupported path input attribute '%s'", name); - Input input; input.attrs = attrs; return input; @@ -121,7 +128,7 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e1ea9b58b..0062878a9 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -184,7 +184,6 @@ DownloadTarballResult downloadTarball( // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { - virtual const std::string inputType() const = 0; const std::set transportUrlSchemes = {"file", "http", "https"}; const bool hasTarballExtension(std::string_view path) const @@ -222,22 +221,27 @@ struct CurlInputScheme : InputScheme url.query.erase("rev"); url.query.erase("revCount"); - input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("url", url.to_string()); return input; } + StringSet allowedAttrs() const override + { + return { + "type", + "url", + "narHash", + "name", + "unpack", + "rev", + "revCount", + "lastModified", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - auto type = maybeGetStrAttr(attrs, "type"); - if (type != inputType()) return {}; - - // FIXME: some of these only apply to TarballInputScheme. - std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"}; - for (auto & [name, value] : attrs) - if (!allowedNames.count(name)) - throw Error("unsupported %s input attribute '%s'", *type, name); - Input input; input.attrs = attrs; @@ -258,14 +262,14 @@ struct CurlInputScheme : InputScheme struct FileInputScheme : CurlInputScheme { - const std::string inputType() const override { return "file"; } + std::string_view schemeName() const override { return "file"; } bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() + ? parsedUrlScheme.application.value() == schemeName() : (!requireTree && !hasTarballExtension(url.path))); } @@ -278,7 +282,7 @@ struct FileInputScheme : CurlInputScheme struct TarballInputScheme : CurlInputScheme { - const std::string inputType() const override { return "tarball"; } + std::string_view schemeName() const override { return "tarball"; } bool isValidURL(const ParsedURL & url, bool requireTree) const override { @@ -286,7 +290,7 @@ struct TarballInputScheme : CurlInputScheme return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() + ? parsedUrlScheme.application.value() == schemeName() : (requireTree || hasTarballExtension(url.path))); } From 077de2968e8cf2d125818999adf8c149baf6384e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Oct 2023 10:30:59 -0400 Subject: [PATCH 244/402] Include fetcher input scheme info in the CLI dump Leverages the previous commit. --- src/libfetchers/fetchers.cc | 13 +++++++++++++ src/libfetchers/fetchers.hh | 3 +++ src/nix/main.cc | 1 + 3 files changed, 17 insertions(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7a5c97399..44b3fa4a5 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -19,6 +19,19 @@ void registerInputScheme(std::shared_ptr && inputScheme) inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } +nlohmann::json dumpRegisterInputSchemeInfo() { + using nlohmann::json; + + auto res = json::object(); + + for (auto & [name, scheme] : *inputSchemes) { + auto & r = res[name] = json::object(); + r["allowedAttrs"] = scheme->allowedAttrs(); + } + + return res; +} + Input Input::fromURL(const std::string & url, bool requireTree) { return fromURL(parseURL(url), requireTree); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b35d87eeb..3a02967f4 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -8,6 +8,7 @@ #include "url.hh" #include +#include namespace nix { class Store; } @@ -170,4 +171,6 @@ struct InputScheme void registerInputScheme(std::shared_ptr && fetcher); +nlohmann::json dumpRegisterInputSchemeInfo(); + } diff --git a/src/nix/main.cc b/src/nix/main.cc index ffba10099..d20bc1f8a 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -188,6 +188,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); + res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); return res.dump(); } From 05316d401fa509557c71140e17bb19814412fcb8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Oct 2023 17:03:06 +0100 Subject: [PATCH 245/402] Cleanup --- src/libexpr/flake/flake.cc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5c2a1623a..45c9ec8f3 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -641,29 +641,28 @@ LockedFlake lockFlake( auto newLockFileS = fmt("%s\n", newLockFile); - if (lockFlags.outputLockFilePath) + if (lockFlags.outputLockFilePath) { + if (lockFlags.commitLockFile) + throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible"); writeFile(*lockFlags.outputLockFilePath, newLockFileS); - else { + } else { auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; + auto outputLockFilePath = *sourcePath + "/" + relPath; - bool lockFileExists = pathExists(*outputLockFilePath); + bool lockFileExists = pathExists(outputLockFilePath); if (lockFileExists) { auto s = chomp(diff); if (s.empty()) - warn("updating lock file '%s'", *outputLockFilePath); + warn("updating lock file '%s'", outputLockFilePath); else - warn("updating lock file '%s':\n%s", *outputLockFilePath, s); + warn("updating lock file '%s':\n%s", outputLockFilePath, s); } else - warn("creating lock file '%s'", *outputLockFilePath); + warn("creating lock file '%s'", outputLockFilePath); std::optional commitMessage = std::nullopt; if (lockFlags.commitLockFile) { - if (lockFlags.outputLockFilePath) { - throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); - } std::string cm; cm = fetchSettings.commitLockFileSummary.get(); From 95f3f9eac978466c812814c06716f26e9f668e54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 22:21:34 +0000 Subject: [PATCH 246/402] build(deps): bump zeebe-io/backport-action from 1.4.0 to 2.0.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.4.0 to 2.0.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.4.0...v2.0.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 12c60c649..312c211dd 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.4.0 + uses: zeebe-io/backport-action@v2.0.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From 1fd0867389c2dd3e98d06decd4d35067885550a0 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Fri, 11 Aug 2023 21:47:16 +0200 Subject: [PATCH 247/402] Fix missing output when creating lockfile --- src/libexpr/flake/flake.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 45c9ec8f3..8cc803ccf 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -651,14 +651,14 @@ LockedFlake lockFlake( bool lockFileExists = pathExists(outputLockFilePath); + auto s = chomp(diff); if (lockFileExists) { - auto s = chomp(diff); if (s.empty()) warn("updating lock file '%s'", outputLockFilePath); else warn("updating lock file '%s':\n%s", outputLockFilePath, s); } else - warn("creating lock file '%s'", outputLockFilePath); + warn("creating lock file '%s': \n%s", outputLockFilePath, s); std::optional commitMessage = std::nullopt; From c762b65dc5314ed631381cf4bf26f5976e825bdc Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Fri, 11 Aug 2023 21:51:03 +0200 Subject: [PATCH 248/402] Fix documentation of flake command output --- src/nix/flake-lock.md | 7 +++++-- src/nix/flake-update.md | 11 ++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 2af0ad81e..100987a88 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -7,8 +7,11 @@ R""( ```console # nix flake lock --update-input nixpkgs --update-input nix - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + warning: creating lock file '/home/myself/repos/testflake/flake.lock': + • Added input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) ``` # Description diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 8c6042d94..b5a5ff0ec 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -6,9 +6,14 @@ R""( lock file: ```console - # nix flake update --commit-lock-file - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + # nix flake update + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11) + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) … warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' ``` From c7dcdb8325be7b8ecc3d480217808be899fc865a Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Sat, 12 Aug 2023 20:51:19 +0200 Subject: [PATCH 249/402] Overhaul nix flake update and lock commands Closes #5110 --- doc/manual/src/release-notes/rl-next.md | 14 +++++++ src/libcmd/command.hh | 6 +++ src/libcmd/installables.cc | 22 +--------- src/libexpr/flake/flake.cc | 11 +++-- src/nix/flake-lock.md | 45 ++++++++++----------- src/nix/flake-update.md | 53 +++++++++++++++++-------- src/nix/flake.cc | 33 +++++++++++++-- tests/functional/completions.sh | 7 ++-- tests/functional/flakes/circular.sh | 3 +- tests/functional/flakes/flakes.sh | 6 +-- tests/functional/flakes/follow-paths.sh | 4 +- 11 files changed, 124 insertions(+), 80 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 276252c37..3cfb53998 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -15,3 +15,17 @@ - `nix-shell` shebang lines now support single-quoted arguments. - `builtins.fetchTree` is now marked as stable. + + +- The interface for creating and updating lock files has been overhauled: + + - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. + It will *never* update existing inputs. + + - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. + - Passing no arguments will update all inputs of the current flake, just like it already did. + - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` + - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. + + - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. + They are superceded by `nix flake update`. diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index dafc0db3b..120c832ac 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -326,6 +326,12 @@ struct MixEnvironment : virtual Args { void setEnviron(); }; +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix); + void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eff18bbf6..3aff601e0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -28,7 +28,7 @@ namespace nix { -static void completeFlakeInputPath( +void completeFlakeInputPath( AddCompletions & completions, ref evalState, const std::vector & flakeRefs, @@ -46,13 +46,6 @@ MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; - addFlag({ - .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", - .category = category, - .handler = {&lockFlags.recreateLockFile, true} - }); - addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", @@ -85,19 +78,6 @@ MixFlakeOptions::MixFlakeOptions() .handler = {&lockFlags.commitLockFile, true} }); - addFlag({ - .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", - .category = category, - .labels = {"input-path"}, - .handler = {[&](std::string s) { - lockFlags.inputUpdates.insert(flake::parseInputPath(s)); - }}, - .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { - completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); - }} - }); - addFlag({ .longName = "override-input", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 8cc803ccf..70ae7b584 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -447,8 +447,8 @@ LockedFlake lockFlake( assert(input.ref); - /* Do we have an entry in the existing lock file? And we - don't have a --update-input flag for this input? */ + /* Do we have an entry in the existing lock file? + And the input is not in updateInputs? */ std::shared_ptr oldLock; updatesUsed.insert(inputPath); @@ -472,9 +472,8 @@ LockedFlake lockFlake( node->inputs.insert_or_assign(id, childNode); - /* If we have an --update-input flag for an input - of this input, then we must fetch the flake to - update it. */ + /* If we have this input in updateInputs, then we + must fetch the flake to update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto mustRefetch = @@ -616,7 +615,7 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputUpdates) if (!updatesUsed.count(i)) - warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + warn("'%s' does not match any input of this flake", printInputPath(i)); /* Check 'follows' inputs. */ newLockFile.check(); diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 100987a88..6d10258e3 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -2,11 +2,10 @@ R""( # Examples -* Update the `nixpkgs` and `nix` inputs of the flake in the current - directory: +* Create the lock file for the flake in the current directory: ```console - # nix flake lock --update-input nixpkgs --update-input nix + # nix flake lock warning: creating lock file '/home/myself/repos/testflake/flake.lock': • Added input 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) @@ -14,28 +13,28 @@ R""( 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) ``` +* Add missing inputs to the lock file for a flake in a different directory: + + ```console + # nix flake lock ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + # Description -This command updates the lock file of a flake (`flake.lock`) so that -it contains a lock for every flake input specified in -`flake.nix`. Existing lock file entries are not updated unless -required by a flag such as `--update-input`. +This command adds inputs to the lock file of a flake (`flake.lock`) +so that it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated. -Note that every command that operates on a flake will also update the -lock file if needed, and supports the same flags. Therefore, - -```console -# nix flake lock --update-input nixpkgs -# nix build -``` - -is equivalent to: - -```console -# nix build --update-input nixpkgs -``` - -Thus, this command is only useful if you want to update the lock file -separately from any other action such as building. +If you want to update existing lock entries, use +[`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) )"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index b5a5ff0ec..63df3b12a 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -2,8 +2,7 @@ R""( # Examples -* Recreate the lock file (i.e. update all inputs) and commit the new - lock file: +* Update all inputs (i.e. recreate the lock file from scratch): ```console # nix flake update @@ -14,26 +13,46 @@ R""( • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) - … - warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' ``` +* Update only a single input: + + ```console + # nix flake update nixpkgs + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + +* Update only a single input of a flake in a different directory: + + ```console + # nix flake update nixpkgs --flake ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + # Description -This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every unlocked input (like `nixpkgs`) to its -current version. This is equivalent to passing `--recreate-lock-file` -to any command that operates on a flake. That is, +This command updates the inputs in a lock file (`flake.lock`). +**By default, all inputs are updated**. If the lock file doesn't exist +yet, it will be created. If inputs are not in the lock file yet, they will be added. -```console -# nix flake update -# nix build -``` +Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs +to update as its positional arguments and operates on the flake in the current directory. +You can pass a different flake-url with `--flake` to override that default. -is equivalent to: - -```console -# nix build --recreate-lock-file -``` +The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) +also creates lock files and adds missing inputs, but is safer as it +will never update inputs already in the lock file. )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 0116eff2e..e8906a252 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -24,8 +24,10 @@ using namespace nix; using namespace nix::flake; using json = nlohmann::json; +struct CmdFlakeUpdate; class FlakeCommand : virtual Args, public MixFlakeOptions { +protected: std::string flakeUrl = "."; public: @@ -63,6 +65,8 @@ public: struct CmdFlakeUpdate : FlakeCommand { +public: + std::string description() override { return "update flake lock file"; @@ -70,9 +74,31 @@ struct CmdFlakeUpdate : FlakeCommand CmdFlakeUpdate() { + expectedArgs.clear(); + addFlag({ + .longName="flake", + .description="The flake to operate on. Default is the current directory.", + .labels={"flake-url"}, + .handler={&flakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }} + }); + expectArgs({ + .label="inputs", + .optional=true, + .handler={[&](std::string inputToUpdate){ + auto inputPath = flake::parseInputPath(inputToUpdate); + if (lockFlags.inputUpdates.contains(inputPath)) + warn("Input '%s' was specified multiple times. You may have done this by accident."); + lockFlags.inputUpdates.insert(inputPath); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + /* Remove flags that don't make sense. */ - removeFlag("recreate-lock-file"); - removeFlag("update-input"); removeFlag("no-update-lock-file"); removeFlag("no-write-lock-file"); } @@ -87,8 +113,9 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { settings.tarballTtl = 0; + auto updateAll = lockFlags.inputUpdates.empty(); - lockFlags.recreateLockFile = true; + lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh index 7c1e4b287..b9886623a 100644 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -48,11 +48,10 @@ EOF [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] -[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]] -[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]] ## Out of order -[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] -[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] +[[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh index 09cd02edf..d3bb8e8a3 100644 --- a/tests/functional/flakes/circular.sh +++ b/tests/functional/flakes/circular.sh @@ -42,7 +42,8 @@ git -C $flakeB commit -a -m 'Foo' sed -i $flakeB/flake.nix -e 's/456/789/' git -C $flakeB commit -a -m 'Foo' -[[ $(nix eval --update-input b $flakeA#foo) = 1912 ]] +nix flake update b --flake $flakeA +[[ $(nix eval $flakeA#foo) = 1912 ]] # Test list-inputs with circular dependencies nix flake metadata $flakeA diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 70de28628..b0038935c 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -300,7 +300,7 @@ nix build -o "$TEST_ROOT/result" flake4#xyzzy nix flake lock "$flake3Dir" [[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] -nix flake update "$flake3Dir" --override-flake flake2 nixpkgs +nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore @@ -437,7 +437,7 @@ cat > "$flake3Dir/flake.nix" < Date: Tue, 24 Oct 2023 22:22:05 +0200 Subject: [PATCH 250/402] fix: segfault in positional arg completion Adding the inputPath as a positional feature uncovered this bug. As positional argument forms were discarded from the `expectedArgs` list, their closures were not. When the `.completer` closure was then called, part of the surrounding object did not exist anymore. This didn't cause an issue before, but with the new call to `getEvalState()` in the "inputs" completer in nix/flake.cc, a segfault was triggered reproducibly on invalid memory access to the `this` pointer, which was always 0. The solution of splicing the argument forms into a new list to extend their lifetime is a bit of a hack, but I was unable to get the "nicer" iterator-based solution to work. --- src/libutil/args.cc | 13 ++++++++++++- src/libutil/args.hh | 14 +++++++++++++- tests/functional/completions.sh | 4 ++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 6bc3cae07..811353c18 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -255,7 +255,18 @@ bool Args::processArgs(const Strings & args, bool finish) } if (!anyCompleted) exp.handler.fun(ss); - expectedArgs.pop_front(); + + /* Move the list element to the processedArgs. This is almost the same as + `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`, + except that it will only adjust the next and prev pointers of the list + elements, meaning the actual contents don't move in memory. This is + critical to prevent invalidating internal pointers! */ + processedArgs.splice( + processedArgs.end(), + expectedArgs, + expectedArgs.begin(), + ++expectedArgs.begin()); + res = true; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ff2bf3cab..e3b41313f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -200,13 +200,25 @@ protected: /** * Queue of expected positional argument forms. * - * Positional arugment descriptions are inserted on the back. + * Positional argument descriptions are inserted on the back. * * As positional arguments are passed, these are popped from the * front, until there are hopefully none left as all args that were * expected in fact were passed. */ std::list expectedArgs; + /** + * List of processed positional argument forms. + * + * All items removed from `expectedArgs` are added here. After all + * arguments were processed, this list should be exactly the same as + * `expectedArgs` was before. + * + * This list is used to extend the lifetime of the argument forms. + * If this is not done, some closures that reference the command + * itself will segfault. + */ + std::list processedArgs; /** * Process some positional arugments diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh index b9886623a..d3d5bbd48 100644 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -44,6 +44,10 @@ EOF # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +cd ./foo +[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]] +cd .. +[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]] ## With multiple input flakes [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion From 1f4525531e9b5e744830a55a2595880b135d93c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 31 Oct 2023 12:01:13 -0400 Subject: [PATCH 251/402] Add configure test to ensure GCC bug is fixed https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 (test is adapted from issue, test does not test for GCC-specific behavior but rather absence of bug, so test is good with other compilers too.) --- configure.ac | 3 +++ m4/gcc_bug_80431.m4 | 64 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 m4/gcc_bug_80431.m4 diff --git a/configure.ac b/configure.ac index 225baf6b5..75ce7d01d 100644 --- a/configure.ac +++ b/configure.ac @@ -68,6 +68,9 @@ case "$host_os" in esac +ENSURE_NO_GCC_BUG_80431 + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 new file mode 100644 index 000000000..e42f01956 --- /dev/null +++ b/m4/gcc_bug_80431.m4 @@ -0,0 +1,64 @@ +# Ensure that this bug is not present in the C++ toolchain we are using. +# +# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 +# +# The test program is from that issue, with only a slight modification +# to set an exit status instead of printing strings. +AC_DEFUN([ENSURE_NO_GCC_BUG_80431], +[ + AC_MSG_CHECKING([that GCC bug 80431 is fixed]) + AC_LANG_PUSH(C++) + AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [[ + #include + + static bool a = true; + static bool b = true; + + struct Options { }; + + struct Option + { + Option(Options * options) + { + a = false; + } + + ~Option() + { + b = false; + } + }; + + struct MyOptions : Options { }; + + struct MyOptions2 : virtual MyOptions + { + Option foo{this}; + }; + ]], + [[ + { + MyOptions2 opts; + } + return (a << 1) | b; + ]])], + [status_80431=0], + [status_80431=$?], + [ + # Assume we're bug-free when cross-compiling + ]) + AC_LANG_POP(C++) + AS_CASE([$status_80431], + [0],[ + AC_MSG_RESULT(yes) + ], + [2],[ + AC_MSG_RESULT(no) + AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) + ], + [ + AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) + ]) +]) From b2cae33aef63644bf6e09dea253ed6e1af847fb8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Oct 2023 18:12:37 -0400 Subject: [PATCH 252/402] Remove bug-avoiding `StoreConfig *` casts for settings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 has been fixed, and per the previous commit we now check that is the case at build time. --- src/libstore/binary-cache-store.hh | 14 +++++++------- src/libstore/legacy-ssh-store.cc | 6 +++--- src/libstore/local-fs-store.hh | 12 ++++-------- src/libstore/local-store.hh | 4 ++-- src/libstore/remote-store.hh | 4 ++-- src/libstore/s3-binary-cache-store.cc | 18 +++++++++--------- src/libstore/ssh-store-config.hh | 8 ++++---- src/libstore/ssh-store.cc | 2 +- 8 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 49f271d24..218a888e3 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", + const Setting compression{this, "xz", "compression", "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + const Setting writeNARListing{this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + const Setting writeDebugInfo{this, false, "index-debug-info", R"( Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to fetch debug info on demand )"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", + const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", + const Setting localNarCache{this, "", "local-nar-cache", "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; - const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", + const Setting parallelCompression{this, false, "parallel-compression", "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", + const Setting compressionLevel{this, -1, "compression-level", R"( The *preset level* to be used when compressing NARs. The meaning and accepted values depend on the compression method selected. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c712f7eb1..38fdf118f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -17,10 +17,10 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + const Setting remoteProgram{this, "nix-store", "remote-program", "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; const std::string name() override { return "SSH Store"; } @@ -38,7 +38,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in // the documentation - const Setting logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; struct Connection { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 488109501..d6bda05d1 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,25 +11,21 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes - // it to omit the call to the Setting constructor. Clang works fine - // either way. - - const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt, + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{(StoreConfig*) this, + const PathSetting stateDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{(StoreConfig*) this, + const PathSetting logDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{(StoreConfig*) this, + const PathSetting realStoreDir{this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e97195f5b..fe26a0f27 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -40,12 +40,12 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, + Setting requireSigs{this, settings.requireSigs, "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; - Setting readOnly{(StoreConfig*) this, + Setting readOnly{this, false, "read-only", R"( diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a1ae82a0f..f0985fdc1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,10 +22,10 @@ struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, + const Setting maxConnectionAge{this, std::numeric_limits::max(), "max-connection-age", "Maximum age of a connection before it is closed."}; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d2fc6abaf..1a62d92d4 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -193,20 +193,20 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", + const Setting profile{this, "", "profile", R"( The name of the AWS configuration profile to use. By default Nix will use the `default` profile. )"}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + const Setting region{this, Aws::Region::US_EAST_1, "region", R"( The region of the S3 bucket. If your bucket is not in `us–east-1`, you should always explicitly specify the region parameter. )"}; - const Setting scheme{(StoreConfig*) this, "", "scheme", + const Setting scheme{this, "", "scheme", R"( The scheme used for S3 requests, `https` (default) or `http`. This option allows you to disable HTTPS for binary caches which don't @@ -218,7 +218,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > information. )"}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", + const Setting endpoint{this, "", "endpoint", R"( The URL of the endpoint of an S3-compatible service such as MinIO. Do not specify this setting if you're using Amazon S3. @@ -229,13 +229,13 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > addressing instead of virtual host based addressing. )"}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + const Setting narinfoCompression{this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", + const Setting logCompression{this, "", "log-compression", R"( Compression method for `log/*` files. It is recommended to use a compression method supported by most web browsers @@ -243,11 +243,11 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig )"}; const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", + this, false, "multipart-upload", "Whether to use multi-part uploads."}; const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index c27a5d00f..bf55d20cf 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -9,16 +9,16 @@ struct CommonSSHStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", + const Setting sshKey{this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; - const Setting compress{(StoreConfig*) this, false, "compress", + const Setting compress{this, false, "compress", "Whether to enable SSH compression."}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", + const Setting remoteStore{this, "", "remote-store", R"( [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used on the remote machine. The default is `auto` diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 9c6c42ef4..4a6aad449 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -16,7 +16,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig using RemoteStoreConfig::RemoteStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + const Setting remoteProgram{this, "nix-daemon", "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; const std::string name() override { return "Experimental SSH Store"; } From 1093d6585ff6478e50a5845de64cfcf114e35a95 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 31 Oct 2023 20:39:39 -0400 Subject: [PATCH 253/402] Make `ParseSink` a bit better I wouldn't call it *good* yet, but this will do for now. - `RetrieveRegularNARSink` renamed to `RegularFileSink` and moved accordingly because it actually has nothing to do with NARs in particular. - its `fd` field is also marked private - `copyRecursive` introduced to dump a `SourceAccessor` into a `ParseSink`. - `NullParseSink` made so `ParseSink` no longer has sketchy default methods. This was done while updating #8918 to work with the new `SourceAccessor`. --- src/libstore/daemon.cc | 6 +-- src/libstore/export-import.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 8 ++-- src/libutil/archive.cc | 8 +--- src/libutil/archive.hh | 27 ----------- src/libutil/fs-sink.cc | 48 +++++++++++++++++++ src/libutil/fs-sink.hh | 86 ++++++++++++++++++++++++++++++----- 8 files changed, 132 insertions(+), 55 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 007ffc05a..105d92f25 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref store, eagerly consume the entire stream it's given, past the length of the Nar. */ TeeSource savedNARSource(from, saved); - ParseSink sink; /* null sink; just parse the NAR */ + NullParseSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); } else { /* Incrementally parse the NAR file, stripping the metadata, and streaming the sole file we expect into `saved`. */ - RetrieveRegularNARSink savedRegular { saved }; + RegularFileSink savedRegular { saved }; parseDump(savedRegular, from); if (!savedRegular.regular) throw Error("regular file expected"); } @@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref store, source = std::make_unique(from, to); else { TeeSource tee { from, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 91b7e30db..52130f8f6 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) /* Extract the NAR from the source. */ StringSink saved; TeeSource tee { source, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); uint32_t magic = readInt(source); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1c2f6023a..a5e9426f8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1200,7 +1200,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, bool narRead = false; Finally cleanup = [&]() { if (!narRead) { - ParseSink sink; + NullParseSink sink; parseDump(sink, source); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0399120d1..e6a4cf9d9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -410,7 +410,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, /* Note that fileSink and unusualHashTee must be mutually exclusive, since they both write to caHashSink. Note that that requisite is currently true because the former is only used in the flat case. */ - RetrieveRegularNARSink fileSink { caHashSink }; + RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 @@ -428,10 +428,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, information to narSink. */ TeeSource tapped { *fileSource, narSink }; - ParseSink blank; + NullParseSink blank; auto & parseSink = method == FileIngestionMethod::Flat - ? fileSink - : blank; + ? (ParseSink &) fileSink + : (ParseSink &) blank; /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3b1a1e0ef..4ca84d357 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -5,12 +5,6 @@ #include // for strcasecmp -#include -#include -#include -#include -#include - #include "archive.hh" #include "util.hh" #include "config.hh" @@ -299,7 +293,7 @@ void copyNAR(Source & source, Sink & sink) // FIXME: if 'source' is the output of dumpPath() followed by EOF, // we should just forward all data directly without parsing. - ParseSink parseSink; /* null sink; just parse the NAR */ + NullParseSink parseSink; /* just parse the NAR */ TeeSource wrapper { source, sink }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 3530783c1..2cf8ee891 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * If the NAR archive contains a single file at top-level, then save - * the contents of the file to `s`. Otherwise barf. - */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular = true; - Sink & sink; - - RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - - void createDirectory(const Path & path) override - { - regular = false; - } - - void receiveContents(std::string_view data) override - { - sink(data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - regular = false; - } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index a08a723a4..925e6f05d 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -5,6 +5,54 @@ namespace nix { +void copyRecursive( + SourceAccessor & accessor, const CanonPath & from, + ParseSink & sink, const Path & to) +{ + auto stat = accessor.lstat(from); + + switch (stat.type) { + case SourceAccessor::tSymlink: + { + sink.createSymlink(to, accessor.readLink(from)); + } + + case SourceAccessor::tRegular: + { + sink.createRegularFile(to); + if (stat.isExecutable) + sink.isExecutable(); + LambdaSink sink2 { + [&](auto d) { + sink.receiveContents(d); + } + }; + accessor.readFile(from, sink2, [&](uint64_t size) { + sink.preallocateContents(size); + }); + break; + } + + case SourceAccessor::tDirectory: + { + sink.createDirectory(to); + for (auto & [name, _] : accessor.readDirectory(from)) { + copyRecursive( + accessor, from + name, + sink, to + "/" + name); + break; + } + } + + case SourceAccessor::tMisc: + throw Error("file '%1%' has an unsupported type", from); + + default: + abort(); + } +} + + struct RestoreSinkSettings : Config { Setting preallocateContents{this, false, "preallocate-contents", diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 6837e2fc4..c22edd390 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "source-accessor.hh" namespace nix { @@ -11,32 +12,93 @@ namespace nix { */ struct ParseSink { - virtual void createDirectory(const Path & path) { }; + virtual void createDirectory(const Path & path) = 0; - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; + virtual void createRegularFile(const Path & path) = 0; + virtual void receiveContents(std::string_view data) = 0; + virtual void isExecutable() = 0; + virtual void closeRegularFile() = 0; + + virtual void createSymlink(const Path & path, const std::string & target) = 0; + + /** + * An optimization. By default, do nothing. + */ virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; }; +/** + * Recusively copy file system objects from the source into the sink. + */ +void copyRecursive( + SourceAccessor & accessor, const CanonPath & sourcePath, + ParseSink & sink, const Path & destPath); + +/** + * Ignore everything and do nothing + */ +struct NullParseSink : ParseSink +{ + void createDirectory(const Path & path) override { } + void receiveContents(std::string_view data) override { } + void createSymlink(const Path & path, const std::string & target) override { } + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } +}; + +/** + * Write files at the given path + */ struct RestoreSink : ParseSink { Path dstPath; - AutoCloseFD fd; - void createDirectory(const Path & path) override; void createRegularFile(const Path & path) override; - void closeRegularFile() override; - void isExecutable() override; - void preallocateContents(uint64_t size) override; void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + AutoCloseFD fd; +}; + +/** + * Restore a single file at the top level, passing along + * `receiveContents` to the underlying `Sink`. For anything but a single + * file, set `regular = true` so the caller can fail accordingly. + */ +struct RegularFileSink : ParseSink +{ + bool regular = true; + Sink & sink; + + RegularFileSink(Sink & sink) : sink(sink) { } + + void createDirectory(const Path & path) override + { + regular = false; + } + + void receiveContents(std::string_view data) override + { + sink(data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + regular = false; + } + + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } }; } From bc4a1695ac71483831ac9ad591c872105794e88f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 11:44:16 +0100 Subject: [PATCH 254/402] doc/hacking: Fix clangd for tests --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 38c144fcc..fe08ceb94 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -210,7 +210,7 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES install +make clean && bear -- make -j$NIX_BUILD_CORES default check install ``` Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). From b2ac6fc040223a58f9b923a89798f72b48e310e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 14:36:40 +0100 Subject: [PATCH 255/402] Remove FSAccessor::Type::tMissing Instead stat() now returns std::nullopt to denote that the file doesn't exist. --- src/libstore/binary-cache-store.cc | 6 +-- src/libstore/fs-accessor.hh | 19 +++++---- src/libstore/local-fs-store.cc | 8 ++-- src/libstore/nar-accessor.cc | 68 +++++++++++++++--------------- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-fs-accessor.hh | 2 +- src/nix/cat.cc | 11 +++-- src/nix/ls.cc | 26 ++++++------ src/nix/run.cc | 2 +- src/nix/why-depends.cc | 7 +-- 10 files changed, 77 insertions(+), 74 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2a91233ec..06d89c478 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string buildIdDir = "/lib/debug/.build-id"; - if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + if (auto st = narAccessor->stat(buildIdDir); st && st->type == FSAccessor::tDirectory) { ThreadPool threadPool(25); @@ -234,14 +234,14 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; - if (narAccessor->stat(dir).type != FSAccessor::tDirectory + if (auto st = narAccessor->stat(dir); !st || st->type != FSAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & s2 : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; - if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + if (auto st = narAccessor->stat(debugPath); !st || st->type != FSAccessor::tRegular || !std::regex_match(s2, regex2)) continue; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 1df19e647..9bae0be74 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -3,6 +3,8 @@ #include "types.hh" +#include + namespace nix { /** @@ -12,28 +14,29 @@ namespace nix { class FSAccessor { public: - enum Type { tMissing, tRegular, tSymlink, tDirectory }; + enum Type { tRegular, tSymlink, tDirectory }; struct Stat { - Type type = tMissing; + Type type; /** - * regular files only + * For regular files only: the size of the file. */ uint64_t fileSize = 0; /** - * regular files only + * For regular files only: whether this is an executable. */ - bool isExecutable = false; // regular files only + bool isExecutable = false; /** - * regular files only + * For regular files only: the position of the contents of this + * file in the NAR. */ - uint64_t narOffset = 0; // regular files only + uint64_t narOffset = 0; }; virtual ~FSAccessor() { } - virtual Stat stat(const Path & path) = 0; + virtual std::optional stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e9..bb83a9cd4 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -27,25 +27,25 @@ struct LocalStoreAccessor : public FSAccessor return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } - FSAccessor::Stat stat(const Path & path) override + std::optional stat(const Path & path) override { auto realPath = toRealPath(path); struct stat st; if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; + if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; throw SysError("getting status of '%1%'", path); } if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) throw Error("file '%1%' has unsupported type", path); - return { + return {{ S_ISREG(st.st_mode) ? Type::tRegular : S_ISLNK(st.st_mode) ? Type::tSymlink : Type::tDirectory, S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } StringSet readDirectory(const Path & path) override diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f0dfcb19b..9123bd59d 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,13 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Type type = FSAccessor::Type::tMissing; - - bool isExecutable = false; - - /* If this is a regular file, position of the contents of this - file in the NAR. */ - uint64_t start = 0, size = 0; + FSAccessor::Stat stat; std::string target; @@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != FSAccessor::Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -79,14 +73,15 @@ struct NarAccessor : public FSAccessor void isExecutable() override { - parents.top()->isExecutable = true; + parents.top()->stat.isExecutable = true; } void preallocateContents(uint64_t size) override { assert(size <= std::numeric_limits::max()); - parents.top()->size = (uint64_t) size; - parents.top()->start = pos; + auto & st = parents.top()->stat; + st.fileSize = (uint64_t) size; + st.narOffset = pos; } void receiveContents(std::string_view data) override @@ -95,7 +90,9 @@ struct NarAccessor : public FSAccessor void createSymlink(const Path & path, const std::string & target) override { createMember(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + NarMember{ + .stat = {.type = FSAccessor::Type::tSymlink}, + .target = target}); } size_t read(char * data, size_t len) override @@ -130,18 +127,20 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.type = FSAccessor::Type::tDirectory; + member.stat = {.type = FSAccessor::Type::tDirectory}; for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { std::string name = i.key(); recurse(member.children[name], i.value()); } } else if (type == "regular") { - member.type = FSAccessor::Type::tRegular; - member.size = v["size"]; - member.isExecutable = v.value("executable", false); - member.start = v["narOffset"]; + member.stat = { + .type = FSAccessor::Type::tRegular, + .fileSize = v["size"], + .isExecutable = v.value("executable", false), + .narOffset = v["narOffset"] + }; } else if (type == "symlink") { - member.type = FSAccessor::Type::tSymlink; + member.stat = {.type = FSAccessor::Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -158,7 +157,7 @@ struct NarAccessor : public FSAccessor for (auto it = path.begin(); it != end; ) { // because it != end, the remaining component is non-empty so we need // a directory - if (current->type != FSAccessor::Type::tDirectory) return nullptr; + if (current->stat.type != FSAccessor::Type::tDirectory) return nullptr; // skip slash (canonPath above ensures that this is always a slash) assert(*it == '/'); @@ -183,19 +182,19 @@ struct NarAccessor : public FSAccessor return *result; } - Stat stat(const Path & path) override + std::optional stat(const Path & path) override { auto i = find(path); if (i == nullptr) - return {FSAccessor::Type::tMissing, 0, false}; - return {i->type, i->size, i->isExecutable, i->start}; + return std::nullopt; + return i->stat; } StringSet readDirectory(const Path & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tDirectory) + if (i.stat.type != FSAccessor::Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; @@ -208,19 +207,19 @@ struct NarAccessor : public FSAccessor std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); - if (i.type != FSAccessor::Type::tRegular) + if (i.stat.type != FSAccessor::Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.start, i.size); + if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); assert(nar); - return std::string(*nar, i.start, i.size); + return std::string(*nar, i.stat.narOffset, i.stat.fileSize); } std::string readLink(const Path & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tSymlink) + if (i.stat.type != FSAccessor::Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } @@ -246,17 +245,19 @@ using nlohmann::json; json listNar(ref accessor, const Path & path, bool recurse) { auto st = accessor->stat(path); + if (!st) + throw Error("path '%s' does not exist in NAR", path); json obj = json::object(); - switch (st.type) { + switch (st->type) { case FSAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st.fileSize; - if (st.isExecutable) + obj["size"] = st->fileSize; + if (st->isExecutable) obj["executable"] = true; - if (st.narOffset) - obj["narOffset"] = st.narOffset; + if (st->narOffset) + obj["narOffset"] = st->narOffset; break; case FSAccessor::Type::tDirectory: obj["type"] = "directory"; @@ -275,9 +276,6 @@ json listNar(ref accessor, const Path & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - case FSAccessor::Type::tMissing: - default: - throw Error("path '%s' does not exist in NAR", path); } return obj; } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fcfb527f5..6c87ebeaa 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -101,7 +101,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::stat(const Path & path) { auto res = fetch(path); return res.first->stat(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index e2673b6f6..5cf759aa0 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -28,7 +28,7 @@ public: RemoteFSAccessor(ref store, const /* FIXME: use std::optional */ Path & cacheDir = ""); - Stat stat(const Path & path) override; + std::optional stat(const Path & path) override; StringSet readDirectory(const Path & path) override; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 60aa66ce0..b5fe2506f 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -11,13 +11,12 @@ struct MixCat : virtual Args void cat(ref accessor) { - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) + if (auto st = accessor->stat(path)) { + if (st->type != FSAccessor::Type::tRegular) + throw Error("path '%1%' is not a regular file", path); + writeFull(STDOUT_FILENO, accessor->readFile(path)); + } else throw Error("path '%1%' does not exist", path); - if (st.type != FSAccessor::Type::tRegular) - throw Error("path '%1%' is not a regular file", path); - - writeFull(STDOUT_FILENO, accessor->readFile(path)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index c990a303c..8dc8a47b4 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -46,23 +46,25 @@ struct MixLs : virtual Args, MixJSON auto showFile = [&](const Path & curPath, const std::string & relPath) { if (verbose) { auto st = accessor->stat(curPath); + assert(st); std::string tp = - st.type == FSAccessor::Type::tRegular ? - (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st->type == FSAccessor::Type::tRegular ? + (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); - if (st.type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st->fileSize, relPath); + if (st->type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st.type == FSAccessor::Type::tDirectory) - doPath(st, curPath, relPath, false); + if (recursive && st->type == FSAccessor::Type::tDirectory) + doPath(*st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { auto st = accessor->stat(curPath); - if (st.type == FSAccessor::Type::tDirectory) - doPath(st, curPath, relPath, false); + assert(st); + if (st->type == FSAccessor::Type::tDirectory) + doPath(*st, curPath, relPath, false); } } }; @@ -79,10 +81,10 @@ struct MixLs : virtual Args, MixJSON }; auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) + if (!st) throw Error("path '%1%' does not exist", path); - doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + doPath(*st, path, + st->type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), showDirectory); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 1baf299ab..f6c229adc 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -120,7 +120,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment unixPath.push_front(store->printStorePath(path) + "/bin"); auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (accessor->stat(propPath).type == FSAccessor::tRegular) { + if (auto st = accessor->stat(propPath); st && st->type == FSAccessor::tRegular) { for (auto & p : tokenizeString(readFile(propPath))) todo.push(store->parseStorePath(p)); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 055cf6d0d..912ba72fb 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -214,6 +214,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions visitPath = [&](const Path & p) { auto st = accessor->stat(p); + assert(st); auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); @@ -221,13 +222,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st.type == FSAccessor::Type::tDirectory) { + if (st->type == FSAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & name : names) visitPath(p + "/" + name); } - else if (st.type == FSAccessor::Type::tRegular) { + else if (st->type == FSAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -245,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st.type == FSAccessor::Type::tSymlink) { + else if (st->type == FSAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { From 8ffd1695ce31ff81b038fdc995dd8da03b180f03 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 14:43:20 +0100 Subject: [PATCH 256/402] Unify FSAccessor::Type and SourceAccessor::Type --- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/fs-accessor.hh | 3 ++- src/libstore/nar-accessor.cc | 30 ++++++++++++++++-------------- src/nix/run.cc | 2 +- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 06d89c478..f9abd8cbd 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string buildIdDir = "/lib/debug/.build-id"; - if (auto st = narAccessor->stat(buildIdDir); st && st->type == FSAccessor::tDirectory) { + if (auto st = narAccessor->stat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -234,14 +234,14 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; - if (auto st = narAccessor->stat(dir); !st || st->type != FSAccessor::tDirectory + if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & s2 : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; - if (auto st = narAccessor->stat(debugPath); !st || st->type != FSAccessor::tRegular + if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 9bae0be74..1e951ec57 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -2,6 +2,7 @@ ///@file #include "types.hh" +#include "source-accessor.hh" #include @@ -14,7 +15,7 @@ namespace nix { class FSAccessor { public: - enum Type { tRegular, tSymlink, tDirectory }; + using Type = SourceAccessor::Type; struct Stat { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9123bd59d..43a78a362 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -51,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->stat.type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -60,12 +60,12 @@ struct NarAccessor : public FSAccessor void createDirectory(const Path & path) override { - createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, {Type::tDirectory, false, 0, 0}); } void createRegularFile(const Path & path) override { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + createMember(path, {Type::tRegular, false, 0, 0}); } void closeRegularFile() override @@ -91,7 +91,7 @@ struct NarAccessor : public FSAccessor { createMember(path, NarMember{ - .stat = {.type = FSAccessor::Type::tSymlink}, + .stat = {.type = Type::tSymlink}, .target = target}); } @@ -127,20 +127,20 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.stat = {.type = FSAccessor::Type::tDirectory}; + member.stat = {.type = Type::tDirectory}; for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { std::string name = i.key(); recurse(member.children[name], i.value()); } } else if (type == "regular") { member.stat = { - .type = FSAccessor::Type::tRegular, + .type = Type::tRegular, .fileSize = v["size"], .isExecutable = v.value("executable", false), .narOffset = v["narOffset"] }; } else if (type == "symlink") { - member.stat = {.type = FSAccessor::Type::tSymlink}; + member.stat = {.type = Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -157,7 +157,7 @@ struct NarAccessor : public FSAccessor for (auto it = path.begin(); it != end; ) { // because it != end, the remaining component is non-empty so we need // a directory - if (current->stat.type != FSAccessor::Type::tDirectory) return nullptr; + if (current->stat.type != Type::tDirectory) return nullptr; // skip slash (canonPath above ensures that this is always a slash) assert(*it == '/'); @@ -194,7 +194,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tDirectory) + if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; @@ -207,7 +207,7 @@ struct NarAccessor : public FSAccessor std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tRegular) + if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); @@ -219,7 +219,7 @@ struct NarAccessor : public FSAccessor std::string readLink(const Path & path) override { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tSymlink) + if (i.stat.type != Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } @@ -251,7 +251,7 @@ json listNar(ref accessor, const Path & path, bool recurse) json obj = json::object(); switch (st->type) { - case FSAccessor::Type::tRegular: + case SourceAccessor::Type::tRegular: obj["type"] = "regular"; obj["size"] = st->fileSize; if (st->isExecutable) @@ -259,7 +259,7 @@ json listNar(ref accessor, const Path & path, bool recurse) if (st->narOffset) obj["narOffset"] = st->narOffset; break; - case FSAccessor::Type::tDirectory: + case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; { obj["entries"] = json::object(); @@ -272,10 +272,12 @@ json listNar(ref accessor, const Path & path, bool recurse) } } break; - case FSAccessor::Type::tSymlink: + case SourceAccessor::Type::tSymlink: obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; + case SourceAccessor::Type::tMisc: + assert(false); // cannot happen for NARs } return obj; } diff --git a/src/nix/run.cc b/src/nix/run.cc index f6c229adc..07806283c 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -120,7 +120,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment unixPath.push_front(store->printStorePath(path) + "/bin"); auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (auto st = accessor->stat(propPath); st && st->type == FSAccessor::tRegular) { + if (auto st = accessor->stat(propPath); st && st->type == SourceAccessor::tRegular) { for (auto & p : tokenizeString(readFile(propPath))) todo.push(store->parseStorePath(p)); } From cdb27c1519cd802f477e8fa90beabe1bddc4bac7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:26:07 +0100 Subject: [PATCH 257/402] SourceAccessor: Change the main interface from lstat() to maybeLstat() --- src/libexpr/primops.cc | 6 ++---- src/libfetchers/fs-input-accessor.cc | 4 ++-- src/libfetchers/memory-input-accessor.cc | 4 ++-- src/libutil/posix-source-accessor.cc | 8 ++++++-- src/libutil/posix-source-accessor.hh | 2 +- src/libutil/source-accessor.cc | 10 +++++----- src/libutil/source-accessor.hh | 4 ++-- .../lang/eval-fail-bad-string-interpolation-2.err.exp | 2 +- tests/functional/lang/eval-fail-nonexist-path.err.exp | 2 +- 9 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 704e7007b..e3c775d90 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1548,10 +1548,8 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, try { auto checked = state.checkSourcePath(path); - auto exists = checked.pathExists(); - if (exists && mustBeDir) { - exists = checked.lstat().type == InputAccessor::tDirectory; - } + auto st = checked.maybeLstat(); + auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); } catch (SysError & e) { /* Don't give away info from errors while canonicalising diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 7638d2d82..81be64482 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -36,11 +36,11 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); } - Stat lstat(const CanonPath & path) override + std::optional maybeLstat(const CanonPath & path) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return PosixSourceAccessor::lstat(absPath); + return PosixSourceAccessor::maybeLstat(absPath); } DirEntries readDirectory(const CanonPath & path) override diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 817d063ba..6468ece41 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -20,12 +20,12 @@ struct MemoryInputAccessorImpl : MemoryInputAccessor return i != files.end(); } - Stat lstat(const CanonPath & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = files.find(path); if (i != files.end()) return Stat { .type = tRegular, .isExecutable = false }; - throw Error("file '%s' does not exist", path); + return std::nullopt; } DirEntries readDirectory(const CanonPath & path) override diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 48b4fe626..8a8d64f3f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -44,9 +44,13 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) return nix::pathExists(path.abs()); } -SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { - auto st = nix::lstat(path.abs()); + struct stat st; + if (::lstat(path.c_str(), &st)) { + if (errno == ENOENT) return std::nullopt; + throw SysError("getting status of '%s'", showPath(path)); + } mtime = std::max(mtime, st.st_mtime); return Stat { .type = diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 608f96ee2..cf087d26e 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -22,7 +22,7 @@ struct PosixSourceAccessor : SourceAccessor bool pathExists(const CanonPath & path) override; - Stat lstat(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d168a9667..5b0c7dd34 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -42,12 +42,12 @@ Hash SourceAccessor::hashPath( return sink.finish().first; } -std::optional SourceAccessor::maybeLstat(const CanonPath & path) +SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) { - // FIXME: merge these into one operation. - if (!pathExists(path)) - return {}; - return lstat(path); + if (auto st = maybeLstat(path)) + return *st; + else + throw Error("path '%s' does not exist", showPath(path)); } std::string SourceAccessor::showPath(const CanonPath & path) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index fd823aa39..80bc02b48 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -61,9 +61,9 @@ struct SourceAccessor bool isExecutable = false; // regular files only }; - virtual Stat lstat(const CanonPath & path) = 0; + Stat lstat(const CanonPath & path); - std::optional maybeLstat(const CanonPath & path); + virtual std::optional maybeLstat(const CanonPath & path) = 0; typedef std::optional DirEntry; diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp index dea119ae8..a287067cd 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -1 +1 @@ -error: getting status of '/pwd/lang/fnord': No such file or directory +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp index dea119ae8..a287067cd 100644 --- a/tests/functional/lang/eval-fail-nonexist-path.err.exp +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -1 +1 @@ -error: getting status of '/pwd/lang/fnord': No such file or directory +error: path '/pwd/lang/fnord' does not exist From 53811238790f4bb5f9df74bb25047fe5b734a61f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:33:19 +0100 Subject: [PATCH 258/402] Unify DirEntries types --- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/fs-accessor.hh | 4 +++- src/libstore/local-fs-store.cc | 6 +++--- src/libstore/nar-accessor.cc | 8 ++++---- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-fs-accessor.hh | 2 +- src/nix/ls.cc | 2 +- src/nix/why-depends.cc | 2 +- 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f9abd8cbd..b61868413 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -231,14 +231,14 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex1("^[0-9a-f]{2}$"); std::regex regex2("^[0-9a-f]{38}\\.debug$"); - for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; - for (auto & s2 : narAccessor->readDirectory(dir)) { + for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 1e951ec57..f04a92206 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -39,7 +39,9 @@ public: virtual std::optional stat(const Path & path) = 0; - virtual StringSet readDirectory(const Path & path) = 0; + using DirEntries = SourceAccessor::DirEntries; + + virtual DirEntries readDirectory(const Path & path) = 0; /** * Read a file inside the store. diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index bb83a9cd4..65cbb9e35 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -48,15 +48,15 @@ struct LocalStoreAccessor : public FSAccessor S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const Path & path) override { auto realPath = toRealPath(path); auto entries = nix::readDirectory(realPath); - StringSet res; + DirEntries res; for (auto & entry : entries) - res.insert(entry.name); + res.insert_or_assign(entry.name, std::nullopt); return res; } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 43a78a362..fe857a60e 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -190,16 +190,16 @@ struct NarAccessor : public FSAccessor return i->stat; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const Path & path) override { auto i = get(path); if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); - StringSet res; + DirEntries res; for (auto & child : i.children) - res.insert(child.first); + res.insert_or_assign(child.first, std::nullopt); return res; } @@ -264,7 +264,7 @@ json listNar(ref accessor, const Path & path, bool recurse) { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & name : accessor->readDirectory(path)) { + for (auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { res2[name] = listNar(accessor, path + "/" + name, true); } else diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 6c87ebeaa..21419700c 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -107,7 +107,7 @@ std::optional RemoteFSAccessor::stat(const Path & path) return res.first->stat(res.second); } -StringSet RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const Path & path) { auto res = fetch(path); return res.first->readDirectory(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 5cf759aa0..8de3b7bcd 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -30,7 +30,7 @@ public: std::optional stat(const Path & path) override; - StringSet readDirectory(const Path & path) override; + DirEntries readDirectory(const Path & path) override; std::string readFile(const Path & path, bool requireValidPath = true) override; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 8dc8a47b4..0ca08cea8 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -74,7 +74,7 @@ struct MixLs : virtual Args, MixJSON { if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); - for (auto & name : names) + for (auto & [name, type] : names) showFile(curPath + "/" + name, relPath + "/" + name); } else showFile(curPath, relPath); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 912ba72fb..04c1a0c1c 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -224,7 +224,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (st->type == FSAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); - for (auto & name : names) + for (auto & [name, type] : names) visitPath(p + "/" + name); } From 50aae0a14c5bbbde5785ead8f46b28333e6248ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:39:40 +0100 Subject: [PATCH 259/402] FSAccessor: Make the fileSize and narOffset fields optional The narOffset field only applies to NAR accessors. The fileSize field may be too expensive to compute for certain accessors (e.g. libgit). --- src/libstore/fs-accessor.hh | 4 ++-- src/libstore/nar-accessor.cc | 11 ++++++----- src/nix/ls.cc | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f04a92206..f6c002a2d 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -23,7 +23,7 @@ public: /** * For regular files only: the size of the file. */ - uint64_t fileSize = 0; + std::optional fileSize; /** * For regular files only: whether this is an executable. */ @@ -32,7 +32,7 @@ public: * For regular files only: the position of the contents of this * file in the NAR. */ - uint64_t narOffset = 0; + std::optional narOffset; }; virtual ~FSAccessor() { } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index fe857a60e..f1be5606e 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -210,10 +210,10 @@ struct NarAccessor : public FSAccessor if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); + if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); assert(nar); - return std::string(*nar, i.stat.narOffset, i.stat.fileSize); + return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } std::string readLink(const Path & path) override @@ -253,11 +253,12 @@ json listNar(ref accessor, const Path & path, bool recurse) switch (st->type) { case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st->fileSize; + if (st->fileSize) + obj["size"] = *st->fileSize; if (st->isExecutable) obj["executable"] = true; - if (st->narOffset) - obj["narOffset"] = st->narOffset; + if (st->narOffset && *st->narOffset) + obj["narOffset"] = *st->narOffset; break; case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 0ca08cea8..da978f379 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -52,7 +52,7 @@ struct MixLs : virtual Args, MixJSON (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st->fileSize, relPath); + auto line = fmt("%s %20d %s", tp, st->fileSize.value_or(0), relPath); if (st->type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); From 581693bdea3981eb0b106c904c7a1fed7f7582ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 16:33:22 +0100 Subject: [PATCH 260/402] fmt(): Handle std::string_view --- src/libutil/fmt.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 727255b45..ac72e47fb 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s) return s; } +inline std::string fmt(std::string_view s) +{ + return std::string(s); +} + inline std::string fmt(const char * s) { return s; From 1a902f5fa7d4f268d0fec3e44a48ecc2445b3b6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 17:09:28 +0100 Subject: [PATCH 261/402] Merge FSAccessor into SourceAccessor --- src/libstore/binary-cache-store.cc | 24 +++++------ src/libstore/binary-cache-store.hh | 2 +- src/libstore/derivations.cc | 1 - src/libstore/dummy-store.cc | 2 +- src/libstore/fs-accessor.hh | 58 --------------------------- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-fs-store.cc | 34 +++++++++------- src/libstore/local-fs-store.hh | 2 +- src/libstore/nar-accessor.cc | 64 ++++++++++++------------------ src/libstore/nar-accessor.hh | 11 ++--- src/libstore/remote-fs-accessor.cc | 28 ++++++------- src/libstore/remote-fs-accessor.hh | 21 +++++----- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 8 ++-- src/libstore/store-api.hh | 4 +- src/libstore/uds-remote-store.hh | 4 +- src/libutil/source-accessor.cc | 5 +++ src/libutil/source-accessor.hh | 22 ++++++++-- src/nix/bundle.cc | 1 - src/nix/cat.cc | 13 +++--- src/nix/ls.cc | 54 +++++++++++-------------- src/nix/run.cc | 8 ++-- src/nix/why-depends.cc | 22 +++++----- tests/functional/nar-access.sh | 8 +++- 25 files changed, 178 insertions(+), 224 deletions(-) delete mode 100644 src/libstore/fs-accessor.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b61868413..dd9e2f3af 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -2,7 +2,7 @@ #include "binary-cache-store.hh" #include "compression.hh" #include "derivations.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "nar-info.hh" #include "sync.hh" @@ -143,7 +143,7 @@ ref BinaryCacheStore::addToStoreCommon( write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink { htSHA256 }; - std::shared_ptr narAccessor; + std::shared_ptr narAccessor; HashSink narHashSink { htSHA256 }; { FdSink fileSink(fdTemp.get()); @@ -195,7 +195,7 @@ ref BinaryCacheStore::addToStoreCommon( if (writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), "", true)}, + {"root", listNar(ref(narAccessor), CanonPath::root, true)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); @@ -206,9 +206,9 @@ ref BinaryCacheStore::addToStoreCommon( specify the NAR file and member containing the debug info. */ if (writeDebugInfo) { - std::string buildIdDir = "/lib/debug/.build-id"; + CanonPath buildIdDir("lib/debug/.build-id"); - if (auto st = narAccessor->stat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { + if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -232,16 +232,16 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex2("^[0-9a-f]{38}\\.debug$"); for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + "/" + s1; + auto dir = buildIdDir + s1; - if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory + if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { - auto debugPath = dir + "/" + s2; + auto debugPath = dir + s2; - if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular + if ( narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; @@ -250,7 +250,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string key = "debuginfo/" + buildId; std::string target = "../" + narInfo->url; - threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target)); } } @@ -503,9 +503,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { upsertFile(filePath, info.toJSON().dump(), "application/json"); } -ref BinaryCacheStore::getFSAccessor() +ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 218a888e3..cea2a571f 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -148,7 +148,7 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a5ceb29dc..efdad18e1 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -6,7 +6,6 @@ #include "split.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" -#include "fs-accessor.hh" #include #include diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b5..821cda399 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store Callback> callback) noexcept override { callback(nullptr); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } }; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh deleted file mode 100644 index f6c002a2d..000000000 --- a/src/libstore/fs-accessor.hh +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once -///@file - -#include "types.hh" -#include "source-accessor.hh" - -#include - -namespace nix { - -/** - * An abstract class for accessing a filesystem-like structure, such - * as a (possibly remote) Nix store or the contents of a NAR file. - */ -class FSAccessor -{ -public: - using Type = SourceAccessor::Type; - - struct Stat - { - Type type; - /** - * For regular files only: the size of the file. - */ - std::optional fileSize; - /** - * For regular files only: whether this is an executable. - */ - bool isExecutable = false; - /** - * For regular files only: the position of the contents of this - * file in the NAR. - */ - std::optional narOffset; - }; - - virtual ~FSAccessor() { } - - virtual std::optional stat(const Path & path) = 0; - - using DirEntries = SourceAccessor::DirEntries; - - virtual DirEntries readDirectory(const Path & path) = 0; - - /** - * Read a file inside the store. - * - * If `requireValidPath` is set to `true` (the default), the path must be - * inside a valid store path, otherwise it just needs to be physically - * present (but not necessarily properly registered) - */ - virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; - - virtual std::string readLink(const Path & path) = 0; -}; - -} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 38fdf118f..731457354 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -363,7 +363,7 @@ public: void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } /** diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 65cbb9e35..63497acbd 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,26 +13,31 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public FSAccessor +struct LocalStoreAccessor : public SourceAccessor { ref store; + bool requireValidPath; - LocalStoreAccessor(ref store) : store(store) { } + LocalStoreAccessor(ref store, bool requireValidPath) + : store(store) + , requireValidPath(requireValidPath) + { } - Path toRealPath(const Path & path, bool requireValidPath = true) + Path toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path).first; + auto storePath = store->toStorePath(path.abs()).first; if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + return store->getRealStoreDir() + path.abs().substr(store->storeDir.size()); } - std::optional stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto realPath = toRealPath(path); + // FIXME: use PosixSourceAccessor. struct stat st; - if (lstat(realPath.c_str(), &st)) { + if (::lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; throw SysError("getting status of '%1%'", path); } @@ -48,7 +53,7 @@ struct LocalStoreAccessor : public FSAccessor S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } - DirEntries readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto realPath = toRealPath(path); @@ -61,21 +66,22 @@ struct LocalStoreAccessor : public FSAccessor return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { - return nix::readFile(toRealPath(path, requireValidPath)); + return nix::readFile(toRealPath(path)); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { return nix::readLink(toRealPath(path)); } }; -ref LocalFSStore::getFSAccessor() +ref LocalFSStore::getFSAccessor(bool requireValidPath) { return make_ref(ref( - std::dynamic_pointer_cast(shared_from_this()))); + std::dynamic_pointer_cast(shared_from_this())), + requireValidPath); } void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index d6bda05d1..bf855b67e 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -43,7 +43,7 @@ public: LocalFSStore(const Params & params); void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; /** * Creates symlink from the `gcRoot` to the `storePath` and diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f1be5606e..02993680f 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,7 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Stat stat; + SourceAccessor::Stat stat; std::string target; @@ -19,7 +19,7 @@ struct NarMember std::map children; }; -struct NarAccessor : public FSAccessor +struct NarAccessor : public SourceAccessor { std::optional nar; @@ -149,48 +149,36 @@ struct NarAccessor : public FSAccessor recurse(root, v); } - NarMember * find(const Path & path) + NarMember * find(const CanonPath & path) { - Path canon = path == "" ? "" : canonPath(path); NarMember * current = &root; - auto end = path.end(); - for (auto it = path.begin(); it != end; ) { - // because it != end, the remaining component is non-empty so we need - // a directory + + for (auto & i : path) { if (current->stat.type != Type::tDirectory) return nullptr; - - // skip slash (canonPath above ensures that this is always a slash) - assert(*it == '/'); - it += 1; - - // lookup current component - auto next = std::find(it, end, '/'); - auto child = current->children.find(std::string(it, next)); + auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; current = &child->second; - - it = next; } return current; } - NarMember & get(const Path & path) { + NarMember & get(const CanonPath & path) { auto result = find(path); - if (result == nullptr) + if (!result) throw Error("NAR file does not contain path '%1%'", path); return *result; } - std::optional stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = find(path); - if (i == nullptr) + if (!i) return std::nullopt; return i->stat; } - DirEntries readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto i = get(path); @@ -204,7 +192,7 @@ struct NarAccessor : public FSAccessor return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { auto i = get(path); if (i.stat.type != Type::tRegular) @@ -216,7 +204,7 @@ struct NarAccessor : public FSAccessor return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { auto i = get(path); if (i.stat.type != Type::tSymlink) @@ -225,40 +213,38 @@ struct NarAccessor : public FSAccessor } }; -ref makeNarAccessor(std::string && nar) +ref makeNarAccessor(std::string && nar) { return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(Source & source) { return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, +ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } using nlohmann::json; -json listNar(ref accessor, const Path & path, bool recurse) +json listNar(ref accessor, const CanonPath & path, bool recurse) { - auto st = accessor->stat(path); - if (!st) - throw Error("path '%s' does not exist in NAR", path); + auto st = accessor->lstat(path); json obj = json::object(); - switch (st->type) { + switch (st.type) { case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - if (st->fileSize) - obj["size"] = *st->fileSize; - if (st->isExecutable) + if (st.fileSize) + obj["size"] = *st.fileSize; + if (st.isExecutable) obj["executable"] = true; - if (st->narOffset && *st->narOffset) - obj["narOffset"] = *st->narOffset; + if (st.narOffset && *st.narOffset) + obj["narOffset"] = *st.narOffset; break; case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; @@ -267,7 +253,7 @@ json listNar(ref accessor, const Path & path, bool recurse) json &res2 = obj["entries"]; for (auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + name, true); } else res2[name] = json::object(); } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 5e19bd3c7..433774524 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,10 +1,11 @@ #pragma once ///@file +#include "source-accessor.hh" + #include #include -#include "fs-accessor.hh" namespace nix { @@ -14,9 +15,9 @@ struct Source; * Return an object that provides access to the contents of a NAR * file. */ -ref makeNarAccessor(std::string && nar); +ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +ref makeNarAccessor(Source & source); /** * Create a NAR accessor from a NAR listing (in the format produced by @@ -26,7 +27,7 @@ ref makeNarAccessor(Source & source); */ typedef std::function GetNarBytes; -ref makeLazyNarAccessor( +ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); @@ -34,6 +35,6 @@ ref makeLazyNarAccessor( * Write a JSON representation of the contents of a NAR (except file * contents). */ -nlohmann::json listNar(ref accessor, const Path & path, bool recurse); +nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 21419700c..03e57a565 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -8,8 +8,9 @@ namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) : store(store) + , requireValidPath(requireValidPath) , cacheDir(cacheDir) { if (cacheDir != "") @@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) +ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { if (cacheDir != "") { try { @@ -38,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, "", true); + nlohmann::json j = listNar(narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); @@ -48,11 +49,10 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str return narAccessor; } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) +std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto path = canonPath(path_); - - auto [storePath, restPath] = store->toStorePath(path); + auto [storePath, restPath_] = store->toStorePath(path.abs()); + auto restPath = CanonPath(restPath_); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); @@ -63,7 +63,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { + if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); @@ -101,25 +101,25 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -std::optional RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { auto res = fetch(path); - return res.first->stat(res.second); + return res.first->maybeLstat(res.second); } -SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path) { auto res = fetch(path); return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) +std::string RemoteFSAccessor::readFile(const CanonPath & path) { - auto res = fetch(path, requireValidPath); + auto res = fetch(path); return res.first->readFile(res.second); } -std::string RemoteFSAccessor::readLink(const Path & path) +std::string RemoteFSAccessor::readLink(const CanonPath & path) { auto res = fetch(path); return res.first->readLink(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 8de3b7bcd..d09762a53 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,40 +1,43 @@ #pragma once ///@file -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "ref.hh" #include "store-api.hh" namespace nix { -class RemoteFSAccessor : public FSAccessor +class RemoteFSAccessor : public SourceAccessor { ref store; - std::map> nars; + std::map> nars; + + bool requireValidPath; Path cacheDir; - std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); + std::pair, CanonPath> fetch(const CanonPath & path); friend class BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); - ref addToCache(std::string_view hashPart, std::string && nar); + ref addToCache(std::string_view hashPart, std::string && nar); public: RemoteFSAccessor(ref store, + bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); - std::optional stat(const Path & path) override; + std::optional maybeLstat(const CanonPath & path) override; - DirEntries readDirectory(const Path & path) override; + DirEntries readDirectory(const CanonPath & path) override; - std::string readFile(const Path & path, bool requireValidPath = true) override; + std::string readFile(const CanonPath & path) override; - std::string readLink(const Path & path) override; + std::string readLink(const CanonPath & path) override; }; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7bdc25433..f16949f42 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -970,7 +970,7 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink) copyNAR(conn->from, sink); } -ref RemoteStore::getFSAccessor() +ref RemoteStore::getFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this())); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f0985fdc1..1cc11af86 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -185,7 +185,7 @@ protected: friend struct ConnectionHandle; - virtual ref getFSAccessor() override; + virtual ref getFSAccessor(bool requireValidPath) override; virtual void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0399120d1..665b5fed7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,5 +1,5 @@ #include "crypto.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "derivations.hh" #include "store-api.hh" @@ -1338,12 +1338,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } -Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) +static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { - auto accessor = store.getFSAccessor(); + auto accessor = store.getFSAccessor(requireValidPath); try { return parseDerivation(store, - accessor->readFile(store.printStorePath(drvPath), requireValidPath), + accessor->readFile(CanonPath(store.printStorePath(drvPath))), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e123fccc5..6aa317e3d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -70,7 +70,7 @@ MakeError(InvalidStoreURI, Error); struct BasicDerivation; struct Derivation; -class FSAccessor; +struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -703,7 +703,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() = 0; + virtual ref getFSAccessor(bool requireValidPath = true) = 0; /** * Repair the contents of the given path by redownloading it using diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index cdb28a001..a5ac9080a 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -35,8 +35,8 @@ public: static std::set uriSchemes() { return {"unix"}; } - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } + ref getFSAccessor(bool requireValidPath) override + { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 5b0c7dd34..e2114e18f 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,6 +10,11 @@ SourceAccessor::SourceAccessor() { } +bool SourceAccessor::pathExists(const CanonPath & path) +{ + return maybeLstat(path).has_value(); +} + std::string SourceAccessor::readFile(const CanonPath & path) { StringSink sink; diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 80bc02b48..1a4e80361 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -40,7 +40,7 @@ struct SourceAccessor Sink & sink, std::function sizeCallback = [](uint64_t size){}); - virtual bool pathExists(const CanonPath & path) = 0; + virtual bool pathExists(const CanonPath & path); enum Type { tRegular, tSymlink, tDirectory, @@ -57,8 +57,24 @@ struct SourceAccessor struct Stat { Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only + + /** + * For regular files only: the size of the file. Not all + * accessors return this since it may be too expensive to + * compute. + */ + std::optional fileSize; + + /** + * For regular files only: whether this is an executable. + */ + bool isExecutable = false; + + /** + * For regular files only: the position of the contents of this + * file in the NAR. Only returned by NAR accessors. + */ + std::optional narOffset; }; Stat lstat(const CanonPath & path); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 504e35c81..54cc6a17f 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -4,7 +4,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fs-accessor.hh" #include "eval-inline.hh" using namespace nix; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index b5fe2506f..6e5a736f2 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" using namespace nix; @@ -9,14 +8,12 @@ struct MixCat : virtual Args { std::string path; - void cat(ref accessor) + void cat(ref accessor) { - if (auto st = accessor->stat(path)) { - if (st->type != FSAccessor::Type::tRegular) - throw Error("path '%1%' is not a regular file", path); - writeFull(STDOUT_FILENO, accessor->readFile(path)); - } else - throw Error("path '%1%' does not exist", path); + auto st = accessor->lstat(CanonPath(path)); + if (st.type != SourceAccessor::Type::tRegular) + throw Error("path '%1%' is not a regular file", path); + writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index da978f379..231456c9c 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" #include @@ -39,63 +38,58 @@ struct MixLs : virtual Args, MixJSON }); } - void listText(ref accessor) + void listText(ref accessor) { - std::function doPath; + std::function doPath; - auto showFile = [&](const Path & curPath, const std::string & relPath) { + auto showFile = [&](const CanonPath & curPath, std::string_view relPath) { if (verbose) { - auto st = accessor->stat(curPath); - assert(st); + auto st = accessor->lstat(curPath); std::string tp = - st->type == FSAccessor::Type::tRegular ? - (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st.type == SourceAccessor::Type::tRegular ? + (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st->fileSize.value_or(0), relPath); - if (st->type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath); + if (st.type == SourceAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st->type == FSAccessor::Type::tDirectory) - doPath(*st, curPath, relPath, false); + if (recursive && st.type == SourceAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { - auto st = accessor->stat(curPath); - assert(st); - if (st->type == FSAccessor::Type::tDirectory) - doPath(*st, curPath, relPath, false); + auto st = accessor->lstat(curPath); + if (st.type == SourceAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); } } }; - doPath = [&](const FSAccessor::Stat & st, const Path & curPath, - const std::string & relPath, bool showDirectory) + doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath, + std::string_view relPath, bool showDirectory) { - if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); for (auto & [name, type] : names) - showFile(curPath + "/" + name, relPath + "/" + name); + showFile(curPath + name, relPath + "/" + name); } else showFile(curPath, relPath); }; - auto st = accessor->stat(path); - if (!st) - throw Error("path '%1%' does not exist", path); - doPath(*st, path, - st->type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + auto path2 = CanonPath(path); + auto st = accessor->lstat(path2); + doPath(st, path2, + st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""), showDirectory); } - void list(ref accessor) + void list(ref accessor) { - if (path == "/") path = ""; - if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, path, recursive)); + logger->cout("%s", listNar(accessor, CanonPath(path), recursive)); } else listText(accessor); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 07806283c..1465e8cde 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -6,7 +6,7 @@ #include "derivations.hh" #include "local-store.hh" #include "finally.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" #include "build/personality.hh" @@ -119,9 +119,9 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) unixPath.push_front(store->printStorePath(path) + "/bin"); - auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (auto st = accessor->stat(propPath); st && st->type == SourceAccessor::tRegular) { - for (auto & p : tokenizeString(readFile(propPath))) + auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; + if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 04c1a0c1c..aecf65922 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "shared.hh" #include @@ -175,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions struct BailOut { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - auto pathS = store->printStorePath(node.path); + CanonPath pathS(store->printStorePath(node.path)); assert(node.dist != inf); if (precise) { @@ -183,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", - pathS); + pathS.abs()); } if (node.path == dependencyPath && !all @@ -210,25 +210,25 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions contain the reference. */ std::map hits; - std::function visitPath; + std::function visitPath; - visitPath = [&](const Path & p) { - auto st = accessor->stat(p); + visitPath = [&](const CanonPath & p) { + auto st = accessor->maybeLstat(p); assert(st); - auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); + auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st->type == FSAccessor::Type::tDirectory) { + if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & [name, type] : names) - visitPath(p + "/" + name); + visitPath(p + name); } - else if (st->type == FSAccessor::Type::tRegular) { + else if (st->type == SourceAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -246,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st->type == FSAccessor::Type::tSymlink) { + else if (st->type == SourceAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index d487d58d2..13d23c342 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -25,6 +25,12 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +# Check that 'nix store cat' fails on invalid store paths. +invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" +mv $storePath $invalidPath +(! nix store cat $invalidPath/foo/baz) +mv $invalidPath $storePath + # Test --json. diff -u \ <(nix nar ls --json $narFile / | jq -S) \ @@ -46,7 +52,7 @@ diff -u \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist' expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. From 2f5c1a27dc71275c1d4c96cff42beffed0d4d2f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 17:22:25 +0100 Subject: [PATCH 262/402] LocalStoreAccessor: Reuse PosixSourceAccessor --- src/libstore/local-fs-store.cc | 48 ++++++++-------------------- src/libutil/posix-source-accessor.cc | 3 +- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 63497acbd..953f3a264 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "source-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,7 +13,7 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public SourceAccessor +struct LocalStoreAccessor : PosixSourceAccessor { ref store; bool requireValidPath; @@ -23,57 +23,35 @@ struct LocalStoreAccessor : public SourceAccessor , requireValidPath(requireValidPath) { } - Path toRealPath(const CanonPath & path) + CanonPath toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path.abs()).first; + auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + path.abs().substr(store->storeDir.size()); + return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest); } std::optional maybeLstat(const CanonPath & path) override { - auto realPath = toRealPath(path); - - // FIXME: use PosixSourceAccessor. - struct stat st; - if (::lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error("file '%1%' has unsupported type", path); - - return {{ - S_ISREG(st.st_mode) ? Type::tRegular : - S_ISLNK(st.st_mode) ? Type::tSymlink : - Type::tDirectory, - S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } DirEntries readDirectory(const CanonPath & path) override { - auto realPath = toRealPath(path); - - auto entries = nix::readDirectory(realPath); - - DirEntries res; - for (auto & entry : entries) - res.insert_or_assign(entry.name, std::nullopt); - - return res; + return PosixSourceAccessor::readDirectory(toRealPath(path)); } - std::string readFile(const CanonPath & path) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { - return nix::readFile(toRealPath(path)); + return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback); } std::string readLink(const CanonPath & path) override { - return nix::readLink(toRealPath(path)); + return PosixSourceAccessor::readLink(toRealPath(path)); } }; diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 8a8d64f3f..d5e32d989 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -58,7 +58,8 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP S_ISDIR(st.st_mode) ? tDirectory : S_ISLNK(st.st_mode) ? tSymlink : tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, }; } From eab92927388bca29027a98199184ebb5e4e3c03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Wed, 1 Nov 2023 18:10:06 +0100 Subject: [PATCH 263/402] fix: gcc complains about if which doesn't guard the indented statement --- src/libstore/build/local-derivation-goal.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 738e7051e..dcb7dc6bc 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1563,10 +1563,11 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - if (pathExists(target)) + if (pathExists(target)) { // There is a similar debug message in doBind, so only run it in this block to not have double messages. debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + } /* Bind-mount the path into the sandbox. This requires entering its mount namespace, which is not possible From e47984ce0b37cb8e00b66e85703c1ff72de80a73 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 20:19:08 +0100 Subject: [PATCH 264/402] Fix whitespace Co-authored-by: John Ericson --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index dd9e2f3af..6a52c4c51 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -241,7 +241,7 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { auto debugPath = dir + s2; - if ( narAccessor->lstat(debugPath).type != SourceAccessor::tRegular + if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; From d7710a40be1a871859d331e9a50cc7f31797d792 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Nov 2023 20:05:23 -0400 Subject: [PATCH 265/402] flake: Temporarily get Nixpkgs ahead of Hydra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/31ed632c692e6a36cfc18083b88ece892f863ed4' (2023-09-21) → 'github:NixOS/nixpkgs/9eb24edd6a0027fed010ccfe300a9734d029983c' (2023-11-01) --- flake.lock | 8 ++++---- flake.nix | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 56df9c3fb..991cef1ee 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695283060, - "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=", + "lastModified": 1698876495, + "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4", + "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "release-23.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 398ba10a0..7cc4ed7fe 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # FIXME go back to nixos-23.05-small once + # https://github.com/NixOS/nixpkgs/pull/264875 is included. + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 4ba8b182be350a04caf5b7efff6b804d789570ad Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 04:50:43 +0200 Subject: [PATCH 266/402] document store objects in terms of their constituent parts this also rephrases the introductory sentence to be more general, in order to avoid the same word being repeated in short succession. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/architecture/architecture.md | 2 +- doc/manual/src/glossary.md | 2 +- doc/manual/src/store/index.md | 5 +++-- doc/manual/src/store/store-object.md | 10 ++++++++++ 5 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 doc/manual/src/store/store-object.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 2fe77d2c6..c728f5296 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -18,6 +18,7 @@ - [Uninstalling Nix](installation/uninstall.md) - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) + - [Store Object](store/store-object.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 6e832e1f9..79429508f 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -63,7 +63,7 @@ The command line interface and Nix expressions are what users deal with most. > The Nix language itself does not have a notion of *packages* or *configurations*. > As far as we are concerned here, the inputs and results of a build plan are just data. -Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them. +Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data, which are made available to the operating system as files. A build plan itself is a series of *build tasks*, together with their build inputs. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index ad3cc147b..b6d8a433a 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -59,7 +59,7 @@ - [store]{#gloss-store} A collection of store objects, with operations to manipulate that collection. - See [Nix Store] for details. + See [Nix store](./store/index.md) for details. There are many types of stores. See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md index 316e04179..8a5305062 100644 --- a/doc/manual/src/store/index.md +++ b/doc/manual/src/store/index.md @@ -1,4 +1,5 @@ # Nix Store -The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them. -There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches. +The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. + +There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches. diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md new file mode 100644 index 000000000..0b2b84ea5 --- /dev/null +++ b/doc/manual/src/store/store-object.md @@ -0,0 +1,10 @@ +## Store Object + +A Nix store is a collection of *store objects* with *references* between them. +A store object consists of + + - A [file system object](./file-system-object.md) as data + - A set of [store paths](@docroot@/glossary.md#gloss-store-path) as references to other store objects + +Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): +Once created, they do not change until they are deleted. From d7b7a79f3ef865ebe5f61962a7c2737cdb5d6445 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 23 Oct 2023 01:39:26 +0200 Subject: [PATCH 267/402] document store paths update the glossary to point to the new page. since this is a cross-cutting concern, it warrants its own section in the manual. Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/glossary.md | 9 ++-- doc/manual/src/store/store-object.md | 2 +- doc/manual/src/store/store-path.md | 69 ++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 doc/manual/src/store/store-path.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c728f5296..794f78a07 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -19,6 +19,7 @@ - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) - [Store Object](store/store-object.md) + - [Store Path](store/store-path.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b6d8a433a..07891175a 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -86,10 +86,13 @@ - [store path]{#gloss-store-path} - The location of a [store object] in the file system, i.e., an - immediate child of the Nix store directory. + The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory. - Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + > **Example** + > + > `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + + See [Store Path](@docroot@/store/store-path.md) for details. [store path]: #gloss-store-path diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md index 0b2b84ea5..caf5657d1 100644 --- a/doc/manual/src/store/store-object.md +++ b/doc/manual/src/store/store-object.md @@ -4,7 +4,7 @@ A Nix store is a collection of *store objects* with *references* between them. A store object consists of - A [file system object](./file-system-object.md) as data - - A set of [store paths](@docroot@/glossary.md#gloss-store-path) as references to other store objects + - A set of [store paths](./store-path.md) as references to other store objects Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): Once created, they do not change until they are deleted. diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md new file mode 100644 index 000000000..b5ad0c654 --- /dev/null +++ b/doc/manual/src/store/store-path.md @@ -0,0 +1,69 @@ +# Store Path + +Nix implements references to [store objects](./index.md#store-object) as *store paths*. + +Think of a store path as an [opaque], [unique identifier]: +The only way to obtain store path is by adding or building store objects. +A store path will always reference exactly one store object. + +[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + +Store paths are pairs of + +- A 20-byte digest for identification +- A symbolic name for people to read + +> **Example** +> +> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +> - Name: `firefox-33.1` + +To make store objects accessible to operating system processes, stores have to expose store objects through the file system. + +A store path is rendered to a file system path as the concatenation of + +- [Store directory](#store-directory) (typically `/nix/store`) +- Path separator (`/`) +- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) +- Hyphen (`-`) +- Name + +> **Example** +> +> ``` +> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +## Store Directory + +Every [Nix store](./index.md) has a store directory. + +Not every store can be accessed through the file system. +But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). + +[file system objects]: ./file-system-object.md + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. + +> **Note** +> +> The store directory defaults to `/nix/store`, but is in principle arbitrary. + +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee referential integrity if store paths do not cross store boundaries. + +Therefore one can only copy store objects to a different store if + +- The source and target stores' directories match + + or + +- The store object in question has no references, that is, contains no store paths + +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. From 55ed09c4f251d87e5aa23c7fb931e87cea63c68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Thu, 2 Nov 2023 09:22:00 +0100 Subject: [PATCH 268/402] Remove stray executable permissions on source files Noticed because of a warning during an rpm build: *** WARNING: ./usr/src/debug/nix-2.18.1-1.fc40.x86_64/src/nix-copy-closure/nix-copy-closure.cc is executable but has no shebang, removing executable bit *** WARNING: ./usr/src/debug/nix-2.18.1-1.fc40.x86_64/src/nix-channel/nix-channel.cc is executable but has no shebang, removing executable bit --- src/nix-channel/nix-channel.cc | 0 src/nix-copy-closure/nix-copy-closure.cc | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/nix-channel/nix-channel.cc mode change 100755 => 100644 src/nix-copy-closure/nix-copy-closure.cc diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc old mode 100755 new mode 100644 diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc old mode 100755 new mode 100644 From d26c317b14bc3f0ce82d5a91acc63e62a8836dee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Nov 2023 13:40:54 +0100 Subject: [PATCH 269/402] Use expect Co-authored-by: John Ericson --- tests/functional/nar-access.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index 13d23c342..218b521fb 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -28,7 +28,7 @@ diff -u baz.cat-nar $storePath/foo/baz # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" mv $storePath $invalidPath -(! nix store cat $invalidPath/foo/baz) +expect 1 nix store cat $invalidPath/foo/baz mv $invalidPath $storePath # Test --json. From b107431816fcbf364aeae6942cc9d1e709635a44 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Nov 2023 11:15:21 -0400 Subject: [PATCH 270/402] Systematize characterization tests a bit more Deduplicating code moreover enforcing the pattern means: - It is easier to write new characterization tests because less boilerplate - It is harder to mess up new tests because there are fewer places to make mistakes. Co-authored-by: Jacek Galowicz --- src/libstore/tests/characterization.hh | 28 ------ src/libstore/tests/common-protocol.cc | 48 ++++------ src/libstore/tests/derivation.cc | 116 ++++++++----------------- src/libstore/tests/libstore.hh | 2 +- src/libstore/tests/protocol.hh | 61 +++++-------- src/libutil/tests/characterization.hh | 111 +++++++++++++++++++++++ 6 files changed, 183 insertions(+), 183 deletions(-) delete mode 100644 src/libstore/tests/characterization.hh create mode 100644 src/libutil/tests/characterization.hh diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh deleted file mode 100644 index 46bf4b2e5..000000000 --- a/src/libstore/tests/characterization.hh +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once -///@file - -namespace nix { - -/** - * The path to the `unit-test-data` directory. See the contributing - * guide in the manual for further details. - */ -static Path getUnitTestData() { - return getEnv("_NIX_TEST_UNIT_DATA").value(); -} - -/** - * Whether we should update "golden masters" instead of running tests - * against them. See the contributing guide in the manual for further - * details. - */ -static bool testAccept() { - return getEnv("_NIX_TEST_ACCEPT") == "1"; -} - -constexpr std::string_view cannotReadGoldenMaster = - "Cannot read golden master because another test is also updating it"; - -constexpr std::string_view updatingGoldenMaster = - "Updating golden master"; -} diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc index b3f4977d2..c09ac6a3e 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/src/libstore/tests/common-protocol.cc @@ -20,16 +20,9 @@ public: * Golden test for `T` reading */ template - void readTest(PathView testStem, T value) + void readProtoTest(PathView testStem, const T & expected) { - if (testAccept()) - { - GTEST_SKIP() << cannotReadGoldenMaster; - } - else - { - auto encoded = readFile(goldenMaster(testStem)); - + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { T got = ({ StringSource from { encoded }; CommonProto::Serialise::read( @@ -37,44 +30,33 @@ public: CommonProto::ReadConn { .from = from }); }); - ASSERT_EQ(got, value); - } + ASSERT_EQ(got, expected); + }); } /** * Golden test for `T` write */ template - void writeTest(PathView testStem, const T & value) + void writeProtoTest(PathView testStem, const T & decoded) { - auto file = goldenMaster(testStem); - - StringSink to; - CommonProto::write( - *store, - CommonProto::WriteConn { .to = to }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << updatingGoldenMaster; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } + CharacterizationTest::writeTest(testStem, [&]() -> std::string { + StringSink to; + CommonProto::Serialise::write( + *store, + CommonProto::WriteConn { .to = to }, + decoded); + return to.s; + }); } }; #define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ TEST_F(CommonProtoTest, NAME ## _read) { \ - readTest(STEM, VALUE); \ + readProtoTest(STEM, VALUE); \ } \ TEST_F(CommonProtoTest, NAME ## _write) { \ - writeTest(STEM, VALUE); \ + writeProtoTest(STEM, VALUE); \ } CHARACTERIZATION_TEST( diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index ca0cdff71..29d5693db 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -11,20 +11,20 @@ namespace nix { using nlohmann::json; -class DerivationTest : public LibStoreTest +class DerivationTest : public CharacterizationTest, public LibStoreTest { + Path unitTestData = getUnitTestData() + "/libstore/derivation"; + public: + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + /** * We set these in tests rather than the regular globals so we don't have * to worry about race conditions if the tests run concurrently. */ ExperimentalFeatureSettings mockXpSettings; - - Path unitTestData = getUnitTestData() + "/libstore/derivation"; - - Path goldenMaster(std::string_view testStem) { - return unitTestData + "/" + testStem; - } }; class CaDerivationTest : public DerivationTest @@ -73,14 +73,8 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { #define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ - if (testAccept()) \ - { \ - GTEST_SKIP() << cannotReadGoldenMaster; \ - } \ - else \ - { \ - auto encoded = json::parse( \ - readFile(goldenMaster("output-" #NAME ".json"))); \ + readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ DerivationOutput got = DerivationOutput::fromJSON( \ *store, \ DRV_NAME, \ @@ -89,28 +83,20 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { mockXpSettings); \ DerivationOutput expected { VAL }; \ ASSERT_EQ(got, expected); \ - } \ + }); \ } \ \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - auto file = goldenMaster("output-" #NAME ".json"); \ - \ - json got = DerivationOutput { VAL }.toJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME); \ - \ - if (testAccept()) \ - { \ - createDirs(dirOf(file)); \ - writeFile(file, got.dump(2) + "\n"); \ - GTEST_SKIP() << updatingGoldenMaster; \ - } \ - else \ - { \ - auto expected = json::parse(readFile(file)); \ - ASSERT_EQ(got, expected); \ - } \ + writeTest("output-" #NAME ".json", [&]() -> json { \ + return DerivationOutput { (VAL) }.toJSON( \ + *store, \ + (DRV_NAME), \ + (OUTPUT_NAME)); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ } TEST_JSON(DerivationTest, inputAddressed, @@ -167,50 +153,30 @@ TEST_JSON(ImpureDerivationTest, impure, #define TEST_JSON(FIXTURE, NAME, VAL) \ TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ - if (testAccept()) \ - { \ - GTEST_SKIP() << cannotReadGoldenMaster; \ - } \ - else \ - { \ - auto encoded = json::parse( \ - readFile(goldenMaster( #NAME ".json"))); \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ Derivation expected { VAL }; \ Derivation got = Derivation::fromJSON( \ *store, \ encoded, \ mockXpSettings); \ ASSERT_EQ(got, expected); \ - } \ + }); \ } \ \ TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - auto file = goldenMaster( #NAME ".json"); \ - \ - json got = Derivation { VAL }.toJSON(*store); \ - \ - if (testAccept()) \ - { \ - createDirs(dirOf(file)); \ - writeFile(file, got.dump(2) + "\n"); \ - GTEST_SKIP() << updatingGoldenMaster; \ - } \ - else \ - { \ - auto expected = json::parse(readFile(file)); \ - ASSERT_EQ(got, expected); \ - } \ + writeTest(#NAME ".json", [&]() -> json { \ + return Derivation { VAL }.toJSON(*store); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ } #define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ - if (testAccept()) \ - { \ - GTEST_SKIP() << cannotReadGoldenMaster; \ - } \ - else \ - { \ - auto encoded = readFile(goldenMaster( #NAME ".drv")); \ + readTest(#NAME ".drv", [&](auto encoded) { \ Derivation expected { VAL }; \ auto got = parseDerivation( \ *store, \ @@ -219,25 +185,13 @@ TEST_JSON(ImpureDerivationTest, impure, mockXpSettings); \ ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ ASSERT_EQ(got, expected); \ - } \ + }); \ } \ \ TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ - auto file = goldenMaster( #NAME ".drv"); \ - \ - auto got = (VAL).unparse(*store, false); \ - \ - if (testAccept()) \ - { \ - createDirs(dirOf(file)); \ - writeFile(file, got); \ - GTEST_SKIP() << updatingGoldenMaster; \ - } \ - else \ - { \ - auto expected = readFile(file); \ - ASSERT_EQ(got, expected); \ - } \ + writeTest(#NAME ".drv", [&]() -> std::string { \ + return (VAL).unparse(*store, false); \ + }); \ } Derivation makeSimpleDrv(const Store & store) { diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh index ef93457b5..78b162b95 100644 --- a/src/libstore/tests/libstore.hh +++ b/src/libstore/tests/libstore.hh @@ -8,7 +8,7 @@ namespace nix { -class LibStoreTest : public ::testing::Test { +class LibStoreTest : public virtual ::testing::Test { public: static void SetUpTestSuite() { initLibStore(); diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 7fdd3e11c..0378b3e1f 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -7,12 +7,11 @@ namespace nix { template -class ProtoTest : public LibStoreTest +class ProtoTest : public CharacterizationTest, public LibStoreTest { -protected: Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; - Path goldenMaster(std::string_view testStem) { + Path goldenMaster(std::string_view testStem) const override { return unitTestData + "/" + testStem + ".bin"; } }; @@ -25,18 +24,11 @@ public: * Golden test for `T` reading */ template - void readTest(PathView testStem, typename Proto::Version version, T value) + void readProtoTest(PathView testStem, typename Proto::Version version, T expected) { - if (testAccept()) - { - GTEST_SKIP() << cannotReadGoldenMaster; - } - else - { - auto expected = readFile(ProtoTest::goldenMaster(testStem)); - + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { T got = ({ - StringSource from { expected }; + StringSource from { encoded }; Proto::template Serialise::read( *LibStoreTest::store, typename Proto::ReadConn { @@ -45,47 +37,36 @@ public: }); }); - ASSERT_EQ(got, value); - } + ASSERT_EQ(got, expected); + }); } /** * Golden test for `T` write */ template - void writeTest(PathView testStem, typename Proto::Version version, const T & value) + void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded) { - auto file = ProtoTest::goldenMaster(testStem); - - StringSink to; - Proto::write( - *LibStoreTest::store, - typename Proto::WriteConn { - .to = to, - .version = version, - }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << updatingGoldenMaster; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } + CharacterizationTest::writeTest(testStem, [&]() { + StringSink to; + Proto::template Serialise::write( + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, + decoded); + return std::move(to.s); + }); } }; #define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ TEST_F(FIXTURE, NAME ## _read) { \ - readTest(STEM, VERSION, VALUE); \ + readProtoTest(STEM, VERSION, VALUE); \ } \ TEST_F(FIXTURE, NAME ## _write) { \ - writeTest(STEM, VERSION, VALUE); \ + writeProtoTest(STEM, VERSION, VALUE); \ } } diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh new file mode 100644 index 000000000..10c8b4f7e --- /dev/null +++ b/src/libutil/tests/characterization.hh @@ -0,0 +1,111 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * The path to the `unit-test-data` directory. See the contributing + * guide in the manual for further details. + */ +static Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +/** + * Mixin class for writing characterization tests + */ +class CharacterizationTest : public virtual ::testing::Test +{ +protected: + /** + * While the "golden master" for this characterization test is + * located. It should not be shared with any other test. + */ + virtual Path goldenMaster(PathView testStem) const = 0; + +public: + /** + * Golden test for reading + * + * @param test hook that takes the contents of the file and does the + * actual work + */ + void readTest(PathView testStem, auto && test) + { + auto file = goldenMaster(testStem); + + if (testAccept()) + { + GTEST_SKIP() + << "Cannot read golden master " + << file + << "because another test is also updating it"; + } + else + { + test(readFile(file)); + } + } + + /** + * Golden test for writing + * + * @param test hook that produces contents of the file and does the + * actual work + */ + template + void writeTest( + PathView testStem, + std::invocable<> auto && test, + std::invocable auto && readFile2, + std::invocable auto && writeFile2) + { + auto file = goldenMaster(testStem); + + T got = test(); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile2(file, got); + GTEST_SKIP() + << "Updating golden master " + << file; + } + else + { + T expected = readFile2(file); + ASSERT_EQ(got, expected); + } + } + + /** + * Specialize to `std::string` + */ + void writeTest(PathView testStem, auto && test) + { + writeTest( + testStem, test, + [](const Path & f) -> std::string { + return readFile(f); + }, + [](const Path & f, const std::string & c) { + return writeFile(f, c); + }); + } +}; + +} From d15c3a33e680228c9deaa6d0898d4680cdc8dbc3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Nov 2023 16:11:20 -0400 Subject: [PATCH 271/402] Don't use `std::invocable` C++ concept yet It s not supported on all platforms yet. Can revert this once it is. --- src/libstore/tests/derivation.cc | 4 ++-- src/libutil/tests/characterization.hh | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 29d5693db..7becfa5ab 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -87,7 +87,7 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { } \ \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - writeTest("output-" #NAME ".json", [&]() -> json { \ + writeTest("output-" #NAME ".json", [&]() -> json { \ return DerivationOutput { (VAL) }.toJSON( \ *store, \ (DRV_NAME), \ @@ -165,7 +165,7 @@ TEST_JSON(ImpureDerivationTest, impure, } \ \ TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - writeTest(#NAME ".json", [&]() -> json { \ + writeTest(#NAME ".json", [&]() -> json { \ return Derivation { VAL }.toJSON(*store); \ }, [](const auto & file) { \ return json::parse(readFile(file)); \ diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh index 10c8b4f7e..6698c5239 100644 --- a/src/libutil/tests/characterization.hh +++ b/src/libutil/tests/characterization.hh @@ -66,16 +66,12 @@ public: * @param test hook that produces contents of the file and does the * actual work */ - template void writeTest( - PathView testStem, - std::invocable<> auto && test, - std::invocable auto && readFile2, - std::invocable auto && writeFile2) + PathView testStem, auto && test, auto && readFile2, auto && writeFile2) { auto file = goldenMaster(testStem); - T got = test(); + auto got = test(); if (testAccept()) { @@ -87,7 +83,7 @@ public: } else { - T expected = readFile2(file); + decltype(got) expected = readFile2(file); ASSERT_EQ(got, expected); } } @@ -97,7 +93,7 @@ public: */ void writeTest(PathView testStem, auto && test) { - writeTest( + writeTest( testStem, test, [](const Path & f) -> std::string { return readFile(f); From e5908212e25f2cb7a36ec176a1c7fcb2d522088b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Nov 2023 11:03:58 +0100 Subject: [PATCH 272/402] Fix nar-access test on macOS --- tests/functional/nar-access.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index 218b521fb..87981e7d9 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -27,9 +27,8 @@ diff -u baz.cat-nar $storePath/foo/baz # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" -mv $storePath $invalidPath +cp -r $storePath $invalidPath expect 1 nix store cat $invalidPath/foo/baz -mv $invalidPath $storePath # Test --json. diff -u \ From 55dd1244d280d768bfebb8ca2ec93e061d7aa4eb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Nov 2023 11:39:50 +0100 Subject: [PATCH 273/402] parseDerivation(): Fix warning about uninitialized 'version' variable --- src/libstore/derivations.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index efdad18e1..1fecd1c97 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -352,7 +352,7 @@ Derivation parseDerivation( expect(str, "erive("); version = DerivationATermVersion::Traditional; break; - case 'r': + case 'r': { expect(str, "rvWithVersion("); auto versionS = parseString(str); if (versionS == "xp-dyn-drv") { @@ -365,6 +365,9 @@ Derivation parseDerivation( expect(str, ","); break; } + default: + throw Error("derivation does not start with 'Derive' or 'DrvWithVersion'"); + } /* Parse the list of outputs. */ expect(str, "["); From b0455e9931fbcd996b1b240a4513132c36cf852c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Nov 2023 11:58:47 +0100 Subject: [PATCH 274/402] Fix uninitialized variable warnings on i686-linux https://hydra.nixos.org/build/239849607 --- src/libcmd/command.cc | 4 ++-- src/libcmd/installables.cc | 2 +- src/libstore/store-api.cc | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index a88ba8134..de9f546fc 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -175,7 +175,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(BuiltPath::Opaque{p}); + paths.emplace_back(BuiltPath::Opaque{p}); } else { paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { @@ -188,7 +188,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) } store->computeFSClosure(pathsRoots, pathsClosure); for (auto & path : pathsClosure) - paths.push_back(BuiltPath::Opaque{path}); + paths.emplace_back(BuiltPath::Opaque{path}); } } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 3aff601e0..bc0b8a988 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -663,7 +663,7 @@ BuiltPaths Installable::toBuiltPaths( BuiltPaths res; for (auto & drvPath : Installable::toDerivations(store, installables, true)) - res.push_back(BuiltPath::Opaque{drvPath}); + res.emplace_back(BuiltPath::Opaque{drvPath}); return res; } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ac96e8bb1..646b0ec7d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -819,7 +819,7 @@ void Store::substitutePaths(const StorePathSet & paths) std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(DerivedPath::Opaque{path}); + paths2.emplace_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index e62c4f6b1..60bc08146 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -449,7 +449,7 @@ static void main_nix_build(int argc, char * * argv) } } for (const auto & src : drv.inputSrcs) { - pathsToBuild.push_back(DerivedPath::Opaque{src}); + pathsToBuild.emplace_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 01742daa8..25068f801 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,12 +481,12 @@ static void printMissing(EvalState & state, DrvInfos & elems) std::vector targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{ + targets.emplace_back(DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else - targets.push_back(DerivedPath::Opaque{ + targets.emplace_back(DerivedPath::Opaque{ .path = i.queryOutPath(), }); From 60b363936d2fd53ac8741d35ba30ff1e4c405a9f Mon Sep 17 00:00:00 2001 From: r-vdp Date: Tue, 31 Oct 2023 17:32:09 +0100 Subject: [PATCH 275/402] libstore/ssh-ng: Fix phase reporting in log files. When doing local builds, we get phase reporting lines in the log file, they look like '@nix {"action":"setPhase","phase":"unpackPhase"}'. With the ssh-ng protocol, we do have access to these messages, but since we are only including messages of type resBuildLogLine in the logs, the phase information does not end up in the log file. The phase reporting could probably be improved altoghether (it looks like it is kind of accidental that these JSON messages for phase reporting show up but others don't, just because they are actually emitted by nixpkgs' stdenv), but as a first step I propose to make ssh-ng behave in the same way as local builds do. --- src/libstore/build/derivation-goal.cc | 23 +++++- tests/nixos/default.nix | 2 + tests/nixos/remote-builds-ssh-ng.nix | 108 ++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 tests/nixos/remote-builds-ssh-ng.nix diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 360c6b70b..0cfa9a148 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1317,9 +1317,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. - if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) { - auto f = (*json)["fields"]; - (*logSink)((f.size() > 0 ? f.at(0).get() : "") + "\n"); + if (s && !isWrittenToLog && logSink) { + const auto type = (*json)["type"]; + const auto fields = (*json)["fields"]; + if (type == resBuildLogLine) { + (*logSink)((fields.size() > 0 ? fields[0].get() : "") + "\n"); + } else if (type == resSetPhase && ! fields.is_null()) { + const auto phase = fields[0]; + if (! phase.is_null()) { + // nixpkgs' stdenv produces lines in the log to signal + // phase changes. + // We want to get the same lines in case of remote builds. + // The format is: + // @nix { "action": "setPhase", "phase": "$curPhase" } + const auto logLine = nlohmann::json::object({ + {"action", "setPhase"}, + {"phase", phase} + }); + (*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n"); + } + } } } currentHookLine.clear(); diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index b391d7ef2..4459aa664 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -21,6 +21,8 @@ in remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; + remoteBuildsSshNg = runNixOSTestFor "x86_64-linux" ./remote-builds-ssh-ng.nix; + nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix new file mode 100644 index 000000000..b59dde9bf --- /dev/null +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -0,0 +1,108 @@ +{ config, lib, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "${ + lib.concatStringsSep "; " [ + ''if [[ -n $NIX_LOG_FD ]]'' + ''then echo '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' >&''$NIX_LOG_FD'' + "fi" + "echo Hello" + "mkdir $out" + "cat /proc/sys/kernel/hostname > $out/host" + ] + }" ]; + outputs = [ "out" ]; + } + ''; +in + +{ + name = "remote-builds-ssh-ng"; + + nodes = + { builder = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + nix.settings.substituters = lib.mkForce [ ]; + }; + + client = + { config, lib, pkgs, ... }: + { nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ { hostName = "builder"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + protocol = "ssh-ng"; + } + ]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builder. + client.wait_for_unit("network.target") + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + + # Perform a build + out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output") + + # Verify that the build was done on the builder + builder.succeed(f"test -e {out.strip()}") + + # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix + buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output") + print(buildOutput) + + # Make sure that we get the expected build output + client.succeed("grep -qF Hello build-output") + + # We don't want phase reporting in the build output + client.fail("grep -qF '@nix' build-output") + + # Get the log file + client.succeed(f"nix-store --read-log {out.strip()} > log-output") + # Prefix the log lines to avoid nix intercepting lines starting with @nix + logOutput = client.succeed("sed -e 's/^/log-file:/' log-output") + print(logOutput) + + # Check that we get phase reporting in the log file + client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output") + ''; +} From 6df32889a51510dff44c776fa312b7ba61ab8edf Mon Sep 17 00:00:00 2001 From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com> Date: Fri, 20 Oct 2023 21:16:56 +0200 Subject: [PATCH 276/402] Add git commit verification input attributes This implements the git input attributes `verifyCommit`, `keytype`, `publicKey` and `publicKeys` as experimental feature `verified-fetches`. `publicKeys` should be a json string. This representation was chosen because all attributes must be of type bool, int or string so they can be included in flake uris (see definition of fetchers::Attr). --- src/libfetchers/fetchers.cc | 5 ++ src/libfetchers/fetchers.hh | 9 +++ src/libfetchers/git.cc | 104 +++++++++++++++++++++++++-- src/libutil/experimental-features.cc | 11 ++- src/libutil/experimental-features.hh | 1 + 5 files changed, 124 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 92692d23a..895515327 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -360,4 +360,9 @@ std::optional InputScheme::experimentalFeature() const return {}; } +std::string publicKeys_to_string(const std::vector& publicKeys) +{ + return ((nlohmann::json) publicKeys).dump(); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 7d768bac1..a056c8939 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -182,4 +182,13 @@ void registerInputScheme(std::shared_ptr && fetcher); nlohmann::json dumpRegisterInputSchemeInfo(); +struct PublicKey +{ + std::string type = "ssh-ed25519"; + std::string key; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key) + +std::string publicKeys_to_string(const std::vector&); + } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d625fe01e..51e551879 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -143,6 +143,69 @@ struct WorkdirInfo bool hasHead = false; }; +std::vector getPublicKeys(const Attrs & attrs) { + std::vector publicKeys; + if (attrs.contains("publicKeys")) { + nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys")); + ensureType(publicKeysJson, nlohmann::json::value_t::array); + publicKeys = publicKeysJson.get>(); + } + else { + publicKeys = {}; + } + if (attrs.contains("publicKey")) + publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")}); + return publicKeys; +} + +void doCommitVerification(const Path repoDir, const Path gitDir, const std::string rev, const std::vector& publicKeys) { + // Create ad-hoc allowedSignersFile and populate it with publicKeys + auto allowedSignersFile = createTempFile().second; + std::string allowedSigners; + for (const PublicKey& k : publicKeys) { + if (k.type != "ssh-dsa" + && k.type != "ssh-ecdsa" + && k.type != "ssh-ecdsa-sk" + && k.type != "ssh-ed25519" + && k.type != "ssh-ed25519-sk" + && k.type != "ssh-rsa") + warn("Unknow keytype: %s\n" + "Please use one of\n" + "- ssh-dsa\n" + "- ssh-ecdsa\n" + "- ssh-ecdsa-sk\n" + "- ssh-ed25519\n" + "- ssh-ed25519-sk\n" + "- ssh-rsa", k.type); + allowedSigners += "* " + k.type + " " + k.key + "\n"; + } + writeFile(allowedSignersFile, allowedSigners); + + // Run verification command + auto [status, output] = runProgram(RunOptions { + .program = "git", + .args = {"-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, "-C", repoDir, + "--git-dir", gitDir, "verify-commit", rev}, + .mergeStderrToStdout = true, + }); + + /* Evaluate result through status code and checking if public key fingerprints appear on stderr + * This is neccessary because the git command might also succeed due to the commit being signed by gpg keys + * that are present in the users key agent. */ + std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; + for (const PublicKey& k : publicKeys){ + // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally + auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); + re += "(" + escaped_fingerprint + ")"; + } + re += "]"; + if (status == 0 && std::regex_search(output, std::regex(re))) + printTalkative("Commit signature verification on commit %s succeeded", rev); + else + throw Error("Commit signature verification on commit %s failed: \n%s", rev, output); +} + // Returns whether a git workdir is clean and has commits. WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) { @@ -272,9 +335,9 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto & [name, value] : url.query) { - if (name == "rev" || name == "ref") + if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules" || name == "allRefs") + else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -306,14 +369,26 @@ struct GitInputScheme : InputScheme "name", "dirtyRev", "dirtyShortRev", + "verifyCommit", + "keytype", + "publicKey", + "publicKeys", }; } std::optional inputFromAttrs(const Attrs & attrs) const override { + for (auto & [name, _] : attrs) + if (name == "verifyCommit" + || name == "keytype" + || name == "publicKey" + || name == "publicKeys") + experimentalFeatureSettings.require(Xp::VerifiedFetches); + maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "allRefs"); + maybeGetBoolAttr(attrs, "verifyCommit"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) @@ -336,6 +411,15 @@ struct GitInputScheme : InputScheme if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) url.query.insert_or_assign("shallow", "1"); + if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false)) + url.query.insert_or_assign("verifyCommit", "1"); + auto publicKeys = getPublicKeys(input.attrs); + if (publicKeys.size() == 1) { + url.query.insert_or_assign("keytype", publicKeys.at(0).type); + url.query.insert_or_assign("publicKey", publicKeys.at(0).key); + } + else if (publicKeys.size() > 1) + url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys)); return url; } @@ -425,6 +509,8 @@ struct GitInputScheme : InputScheme bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); + std::vector publicKeys = getPublicKeys(input.attrs); + bool verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty()); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -445,6 +531,8 @@ struct GitInputScheme : InputScheme {"type", cacheType}, {"name", name}, {"rev", input.getRev()->gitRev()}, + {"verifyCommit", verifyCommit}, + {"publicKeys", publicKeys_to_string(publicKeys)}, }); }; @@ -467,12 +555,15 @@ struct GitInputScheme : InputScheme auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug - /* If this is a local directory and no ref or revision is given, + /* If this is a local directory, no ref or revision is given and no signature verification is needed, allow fetching directly from a dirty workdir. */ if (!input.getRef() && !input.getRev() && isLocal) { auto workdirInfo = getWorkdirInfo(input, actualUrl); if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); + if (verifyCommit) + throw Error("Can't fetch from a dirty workdir with commit signature verification enabled."); + else + return fetchFromWorkdir(store, input, actualUrl, workdirInfo); } } @@ -480,6 +571,8 @@ struct GitInputScheme : InputScheme {"type", cacheType}, {"name", name}, {"url", actualUrl}, + {"verifyCommit", verifyCommit}, + {"publicKeys", publicKeys_to_string(publicKeys)}, }); Path repoDir; @@ -637,6 +730,9 @@ struct GitInputScheme : InputScheme ); } + if (verifyCommit) + doCommitVerification(repoDir, gitDir, input.getRev()->gitRev(), publicKeys); + if (submodules) { Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 74af9aae0..47edca3a5 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -227,7 +227,14 @@ constexpr std::array xpFeatureDetails = {{ .description = R"( Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. )", - } + }, + { + .tag = Xp::VerifiedFetches, + .name = "verified-fetches", + .description = R"( + Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index e02f8353e..f005cc9ee 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -32,6 +32,7 @@ enum struct ExperimentalFeature ParseTomlTimestamps, ReadOnlyLocalStore, ConfigurableImpureEnv, + VerifiedFetches, }; /** From 098f0615c9401414a76e66653fbf4c9dd30d55a7 Mon Sep 17 00:00:00 2001 From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com> Date: Fri, 20 Oct 2023 21:17:14 +0200 Subject: [PATCH 277/402] fetchGit and flake: add publicKeys list input This adds publicKeys as an optional fetcher input attribute to flakes and builtins.fetchGit to provide a nix interface for the json-encoded `publicKeys` attribute of the git fetcher. Co-authored-by: Valentin Gagarin --- doc/manual/src/release-notes/rl-next.md | 3 +- src/libexpr/flake/flake.cc | 10 ++++- src/libexpr/primops/fetchTree.cc | 56 +++++++++++++++++++++++++ src/libfetchers/git.cc | 14 +++---- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 3cfb53998..8cd69f8fd 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -16,7 +16,6 @@ - `builtins.fetchTree` is now marked as stable. - - The interface for creating and updating lock files has been overhauled: - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. @@ -29,3 +28,5 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. + +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 70ae7b584..ded132695 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -8,6 +8,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "value-to-json.hh" namespace nix { @@ -140,8 +141,13 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)); + if (attr.name == state.symbols.create("publicKeys")) { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + NixStringContext emptyContext = {}; + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); + } else + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)); } #pragma GCC diagnostic pop } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 767f559be..3717b9022 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -7,6 +7,7 @@ #include "registry.hh" #include "tarball.hh" #include "url.hh" +#include "value-to-json.hh" #include #include @@ -125,6 +126,10 @@ static void fetchTree( attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean}); else if (attr.value->type() == nInt) attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer)); + else if (state.symbols[attr.name] == "publicKeys") { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); + } else state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", state.symbols[attr.name], showType(*attr.value))); @@ -427,6 +432,42 @@ static RegisterPrimOp primop_fetchGit({ With this argument being true, it's possible to load a `rev` from *any* `ref` (by default only `rev`s from the specified `ref` are supported). + - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`) + + Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. + If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKey` + + The public key against which `rev` is verified if `verifyCommit` is enabled. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `keytype` (default: `"ssh-ed25519"`) + + The key type of `publicKey`. + Possible values: + - `"ssh-dsa"` + - `"ssh-ecdsa"` + - `"ssh-ecdsa-sk"` + - `"ssh-ed25519"` + - `"ssh-ed25519-sk"` + - `"ssh-rsa"` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKeys` + + The public keys against which `rev` is verified if `verifyCommit` is enabled. + Must be given as a list of attribute sets with the following form: + ```nix + { + key = ""; + type = ""; # optional, default: "ssh-ed25519" + } + ``` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + Here are some examples of how to use `fetchGit`. - To fetch a private repository over SSH: @@ -501,6 +542,21 @@ static RegisterPrimOp primop_fetchGit({ } ``` + - To verify the commit signature: + + ```nix + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + verifyCommit = true; + publicKeys = [ + { + type = "ssh-ed25519"; + key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi"; + } + ]; + } + ``` + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 51e551879..72fba0582 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -169,14 +169,14 @@ void doCommitVerification(const Path repoDir, const Path gitDir, const std::stri && k.type != "ssh-ed25519" && k.type != "ssh-ed25519-sk" && k.type != "ssh-rsa") - warn("Unknow keytype: %s\n" + warn("Unknown keytype: %s\n" "Please use one of\n" "- ssh-dsa\n" - "- ssh-ecdsa\n" - "- ssh-ecdsa-sk\n" - "- ssh-ed25519\n" - "- ssh-ed25519-sk\n" - "- ssh-rsa", k.type); + " ssh-ecdsa\n" + " ssh-ecdsa-sk\n" + " ssh-ed25519\n" + " ssh-ed25519-sk\n" + " ssh-rsa", k.type); allowedSigners += "* " + k.type + " " + k.key + "\n"; } writeFile(allowedSignersFile, allowedSigners); @@ -201,7 +201,7 @@ void doCommitVerification(const Path repoDir, const Path gitDir, const std::stri } re += "]"; if (status == 0 && std::regex_search(output, std::regex(re))) - printTalkative("Commit signature verification on commit %s succeeded", rev); + printTalkative("Signature verification on commit %s succeeded", rev); else throw Error("Commit signature verification on commit %s failed: \n%s", rev, output); } From 271932782dd3d44e0e238bd3234ca1e97996cfea Mon Sep 17 00:00:00 2001 From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com> Date: Fri, 20 Oct 2023 21:18:01 +0200 Subject: [PATCH 278/402] fetchGit and flake: add commit signature verification tests This adds simple tests of the commit signature verification mechanism of fetchGit and its flake input wrapper. OpenSSH is added to the build dependencies since it's needed to create a key when testing the functionality. It is neither a built- nor a runtime dependency. --- flake.nix | 1 + tests/functional/fetchGitVerification.sh | 76 ++++++++++++++++++++++++ tests/functional/local.mk | 1 + 3 files changed, 78 insertions(+) create mode 100644 tests/functional/fetchGitVerification.sh diff --git a/flake.nix b/flake.nix index 7cc4ed7fe..51d818423 100644 --- a/flake.nix +++ b/flake.nix @@ -185,6 +185,7 @@ buildPackages.git buildPackages.mercurial # FIXME: remove? only needed for tests buildPackages.jq # Also for custom mdBook preprocessor. + buildPackages.openssh # only needed for tests (ssh-keygen) ] ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh new file mode 100644 index 000000000..4d9209498 --- /dev/null +++ b/tests/functional/fetchGitVerification.sh @@ -0,0 +1,76 @@ +source common.sh + +requireGit +[[ $(type -p ssh-keygen) ]] || skipTest "ssh-keygen not installed" # require ssh-keygen + +enableFeatures "verified-fetches" + +clearStore + +repo="$TEST_ROOT/git" + +# generate signing keys +keysDir=$TEST_ROOT/.ssh +mkdir -p "$keysDir" +ssh-keygen -f "$keysDir/testkey1" -t ed25519 -P "" -C "test key 1" +key1File="$keysDir/testkey1.pub" +publicKey1=$(awk '{print $2}' "$key1File") +ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2" +key2File="$keysDir/testkey2.pub" +publicKey2=$(awk '{print $2}' "$key2File") + +git init $repo +git -C $repo config user.email "foobar@example.com" +git -C $repo config user.name "Foobar" +git -C $repo config gpg.format ssh + +echo 'hello' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key1File" commit -S -m 'initial commit' + +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKey = \"$publicKey1\"; } + \"/text\")") = 'hello' ]] + +echo 'hello world' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key2File" commit -S -m 'second commit' + +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKeys = [{key = \"$publicKey1\";} {type = \"ssh-rsa\"; key = \"$publicKey2\";}]; } + \"/text\")") = 'hello world' ]] + +# Flake input test +flakeDir="$TEST_ROOT/flake" +mkdir -p "$flakeDir" +cat > "$flakeDir/flake.nix" < "$flakeDir/flake.nix" <&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] \ No newline at end of file diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 3679349f8..fe0d0c4ed 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -55,6 +55,7 @@ nix_tests = \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ + fetchGitVerification.sh \ flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ From 9b880e3e29c7a485b0e21495f2d089c5151589cc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Nov 2023 19:39:09 -0400 Subject: [PATCH 279/402] Factor out `MemorySourceAccessor`, implement missing features The new `MemorySourceAccessor` rather than being a slightly lossy flat map is a complete in-memory model of file system objects. Co-authored-by: Eelco Dolstra --- src/libfetchers/input-accessor.hh | 2 +- src/libfetchers/memory-input-accessor.cc | 44 ++------ src/libutil/memory-source-accessor.cc | 124 +++++++++++++++++++++++ src/libutil/memory-source-accessor.hh | 74 ++++++++++++++ 4 files changed, 205 insertions(+), 39 deletions(-) create mode 100644 src/libutil/memory-source-accessor.cc create mode 100644 src/libutil/memory-source-accessor.hh diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5dc05a363..68fdf07a7 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -14,7 +14,7 @@ struct SourcePath; class StorePath; class Store; -struct InputAccessor : SourceAccessor, std::enable_shared_from_this +struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this { /** * Return the maximum last-modified time of the files in this diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 6468ece41..057f3e37f 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -1,48 +1,16 @@ #include "memory-input-accessor.hh" +#include "memory-source-accessor.hh" namespace nix { -struct MemoryInputAccessorImpl : MemoryInputAccessor +struct MemoryInputAccessorImpl : MemoryInputAccessor, MemorySourceAccessor { - std::map files; - - std::string readFile(const CanonPath & path) override - { - auto i = files.find(path); - if (i == files.end()) - throw Error("file '%s' does not exist", path); - return i->second; - } - - bool pathExists(const CanonPath & path) override - { - auto i = files.find(path); - return i != files.end(); - } - - std::optional maybeLstat(const CanonPath & path) override - { - auto i = files.find(path); - if (i != files.end()) - return Stat { .type = tRegular, .isExecutable = false }; - return std::nullopt; - } - - DirEntries readDirectory(const CanonPath & path) override - { - return {}; - } - - std::string readLink(const CanonPath & path) override - { - throw UnimplementedError("MemoryInputAccessor::readLink"); - } - SourcePath addFile(CanonPath path, std::string && contents) override { - files.emplace(path, std::move(contents)); - - return {ref(shared_from_this()), std::move(path)}; + return { + ref(shared_from_this()), + MemorySourceAccessor::addFile(path, std::move(contents)) + }; } }; diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc new file mode 100644 index 000000000..f34f6c091 --- /dev/null +++ b/src/libutil/memory-source-accessor.cc @@ -0,0 +1,124 @@ +#include "memory-source-accessor.hh" + +namespace nix { + +MemorySourceAccessor::File * +MemorySourceAccessor::open(const CanonPath & path, std::optional create) +{ + File * cur = &root; + + bool newF = false; + + for (std::string_view name : path) + { + auto * curDirP = std::get_if(&cur->raw); + if (!curDirP) + return nullptr; + auto & curDir = *curDirP; + + auto i = curDir.contents.find(name); + if (i == curDir.contents.end()) { + if (!create) + return nullptr; + else { + newF = true; + i = curDir.contents.insert(i, { + std::string { name }, + File::Directory {}, + }); + } + } + cur = &i->second; + } + + if (newF && create) *cur = std::move(*create); + + return cur; +} + +std::string MemorySourceAccessor::readFile(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * r = std::get_if(&f->raw)) + return r->contents; + else + throw Error("file '%s' is not a regular file", path); +} + +bool MemorySourceAccessor::pathExists(const CanonPath & path) +{ + return open(path, std::nullopt); +} + +MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +{ + return std::visit(overloaded { + [](const Regular & r) { + return Stat { + .type = tRegular, + .fileSize = r.contents.size(), + .isExecutable = r.executable, + }; + }, + [](const Directory &) { + return Stat { + .type = tDirectory, + }; + }, + [](const Symlink &) { + return Stat { + .type = tSymlink, + }; + }, + }, this->raw); +} + +std::optional +MemorySourceAccessor::maybeLstat(const CanonPath & path) +{ + const auto * f = open(path, std::nullopt); + return f ? std::optional { f->lstat() } : std::nullopt; +} + +MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * d = std::get_if(&f->raw)) { + DirEntries res; + for (auto & [name, file] : d->contents) + res.insert_or_assign(name, file.lstat().type); + return res; + } else + throw Error("file '%s' is not a directory", path); + return {}; +} + +std::string MemorySourceAccessor::readLink(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * s = std::get_if(&f->raw)) + return s->target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +{ + auto * f = open(path, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * r = std::get_if(&f->raw)) + r->contents = std::move(contents); + else + throw Error("file '%s' is not a regular file", path); + + return path; +} + +} diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh new file mode 100644 index 000000000..014fa8098 --- /dev/null +++ b/src/libutil/memory-source-accessor.hh @@ -0,0 +1,74 @@ +#include "source-accessor.hh" +#include "variant-wrapper.hh" + +namespace nix { + +/** + * An source accessor for an in-memory file system. + */ +struct MemorySourceAccessor : virtual SourceAccessor +{ + /** + * In addition to being part of the implementation of + * `MemorySourceAccessor`, this has a side benefit of nicely + * defining what a "file system object" is in Nix. + */ + struct File { + struct Regular { + bool executable = false; + std::string contents; + + GENERATE_CMP(Regular, me->executable, me->contents); + }; + + struct Directory { + using Name = std::string; + + std::map> contents; + + GENERATE_CMP(Directory, me->contents); + }; + + struct Symlink { + std::string target; + + GENERATE_CMP(Symlink, me->target); + }; + + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(File); + + GENERATE_CMP(File, me->raw); + + Stat lstat() const; + }; + + File root { File::Directory {} }; + + GENERATE_CMP(MemorySourceAccessor, me->root); + + std::string readFile(const CanonPath & path) override; + bool pathExists(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; + DirEntries readDirectory(const CanonPath & path) override; + std::string readLink(const CanonPath & path) override; + + /** + * @param create If present, create this file and any parent directories + * that are needed. + * + * Return null if + * + * - `create = false`: File does not exist. + * + * - `create = true`: some parent file was not a dir, so couldn't + * look/create inside. + */ + File * open(const CanonPath & path, std::optional create); + + CanonPath addFile(CanonPath path, std::string && contents); +}; + +} From 2678b51b31febdc6464935e1680d2272a954c3b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 5 Nov 2023 12:17:33 -0500 Subject: [PATCH 280/402] Narrower scope for `nativeSystem` I don't think we need a CPP defininition and a header entry, and this way allows constant expression elimination. --- src/libstore/build/local-derivation-goal.cc | 2 ++ src/libutil/error.cc | 2 -- src/libutil/util.hh | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index dcb7dc6bc..e1794139f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1620,6 +1620,8 @@ void setupSeccomp() seccomp_release(ctx); }); + constexpr std::string_view nativeSystem = SYSTEM; + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index dd9612471..1badc1069 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -7,8 +7,6 @@ namespace nix { -const std::string nativeSystem = SYSTEM; - void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b302d6f45..75683f8fe 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -34,12 +34,6 @@ struct Source; void initLibUtil(); -/** - * The system for which Nix is compiled. - */ -extern const std::string nativeSystem; - - /** * @return an environment variable. */ From ac89bb064aeea85a62b82a6daf0ecca7190a28b7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Oct 2023 00:43:36 -0400 Subject: [PATCH 281/402] Split up `util.{hh,cc}` All OS and IO operations should be moved out, leaving only some misc portable pure functions. This is useful to avoid copious CPP when doing things like Windows and Emscripten ports. Newly exposed functions to break cycles: - `restoreSignals` - `updateWindowSize` --- perl/lib/Nix/Store.xs | 1 - src/libcmd/built-path.hh | 3 + src/libcmd/common-eval-args.cc | 1 - src/libcmd/editor-for.cc | 2 +- src/libcmd/installable-attr-path.hh | 1 - src/libcmd/installables.cc | 1 + src/libcmd/installables.hh | 1 - src/libcmd/markdown.cc | 1 + src/libcmd/repl.cc | 3 + src/libexpr/attr-path.cc | 1 - src/libexpr/eval-cache.cc | 1 + src/libexpr/eval-settings.cc | 1 + src/libexpr/eval-settings.hh | 2 + src/libexpr/eval.cc | 1 + src/libexpr/flake/config.cc | 3 +- src/libexpr/flake/flake.cc | 1 + src/libexpr/get-drvs.cc | 1 - src/libexpr/parser.y | 1 + src/libexpr/primops.cc | 1 + src/libexpr/search-path.cc | 1 - src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 2 +- src/libexpr/value/context.cc | 1 + src/libexpr/value/context.hh | 1 - src/libfetchers/cache.cc | 1 + src/libfetchers/fetch-settings.hh | 1 - src/libfetchers/git.cc | 3 +- src/libfetchers/input-accessor.hh | 2 + src/libfetchers/mercurial.cc | 2 + src/libfetchers/registry.cc | 2 +- src/libmain/common-args.cc | 2 + src/libmain/loggers.cc | 2 +- src/libmain/progress-bar.cc | 2 +- src/libmain/shared.cc | 3 +- src/libmain/shared.hh | 2 +- src/libstore/binary-cache-store.cc | 1 + src/libstore/build/child.cc | 37 + src/libstore/build/child.hh | 11 + src/libstore/build/hook-instance.cc | 2 + src/libstore/build/hook-instance.hh | 1 + src/libstore/build/local-derivation-goal.cc | 3 + src/libstore/build/local-derivation-goal.hh | 1 + src/libstore/build/worker.cc | 1 + src/libstore/common-protocol.cc | 1 - src/libstore/crypto.cc | 1 + src/libstore/derived-path-map.cc | 1 + src/libstore/derived-path-map.hh | 1 + src/libstore/derived-path.hh | 2 +- src/libstore/filetransfer.cc | 3 +- src/libstore/gc.cc | 7 + src/libstore/globals.cc | 13 +- src/libstore/globals.hh | 2 +- src/libstore/local-store.cc | 1 + src/libstore/local-store.hh | 1 - src/libstore/lock.cc | 1 + src/libstore/machines.cc | 1 - src/libstore/nar-info-disk-cache.cc | 1 + src/libstore/optimise-store.cc | 2 +- src/libstore/path-references.cc | 1 - src/libstore/path-references.hh | 1 + src/libstore/pathlocks.cc | 1 + src/libstore/pathlocks.hh | 2 +- src/libstore/profiles.cc | 2 +- src/libstore/remote-store-connection.hh | 3 + src/libstore/serve-protocol.cc | 1 - src/libstore/sqlite.cc | 1 + src/libstore/ssh.cc | 3 + src/libstore/ssh.hh | 3 +- src/libstore/store-api.cc | 2 + src/libstore/tests/machines.cc | 2 + src/libstore/tests/protocol.hh | 3 + src/libstore/uds-remote-store.cc | 1 + src/libstore/worker-protocol.cc | 1 - src/libutil/archive.cc | 3 +- src/libutil/args.cc | 3 + src/libutil/args.hh | 5 +- src/libutil/canon-path.cc | 2 +- src/libutil/cgroup.cc | 1 + src/libutil/compression.cc | 2 +- src/libutil/config.cc | 2 + src/libutil/current-process.cc | 110 + src/libutil/current-process.hh | 34 + src/libutil/environment-variables.cc | 49 + src/libutil/environment-variables.hh | 41 + src/libutil/error.cc | 3 + src/libutil/file-descriptor.cc | 254 +++ src/libutil/file-descriptor.hh | 84 + src/libutil/file-system.cc | 647 ++++++ src/libutil/file-system.hh | 238 +++ src/libutil/filesystem.cc | 162 -- src/libutil/fs-sink.hh | 1 + src/libutil/hash.cc | 1 - src/libutil/hash.hh | 1 + src/libutil/logging.cc | 3 + src/libutil/monitor-fd.hh | 2 + src/libutil/namespaces.cc | 69 +- src/libutil/namespaces.hh | 23 + src/libutil/posix-source-accessor.cc | 1 + src/libutil/processes.cc | 421 ++++ src/libutil/processes.hh | 123 ++ src/libutil/references.cc | 1 - src/libutil/serialise.cc | 2 +- src/libutil/serialise.hh | 1 + src/libutil/signals.cc | 188 ++ src/libutil/signals.hh | 104 + src/libutil/suggestions.cc | 4 +- src/libutil/tarfile.cc | 1 + src/libutil/terminal.cc | 108 + src/libutil/terminal.hh | 38 + src/libutil/tests/logging.cc | 1 - src/libutil/tests/tests.cc | 3 + src/libutil/thread-pool.cc | 2 + src/libutil/thread-pool.hh | 2 +- src/libutil/unix-domain-socket.cc | 100 + src/libutil/unix-domain-socket.hh | 31 + src/libutil/users.cc | 116 ++ src/libutil/users.hh | 58 + src/libutil/util.cc | 1816 +---------------- src/libutil/util.hh | 617 ------ src/nix-build/nix-build.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- .../nix-collect-garbage.cc | 2 + src/nix-env/nix-env.cc | 2 +- src/nix-env/user-env.cc | 1 - src/nix-instantiate/nix-instantiate.cc | 1 - src/nix-store/dotgraph.cc | 1 - src/nix-store/graphml.cc | 1 - src/nix-store/nix-store.cc | 1 - src/nix/daemon.cc | 3 +- src/nix/develop.cc | 1 - src/nix/doctor.cc | 1 - src/nix/edit.cc | 1 + src/nix/flake.cc | 1 + src/nix/main.cc | 2 + src/nix/run.cc | 1 + src/nix/sigs.cc | 1 + src/nix/upgrade-nix.cc | 1 + src/nix/verify.cc | 1 + 138 files changed, 3028 insertions(+), 2654 deletions(-) create mode 100644 src/libstore/build/child.cc create mode 100644 src/libstore/build/child.hh create mode 100644 src/libutil/current-process.cc create mode 100644 src/libutil/current-process.hh create mode 100644 src/libutil/environment-variables.cc create mode 100644 src/libutil/environment-variables.hh create mode 100644 src/libutil/file-descriptor.cc create mode 100644 src/libutil/file-descriptor.hh create mode 100644 src/libutil/file-system.cc create mode 100644 src/libutil/file-system.hh delete mode 100644 src/libutil/filesystem.cc create mode 100644 src/libutil/processes.cc create mode 100644 src/libutil/processes.hh create mode 100644 src/libutil/signals.cc create mode 100644 src/libutil/signals.hh create mode 100644 src/libutil/terminal.cc create mode 100644 src/libutil/terminal.hh create mode 100644 src/libutil/unix-domain-socket.cc create mode 100644 src/libutil/unix-domain-socket.hh create mode 100644 src/libutil/users.cc create mode 100644 src/libutil/users.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 08f812b31..f89ac4077 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -11,7 +11,6 @@ #include "derivations.hh" #include "globals.hh" #include "store-api.hh" -#include "util.hh" #include "crypto.hh" #include diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index e677bc810..7154cc504 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "derived-path.hh" #include "realisation.hh" diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e53bc4c01..91fa881b1 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -2,7 +2,6 @@ #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" -#include "util.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index a17c6f12a..619d3673f 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,5 @@ -#include "util.hh" #include "editor-for.hh" +#include "environment-variables.hh" namespace nix { diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index e9f0c33da..86c2f8219 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -4,7 +4,6 @@ #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index bc0b8a988..e7f58556f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -4,6 +4,7 @@ #include "installable-attr-path.hh" #include "installable-flake.hh" #include "outputs-spec.hh" +#include "users.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b0dc0dc02..e087f935c 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 668a07763..8b3bbc1b5 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,6 +1,7 @@ #include "markdown.hh" #include "util.hh" #include "finally.hh" +#include "terminal.hh" #include #include diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 2e17a29a7..bf5643a5c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -22,6 +22,7 @@ extern "C" { #include "repl.hh" #include "ansicolor.hh" +#include "signals.hh" #include "shared.hh" #include "eval.hh" #include "eval-cache.hh" @@ -36,6 +37,8 @@ extern "C" { #include "globals.hh" #include "flake/flake.hh" #include "flake/lockfile.hh" +#include "users.hh" +#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index d12345710..7481a2232 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,5 @@ #include "attr-path.hh" #include "eval-inline.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 10fc799a9..6c0e33709 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 93b4a5289..444a7d7d6 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 5473d688e..db2971acb 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -1,4 +1,6 @@ #pragma once +///@file + #include "config.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d26cde423..dfe81cbf7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -14,6 +14,7 @@ #include "print.hh" #include "fs-input-accessor.hh" #include "memory-input-accessor.hh" +#include "signals.hh" #include #include diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index e89014862..3c7ed5d8a 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,6 +1,7 @@ -#include "flake.hh" +#include "users.hh" #include "globals.hh" #include "fetch-settings.hh" +#include "flake.hh" #include diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ded132695..54de53e0b 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,3 +1,4 @@ +#include "terminal.hh" #include "flake.hh" #include "eval.hh" #include "eval-settings.hh" diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index fe3e6f7ee..d4e946d81 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,5 +1,4 @@ #include "get-drvs.hh" -#include "util.hh" #include "eval-inline.hh" #include "derivations.hh" #include "store-api.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 607795937..b86cef217 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -19,6 +19,7 @@ #include #include "util.hh" +#include "users.hh" #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e3c775d90..36340d0f9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "path-references.hh" #include "store-api.hh" #include "util.hh" +#include "processes.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc index 180d5f8b1..a25767496 100644 --- a/src/libexpr/search-path.cc +++ b/src/libexpr/search-path.cc @@ -1,5 +1,4 @@ #include "search-path.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index cbc91f509..74b3ebf13 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,7 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" -#include "util.hh" #include "store-api.hh" +#include "signals.hh" #include #include diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index bd7a4ae30..5032115bb 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -1,7 +1,7 @@ #include "value-to-xml.hh" #include "xml-writer.hh" #include "eval-inline.hh" -#include "util.hh" +#include "signals.hh" #include diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 22361d8fa..6d9633268 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,3 +1,4 @@ +#include "util.hh" #include "value/context.hh" #include diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 9f1d59317..51fd30a44 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "comparator.hh" #include "derived-path.hh" #include "variant-wrapper.hh" diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d..b72a464e8 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "cache.hh" +#include "users.hh" #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 6108a179c..f095963a8 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 72fba0582..cc735996b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,11 +1,12 @@ #include "fetchers.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" -#include "util.hh" +#include "processes.hh" #include "git.hh" #include "fetch-settings.hh" diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5dc05a363..6857ce156 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -1,8 +1,10 @@ #pragma once +///@file #include "source-accessor.hh" #include "ref.hh" #include "types.hh" +#include "file-system.hh" #include "repair-flag.hh" #include "content-address.hh" diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index eda33dfe7..9244acf39 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,4 +1,6 @@ #include "fetchers.hh" +#include "processes.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index a0fff9ceb..9c7bc0cfe 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,6 +1,6 @@ #include "registry.hh" #include "tarball.hh" -#include "util.hh" +#include "users.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 205b77808..5b49aaabc 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,7 +1,9 @@ #include "common-args.hh" #include "args/root.hh" #include "globals.hh" +#include "logging.hh" #include "loggers.hh" +#include "util.hh" namespace nix { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cda5cb939..9829859de 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,6 +1,6 @@ #include "loggers.hh" +#include "environment-variables.hh" #include "progress-bar.hh" -#include "util.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 45b1fdfd1..a7aee47c3 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,5 +1,5 @@ #include "progress-bar.hh" -#include "util.hh" +#include "terminal.hh" #include "sync.hh" #include "store-api.hh" #include "names.hh" diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 9c2ad039a..862ef355b 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,10 +1,11 @@ #include "globals.hh" +#include "current-process.hh" #include "shared.hh" #include "store-api.hh" #include "gc-store.hh" -#include "util.hh" #include "loggers.hh" #include "progress-bar.hh" +#include "signals.hh" #include #include diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 3159fe479..c68f6cd83 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "processes.hh" #include "args.hh" #include "args/root.hh" #include "common-args.hh" diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6a52c4c51..ae483c95e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -11,6 +11,7 @@ #include "nar-accessor.hh" #include "thread-pool.hh" #include "callback.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/build/child.cc b/src/libstore/build/child.cc new file mode 100644 index 000000000..aa31c3caf --- /dev/null +++ b/src/libstore/build/child.cc @@ -0,0 +1,37 @@ +#include "child.hh" +#include "current-process.hh" +#include "logging.hh" + +#include +#include + +namespace nix { + +void commonChildInit() +{ + logger = makeSimpleLogger(); + + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +} diff --git a/src/libstore/build/child.hh b/src/libstore/build/child.hh new file mode 100644 index 000000000..3dfc552b9 --- /dev/null +++ b/src/libstore/build/child.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +namespace nix { + +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +} diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 337c60bd4..5d045ec3d 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,5 +1,7 @@ #include "globals.hh" #include "hook-instance.hh" +#include "file-system.hh" +#include "child.hh" namespace nix { diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index d84f62877..61cf534f4 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -3,6 +3,7 @@ #include "logging.hh" #include "serialise.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e1794139f..adb011e30 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,7 +15,10 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "current-process.hh" #include "namespaces.hh" +#include "child.hh" +#include "unix-domain-socket.hh" #include #include diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 1cb68a869..88152a645 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -3,6 +3,7 @@ #include "derivation-goal.hh" #include "local-store.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 37cb86b91..01914e2d6 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -4,6 +4,7 @@ #include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" +#include "signals.hh" #include diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index f906814bc..68445258f 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1027469c9..1b705733c 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,4 +1,5 @@ #include "crypto.hh" +#include "file-system.hh" #include "util.hh" #include "globals.hh" diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 5982c04b3..4c1ea417a 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -1,4 +1,5 @@ #include "derived-path-map.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 4d72b301e..393cdedf7 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "derived-path.hh" diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2..6c5dfeed9 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "comparator.hh" +#include "config.hh" #include diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a283af5a2..dcbec4acd 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,11 +1,12 @@ #include "filetransfer.hh" -#include "util.hh" +#include "namespaces.hh" #include "globals.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" #include "finally.hh" #include "callback.hh" +#include "signals.hh" #if ENABLE_S3 #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index fb7895817..8d05ae4bd 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -2,6 +2,13 @@ #include "globals.hh" #include "local-store.hh" #include "finally.hh" +#include "unix-domain-socket.hh" +#include "signals.hh" + +#if !defined(__linux__) +// For shelling out to lsof +# include "processes.hh" +#endif #include #include diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9c25d9868..cc416a4d6 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,7 +1,8 @@ #include "globals.hh" -#include "util.hh" +#include "current-process.hh" #include "archive.hh" #include "args.hh" +#include "users.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" @@ -17,9 +18,13 @@ #include #ifdef __GLIBC__ -#include -#include -#include +# include +# include +# include +#endif + +#if __APPLE__ +# include "processes.hh" #endif #include "config-impl.hh" diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 12fb48d93..8e034f5a9 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -3,7 +3,7 @@ #include "types.hh" #include "config.hh" -#include "util.hh" +#include "environment-variables.hh" #include "experimental-features.hh" #include diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a5e9426f8..2a3582ad8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,6 +10,7 @@ #include "topo-sort.hh" #include "finally.hh" #include "compression.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fe26a0f27..6d589bee5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -7,7 +7,6 @@ #include "store-api.hh" #include "indirect-root-store.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 165e4969f..87f55ce49 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index e87f46980..512115893 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,5 +1,4 @@ #include "machines.hh" -#include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index cdbcf7e74..e50c15939 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,4 +1,5 @@ #include "nar-info-disk-cache.hh" +#include "users.hh" #include "sync.hh" #include "sqlite.hh" #include "globals.hh" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 23c6a41e4..a4ac413b3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,6 @@ -#include "util.hh" #include "local-store.hh" #include "globals.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 33cf66ce3..274b596c0 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,6 +1,5 @@ #include "path-references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh index 7b44e3261..0553003f8 100644 --- a/src/libstore/path-references.hh +++ b/src/libstore/path-references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "references.hh" #include "path.hh" diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index adc763e6a..2b5b8dfe7 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,6 +1,7 @@ #include "pathlocks.hh" #include "util.hh" #include "sync.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 4921df352..7fcfa2e40 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 239047dd6..e8b88693d 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,7 +1,7 @@ #include "profiles.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" +#include "users.hh" #include #include diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index e4a9cacb9..44328b06b 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "remote-store.hh" #include "worker-protocol.hh" #include "pool.hh" diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 97a0ddf0e..9bfcc279c 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 7c8decb74..d7432a305 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "util.hh" #include "url.hh" +#include "signals.hh" #include diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index da32f1b79..03b2f0be9 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,5 +1,8 @@ #include "ssh.hh" #include "finally.hh" +#include "current-process.hh" +#include "environment-variables.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 94b952af9..bfcd6f21c 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "util.hh" #include "sync.hh" +#include "processes.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 646b0ec7d..c9ebb6c14 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -14,6 +14,8 @@ // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" +#include "signals.hh" +#include "users.hh" #include #include diff --git a/src/libstore/tests/machines.cc b/src/libstore/tests/machines.cc index f51052b14..fede328ea 100644 --- a/src/libstore/tests/machines.cc +++ b/src/libstore/tests/machines.cc @@ -1,5 +1,7 @@ #include "machines.hh" #include "globals.hh" +#include "file-system.hh" +#include "util.hh" #include diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 0378b3e1f..466032a79 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 99589f8b2..226cdf717 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "unix-domain-socket.hh" #include "worker-protocol.hh" #include diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index d618b9bd8..1d202f8d1 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 4ca84d357..465df2073 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,9 +6,10 @@ #include // for strcasecmp #include "archive.hh" -#include "util.hh" #include "config.hh" #include "posix-source-accessor.hh" +#include "file-system.hh" +#include "signals.hh" namespace nix { diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 811353c18..0b65519a3 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,6 +1,9 @@ #include "args.hh" #include "args/root.hh" #include "hash.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "users.hh" #include "json-utils.hh" #include diff --git a/src/libutil/args.hh b/src/libutil/args.hh index e3b41313f..45fd678e7 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -2,12 +2,15 @@ ///@file #include +#include #include #include +#include #include -#include "util.hh" +#include "types.hh" +#include "experimental-features.hh" namespace nix { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 040464532..f678fae94 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,5 +1,5 @@ #include "canon-path.hh" -#include "util.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index a008481ca..4c2bf31ff 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "file-system.hh" #include "finally.hh" #include diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index ba0847cde..d06f1f87b 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,6 @@ #include "compression.hh" +#include "signals.hh" #include "tarfile.hh" -#include "util.hh" #include "finally.hh" #include "logging.hh" diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e06273ee..0bf36c987 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -2,6 +2,8 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "util.hh" +#include "file-system.hh" #include "config-impl.hh" diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc new file mode 100644 index 000000000..352a6a0fb --- /dev/null +++ b/src/libutil/current-process.cc @@ -0,0 +1,110 @@ +#include "current-process.hh" +#include "namespaces.hh" +#include "util.hh" +#include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#ifdef __APPLE__ +# include +#endif + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif + +#include + +namespace nix { + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} + + +////////////////////////////////////////////////////////////////////// + + +#if __linux__ +rlim_t savedStackSize = 0; +#endif + +void setStackSize(size_t stackSize) +{ + #if __linux__ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif +} + +void restoreProcessContext(bool restoreMounts) +{ + restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } + + #if __linux__ + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } + #endif +} + + +////////////////////////////////////////////////////////////////////// + + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + +} diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh new file mode 100644 index 000000000..826d6fe20 --- /dev/null +++ b/src/libutil/current-process.hh @@ -0,0 +1,34 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + +/** + * Change the stack size. + */ +void setStackSize(size_t stackSize); + +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ +void restoreProcessContext(bool restoreMounts = true); + +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +} diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc new file mode 100644 index 000000000..6618d7872 --- /dev/null +++ b/src/libutil/environment-variables.cc @@ -0,0 +1,49 @@ +#include "util.hh" +#include "environment-variables.hh" + +extern char * * environ __attribute__((weak)); + +namespace nix { + +std::optional getEnv(const std::string & key) +{ + char * value = getenv(key.c_str()); + if (!value) return {}; + return std::string(value); +} + +std::optional getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(const std::map & newEnv) +{ + clearEnv(); + for (auto & newEnvVar : newEnv) + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); +} + +} diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh new file mode 100644 index 000000000..21eb4619b --- /dev/null +++ b/src/libutil/environment-variables.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Utilities for working with the current process's environment + * variables. + */ + +#include + +#include "types.hh" + +namespace nix { + +/** + * @return an environment variable. + */ +std::optional getEnv(const std::string & key); + +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ +std::map getEnv(); + +/** + * Clear the environment. + */ +void clearEnv(); + +/** + * Replace the entire environment with the given one. + */ +void replaceEnv(const std::map & newEnv); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 1badc1069..8488e7e21 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,4 +1,7 @@ #include "error.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 000000000..38dd70c8e --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,254 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +std::string drainFD(int fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); + drainFD(fd, sink, block); + return std::move(sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({(char *) buf.data(), (size_t) rd}); + } +} + +////////////////////////////////////////////////////////////////////// + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError("closing file descriptor %1%", fd); + fd = -1; + } +} + +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + +////////////////////////////////////////////////////////////////////// + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 000000000..80ec86135 --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,84 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" + +namespace nix { + +struct Sink; +struct Source; + +/** + * Read the contents of a resource into a string. + */ +std::string readFile(int fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(int fd, char * buf, size_t count); + +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a line from a file descriptor. + */ +std::string readLine(int fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(int fd, std::string s); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); + +void drainFD(int fd, Sink & sink, bool block = true); + +/** + * Automatic cleanup of resources. + */ +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd); + int get() const; + explicit operator bool() const; + int release(); + void close(); + void fsync(); +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(int fd); + +MakeError(EndOfFile, Error); + +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc new file mode 100644 index 000000000..c96effff9 --- /dev/null +++ b/src/libutil/file-system.cc @@ -0,0 +1,647 @@ +#include "environment-variables.hh" +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace nix { + +Path absPath(Path path, std::optional dir, bool resolveSymlinks) +{ + if (path[0] != '/') { + if (!dir) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + path = concatStrings(buf, "/", path); +#ifdef __GNU__ + free(buf); +#endif + } else + path = concatStrings(*dir, "/", path); + } + return canonPath(path, resolveSymlinks); +} + + +Path canonPath(PathView path, bool resolveSymlinks) +{ + assert(path != ""); + + std::string s; + s.reserve(256); + + if (path[0] != '/') + throw Error("not an absolute path: '%1%'", path); + + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (!path.empty() && path[0] == '/') path.remove_prefix(1); + if (path.empty()) break; + + /* Ignore `.'. */ + if (path == "." || path.substr(0, 2) == "./") + path.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (path == ".." || path.substr(0, 3) == "../") + { + if (!s.empty()) s.erase(s.rfind('/')); + path.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + s += '/'; + if (const auto slash = path.find('/'); slash == std::string::npos) { + s += path; + path = {}; + } else { + s += path.substr(0, slash); + path = path.substr(slash); + } + + /* If s points to a symlink, resolve it and continue from there */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error("infinite symlink recursion in path '%1%'", path); + temp = concatStrings(readLink(s), path); + path = temp; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } + } + } + } + } + + return s.empty() ? "/" : std::move(s); +} + + +Path dirOf(const PathView path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == std::string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +std::string_view baseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == std::string::npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + + +bool isInDir(std::string_view path, std::string_view dir) +{ + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(std::string_view path, std::string_view dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError("getting status of %1%", path); + return false; +} + +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return std::string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(DIR *dir, const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError("reading directory '%1%'", path); + + return entries; +} + +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +std::string readFile(const Path & path) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + + +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + + std::vector buf(64 * 1024); + + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), {buf.data(), n}); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); +} + + +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) +{ + checkInterrupt(); + + std::string name(baseNameOf(path)); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) return; + throw SysError("getting status of '%1%'", path); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +} + +static void _deletePath(const Path & path, uint64_t & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; + if (!dirfd) { + if (errno == ENOENT) return; + throw SysError("opening directory '%1%'", path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + + +void deletePath(const Path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError("creating directory '%1%'", path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError("statting symlink '%1%'", path); + + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); + + return created; +} + + +void deletePath(const Path & path, uint64_t & bytesFreed) +{ + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError("cannot unlink '%1%'", path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +////////////////////////////////////////////////////////////////////// + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); +} + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); + } + } +} + +////////////////////////////////////////////////////////////////////// + +} diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh new file mode 100644 index 000000000..4637507b3 --- /dev/null +++ b/src/libutil/file-system.hh @@ -0,0 +1,238 @@ +#pragma once +/** + * @file + * + * Utiltities for working with the file sytem and file paths. + */ + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "file-descriptor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ +Path absPath(Path path, + std::optional dir = {}, + bool resolveSymlinks = false); + +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ +Path canonPath(PathView path, bool resolveSymlinks = false); + +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ +Path dirOf(const PathView path); + +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ +std::string_view baseNameOf(std::string_view path); + +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ +bool isInDir(std::string_view path, std::string_view dir); + +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ +bool isDirOrInDir(std::string_view path, std::string_view dir); + +/** + * Get status of `path`. + */ +struct stat stat(const Path & path); +struct stat lstat(const Path & path); + +/** + * @return true iff the given path exists. + */ +bool pathExists(const Path & path); + +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ +struct DirEntry +{ + std::string name; + ino_t ino; + /** + * one of DT_* + */ + unsigned char type; + DirEntry(std::string name, ino_t ino, unsigned char type) + : name(std::move(name)), ino(ino), type(type) { } +}; + +typedef std::vector DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); + +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a file's parent directory to disk + */ +void syncParent(const Path & path); + +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ +void deletePath(const Path & path); + +void deletePath(const Path & path, uint64_t & bytesFreed); + +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ +Paths createDirs(const Path & path); +inline Paths createDirs(PathView path) +{ + return createDirs(Path(path)); +} + +/** + * Create a symlink. + */ +void createSymlink(const Path & target, const Path & link); + +/** + * Atomically create or replace a symlink. + */ +void replaceSymlink(const Path & target, const Path & link); + +void renameFile(const Path & src, const Path & dst); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + + +/** + * Automatic cleanup of resources. + */ +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } + operator PathView() const { return path; } +}; + + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + + +/** + * Create a temporary directory. + */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/** + * Create a temporary file, returning a file handle and its path. + */ +std::pair createTempFile(const Path & prefix = "nix"); + + +/** + * Used in various places. + */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc deleted file mode 100644 index 11cc0c0e7..000000000 --- a/src/libutil/filesystem.cc +++ /dev/null @@ -1,162 +0,0 @@ -#include -#include -#include - -#include "finally.hh" -#include "util.hh" -#include "types.hh" - -namespace fs = std::filesystem; - -namespace nix { - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - std::atomic & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); - else - return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); -} - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static std::atomic globalCounter = 0; - std::atomic localCounter = 0; - auto & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); -} - -void replaceSymlink(const Path & target, const Path & link) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - renameFile(tmp, link); - - break; - } -} - -void setWriteTime(const fs::path & p, const struct stat & st) -{ - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, - }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} - -void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) -{ - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.path().c_str()); - auto fromStatus = from.symlink_status(); - - // Mark the directory as writable so that we can delete its children - if (andDelete && fs::is_directory(fromStatus)) { - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - } - - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from.path())) { - copy(entry, to / entry.path().filename(), andDelete); - } - } else { - throw Error("file '%s' has an unsupported type", from.path()); - } - - setWriteTime(to, statOfFrom); - if (andDelete) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from.path()); - } -} - -void renameFile(const Path & oldName, const Path & newName) -{ - fs::rename(oldName, newName); -} - -void moveFile(const Path & oldName, const Path & newName) -{ - try { - renameFile(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); - // For the move to be as atomic as possible, copy to a temporary - // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; - auto tempCopyTarget = temp / "copy-target"; - if (e.code().value() == EXDEV) { - fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), tempCopyTarget, true); - renameFile(tempCopyTarget, newPath); - } - } -} - -} diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index c22edd390..bf54b7301 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "serialise.hh" #include "source-accessor.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e297c245b..144f7ae7e 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -9,7 +9,6 @@ #include "hash.hh" #include "archive.hh" #include "split.hh" -#include "util.hh" #include #include diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index cab3e6eca..6ade6555c 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 9d7a141b3..60b0865bf 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,4 +1,7 @@ #include "logging.hh" +#include "file-descriptor.hh" +#include "environment-variables.hh" +#include "terminal.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 86d0115fc..228fb13f8 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -10,6 +10,8 @@ #include #include +#include "signals.hh" + namespace nix { diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index f66accb10..a789b321e 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,13 +1,22 @@ -#if __linux__ - -#include "namespaces.hh" +#include "current-process.hh" #include "util.hh" #include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif #include namespace nix { +#if __linux__ + bool userNamespacesSupported() { static auto res = [&]() -> bool @@ -92,6 +101,60 @@ bool mountAndPidNamespacesSupported() return res; } +#endif + + +////////////////////////////////////////////////////////////////////// + +#if __linux__ +static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; +#endif + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); + }); +#endif } +void restoreMountNamespace() +{ +#if __linux__ + try { + auto savedCwd = absPath("."); + + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); + } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); + } catch (Error & e) { + debug(e.msg()); + } #endif +} + +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); +#endif +} + +} diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 0b7eeb66c..7e4e921a8 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -1,8 +1,31 @@ #pragma once ///@file +#include + +#include "types.hh" + namespace nix { +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ +void saveMountNamespace(); + +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ +void restoreMountNamespace(); + +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ +void unshareFilesystem(); + #if __linux__ bool userNamespacesSupported(); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index d5e32d989..dc96f84e5 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,4 +1,5 @@ #include "posix-source-accessor.hh" +#include "signals.hh" namespace nix { diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc new file mode 100644 index 000000000..91a0ea66f --- /dev/null +++ b/src/libutil/processes.cc @@ -0,0 +1,421 @@ +#include "current-process.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "processes.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug("killing process %1%", pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get exit status of PID %d", pid); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug("killing all processes running under uid '%1%'", uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ + if (errno != EINTR) + throw SysError("cannot kill processes for uid '%1%'", uid); + } + + _exit(0); + }); + + int status = pid.wait(); + if (status != 0) + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function fun) +{ +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else + pid_t pid = fork(); +#endif + if (pid != 0) return pid; + fun(); + abort(); +} + + +#if __linux__ +static int childEntry(void * arg) +{ + auto main = (std::function *) arg; + (*main)(); + return 1; +} +#endif + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + std::function wrapper = [&]() { + if (!options.allowVfork) + logger = makeSimpleLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + + +std::string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + processOptions.allowVfork = !options.environment; + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreProcessContext(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide.close(); + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +////////////////////////////////////////////////////////////////////// + +std::string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return fmt("failed with exit code %1%", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return fmt("failed due to signal %1% (%2%)", sig, description); +#else + return fmt("failed due to signal %1%", sig); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +} diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh new file mode 100644 index 000000000..978c37105 --- /dev/null +++ b/src/libutil/processes.hh @@ -0,0 +1,123 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +struct Sink; +struct Source; + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + + +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ +void killUser(uid_t uid); + + +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ +struct ProcessOptions +{ + std::string errorPrefix = ""; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; +}; + +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); + + +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ +std::string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional & input = {}, bool isInteractive = false); + +struct RunOptions +{ + Path program; + bool searchPath = true; + Strings args; + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool isInteractive = false; +}; + +std::pair runProgram(RunOptions && options); + +void runProgram2(const RunOptions & options); + + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, const Args & ... args) + : Error(args...), status(status) + { } +}; + + +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ +std::string statusToString(int status); + +bool statusOk(int status); + +} diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 7f59b4c09..9d75606ef 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -1,6 +1,5 @@ #include "references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 3d5121a19..725ddbb8d 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,5 @@ #include "serialise.hh" -#include "util.hh" +#include "signals.hh" #include #include diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 333c254ea..9e07226bf 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc new file mode 100644 index 000000000..4632aa319 --- /dev/null +++ b/src/libutil/signals.cc @@ -0,0 +1,188 @@ +#include "signals.hh" +#include "util.hh" +#include "error.hh" +#include "sync.hh" +#include "terminal.hh" + +#include + +namespace nix { + +std::atomic _isInterrupted = false; + +static thread_local bool interruptThrown = false; +thread_local std::function interruptCheck; + +void setInterruptThrown() +{ + interruptThrown = true; +} + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exceptions()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + + +////////////////////////////////////////////////////////////////////// + + +/* We keep track of interrupt callbacks using integer tokens, so we can iterate + safely without having to lock the data structure while executing arbitrary + functions. + */ +struct InterruptCallbacks { + typedef int64_t Token; + + /* We use unique tokens so that we can't accidentally delete the wrong + handler because of an erroneous double delete. */ + Token nextToken = 0; + + /* Used as a list, see InterruptCallbacks comment. */ + std::map> callbacks; +}; + +static Sync _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() +{ + _isInterrupted = true; + + { + InterruptCallbacks::Token i = 0; + while (true) { + std::function callback; + { + auto interruptCallbacks(_interruptCallbacks.lock()); + auto lb = interruptCallbacks->callbacks.lower_bound(i); + if (lb == interruptCallbacks->callbacks.end()) + break; + + callback = lb->second; + i = lb->first + 1; + } + + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + + +static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} + +void startSignalHandlerThread() +{ + updateWindowSize(); + + saveSignalMask(); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +void restoreSignals() +{ + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + InterruptCallbacks::Token token; + ~InterruptCallbackImpl() override + { + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->callbacks.erase(token); + } +}; + +std::unique_ptr createInterruptCallback(std::function callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + auto token = interruptCallbacks->nextToken++; + interruptCallbacks->callbacks.emplace(token, callback); + + auto res = std::make_unique(); + res->token = token; + + return std::unique_ptr(res.release()); +} + +} diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh new file mode 100644 index 000000000..7e8beff33 --- /dev/null +++ b/src/libutil/signals.hh @@ -0,0 +1,104 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +/* User interruption. */ + +extern std::atomic _isInterrupted; + +extern thread_local std::function interruptCheck; + +void setInterruptThrown(); + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +MakeError(Interrupted, BaseError); + + +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ +void startSignalHandlerThread(); + +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * To use in a process that already called `startSignalHandlerThread()` + * or `saveSignalMask()` first. + */ +void restoreSignals(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ +std::unique_ptr createInterruptCallback( + std::function callback); + +void triggerInterrupt(); + +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 9510a5f0c..e67e986fb 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -1,7 +1,9 @@ #include "suggestions.hh" #include "ansicolor.hh" -#include "util.hh" +#include "terminal.hh" + #include +#include namespace nix { diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 5060a8f24..1733c791c 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -3,6 +3,7 @@ #include "serialise.hh" #include "tarfile.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc new file mode 100644 index 000000000..8febc8771 --- /dev/null +++ b/src/libutil/terminal.cc @@ -0,0 +1,108 @@ +#include "terminal.hh" +#include "environment-variables.hh" +#include "sync.hh" + +#include +#include + +namespace nix { + +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); +} + +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r' || *i == '\a') + // do nothing for now + i++; + + else { + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; + } + } + + return t; +} + + +////////////////////////////////////////////////////////////////////// + +static Sync> windowSize{{0, 0}}; + + +void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + +} diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh new file mode 100644 index 000000000..9cb191308 --- /dev/null +++ b/src/libutil/terminal.hh @@ -0,0 +1,38 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ +bool shouldANSI(); + +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + +/** + * Recalculate the window size, updating a global variable. Used in the + * `SIGWINCH` signal handler. + */ +void updateWindowSize(); + +/** + * @return the number of rows and columns of the terminal. + * + * The value is cached so this is quick. The cached result is computed + * by `updateWindowSize()`. + */ +std::pair getWindowSize(); + +} diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 2ffdc2e9b..c6dfe63d3 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -2,7 +2,6 @@ #include "logging.hh" #include "nixexpr.hh" -#include "util.hh" #include #include diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index f3c1e8248..568f03f70 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -1,5 +1,8 @@ #include "util.hh" #include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index dc4067f1b..c5e735617 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,4 +1,6 @@ #include "thread-pool.hh" +#include "signals.hh" +#include "util.hh" namespace nix { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 0e09fae97..02765badc 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,8 +1,8 @@ #pragma once ///@file +#include "error.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc new file mode 100644 index 000000000..8949461d2 --- /dev/null +++ b/src/libutil/unix-domain-socket.cc @@ -0,0 +1,100 @@ +#include "file-system.hh" +#include "processes.hh" +#include "unix-domain-socket.hh" + +#include +#include +#include + +namespace nix { + +AutoCloseFD createUnixDomainSocket() +{ + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); + + bind(fdSocket.get(), path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(fdSocket.get(), 100) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + + +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot bind to socket '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + } +} + + +void connect(int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot connect to socket at '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + } +} + +} diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh new file mode 100644 index 000000000..b78feb454 --- /dev/null +++ b/src/libutil/unix-domain-socket.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include "types.hh" +#include "file-descriptor.hh" + +#include + +namespace nix { + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + +/** + * Bind a Unix domain socket to a path. + */ +void bind(int fd, const std::string & path); + +/** + * Connect to a Unix domain socket. + */ +void connect(int fd, const std::string & path); + +} diff --git a/src/libutil/users.cc b/src/libutil/users.cc new file mode 100644 index 000000000..95a641322 --- /dev/null +++ b/src/libutil/users.cc @@ -0,0 +1,116 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +#include +#include +#include + +namespace nix { + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; +} + +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + + +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh new file mode 100644 index 000000000..cecbb8bfb --- /dev/null +++ b/src/libutil/users.hh @@ -0,0 +1,58 @@ +#pragma once +///@file + +#include "types.hh" + +#include + +namespace nix { + +std::string getUserName(); + +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ +Path getHome(); + +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ +Path getCacheDir(); + +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ +Path getConfigDir(); + +/** + * @return the directories to search for user configuration files + */ +std::vector getConfigDirs(); + +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ +Path getDataDir(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ +Path getStateDir(); + +/** + * Create the Nix state directory and return the path to it. + */ +Path createNixStateDir(); + +/** + * Perform tilde expansion on a path, replacing tilde with the user's + * home directory. + */ +std::string expandTilde(std::string_view path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3b4c181e5..ee7a22849 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,49 +1,10 @@ #include "util.hh" -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" -#include "cgroup.hh" +#include "fmt.hh" #include #include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include - -#include #include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#endif - -#ifdef __linux__ -#include -#include -#include - -#include -#endif - - -extern char * * environ __attribute__((weak)); - namespace nix { @@ -67,1099 +28,8 @@ void initLibUtil() { assert(caught); } -std::optional getEnv(const std::string & key) -{ - char * value = getenv(key.c_str()); - if (!value) return {}; - return std::string(value); -} - -std::optional getEnvNonEmpty(const std::string & key) { - auto value = getEnv(key); - if (value == "") return {}; - return value; -} - -std::map getEnv() -{ - std::map env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; -} - - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); -} - -void replaceEnv(const std::map & newEnv) -{ - clearEnv(); - for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); -} - - -Path absPath(Path path, std::optional dir, bool resolveSymlinks) -{ - if (path[0] != '/') { - if (!dir) { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); -#ifdef __GNU__ - free(buf); -#endif - } else - path = concatStrings(*dir, "/", path); - } - return canonPath(path, resolveSymlinks); -} - - -Path canonPath(PathView path, bool resolveSymlinks) -{ - assert(path != ""); - - std::string s; - s.reserve(256); - - if (path[0] != '/') - throw Error("not an absolute path: '%1%'", path); - - std::string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ - } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); - } - } - } - } - } - - return s.empty() ? "/" : std::move(s); -} - - -Path dirOf(const PathView path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -std::string_view baseNameOf(std::string_view path) -{ - if (path.empty()) - return ""; - - auto last = path.size() - 1; - if (path[last] == '/' && last > 0) - last -= 1; - - auto pos = path.rfind('/', last); - if (pos == std::string::npos) - pos = 0; - else - pos += 1; - - return path.substr(pos, last - pos + 1); -} - - -std::string expandTilde(std::string_view path) -{ - // TODO: expand ~user ? - auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome() + std::string(path.substr(1)); - else - return std::string(path); -} - - -bool isInDir(std::string_view path, std::string_view dir) -{ - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -bool isDirOrInDir(std::string_view path, std::string_view dir) -{ - return path == dir || isInDir(path, dir); -} - - -struct stat stat(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -bool pathExists(const Path & path) -{ - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError("getting status of %1%", path); - return false; -} - -bool pathAccessible(const Path & path) -{ - try { - return pathExists(path); - } catch (SysError & e) { - // swallow EPERM - if (e.errNo == EPERM) return false; - throw; - } -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return std::string(buf.data(), rlSize); - } -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -DirEntries readDirectory(DIR *dir, const Path & path) -{ - DirEntries entries; - entries.reserve(64); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, -#ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type -#else - DT_UNKNOWN -#endif - ); - } - if (errno) throw SysError("reading directory '%1%'", path); - - return entries; -} - -DirEntries readDirectory(const Path & path) -{ - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError("opening directory '%1%'", path); - - return readDirectory(dir.get(), path); -} - - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; -} - - -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -std::string readFile(const Path & path) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - return readFile(fd.get()); -} - - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); -} - - -void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - try { - writeFull(fd.get(), s); - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - - -void writeFile(const Path & path, Source & source, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - - std::vector buf(64 * 1024); - - try { - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {buf.data(), n}); - } catch (EndOfFile &) { break; } - } - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - -void syncParent(const Path & path) -{ - AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); -} - -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) -{ - checkInterrupt(); - - std::string name(baseNameOf(path)); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); - } - - int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError("opening directory '%1%'", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory '%1%'", path); - for (auto & i : readDirectory(dir.get(), path)) - _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); - } -} - -static void _deletePath(const Path & path, uint64_t & bytesFreed) -{ - Path dir = dirOf(path); - if (dir == "") - dir = "/"; - - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; - if (!dirfd) { - if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); - } - - _deletePath(dirfd.get(), path, bytesFreed); -} - - -void deletePath(const Path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, uint64_t & bytesFreed) -{ - //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - -Path getHomeOf(uid_t userId) -{ - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - return pw->pw_dir; -} - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - homeDir = getHomeOf(geteuid()); - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - -Path getCacheDir() -{ - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; -} - - -Path getConfigDir() -{ - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; -} - -std::vector getConfigDirs() -{ - Path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - std::vector result = tokenizeString>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; -} - - -Path getDataDir() -{ - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; -} - -Path getStateDir() -{ - auto stateDir = getEnv("XDG_STATE_HOME"); - return stateDir ? *stateDir : getHome() + "/.local/state"; -} - -Path createNixStateDir() -{ - Path dir = getStateDir() + "/nix"; - createDirs(dir); - return dir; -} - - -std::optional getSelfExe() -{ - static auto cached = []() -> std::optional - { - #if __linux__ - return readLink("/proc/self/exe"); - #elif __APPLE__ - char buf[1024]; - uint32_t size = sizeof(buf); - if (_NSGetExecutablePath(buf, &size) == 0) - return buf; - else - return std::nullopt; - #else - return std::nullopt; - #endif - }(); - return cached; -} - - -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = lstat(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; -} - - -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } -} - ////////////////////////////////////////////////////////////////////// -unsigned int getMaxCPU() -{ - #if __linux__ - try { - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } - #endif - - return 0; -} - -////////////////////////////////////////////////////////////////////// - - -AutoDelete::AutoDelete() : del{false} {} - -AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError("cannot unlink '%1%'", path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = fds[0]; - writeSide = fds[1]; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} - - -////////////////////////////////////////////////////////////////////// - - -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() -{ - if (pid != -1) kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -int Pid::kill() -{ - assert(pid != -1); - - debug("killing process %1%", pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ -#if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) -#endif - logError(SysError("killing process %d", pid).info()); - } - - return wait(); -} - - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; -} - - -void killUser(uid_t uid) -{ - debug("killing all processes running under uid '%1%'", uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH || errno == EPERM) break; /* no more processes */ - if (errno != EINTR) - throw SysError("cannot kill processes for uid '%1%'", uid); - } - - _exit(0); - }); - - int status = pid.wait(); - if (status != 0) - throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); - - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ -} - - -////////////////////////////////////////////////////////////////////// - - -/* Wrapper around vfork to prevent the child process from clobbering - the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function fun) -{ -#ifdef __linux__ - pid_t pid = allowVfork ? vfork() : fork(); -#else - pid_t pid = fork(); -#endif - if (pid != 0) return pid; - fun(); - abort(); -} - - -#if __linux__ -static int childEntry(void * arg) -{ - auto main = (std::function *) arg; - (*main)(); - return 1; -} -#endif - - -pid_t startProcess(std::function fun, const ProcessOptions & options) -{ - std::function wrapper = [&]() { - if (!options.allowVfork) - logger = makeSimpleLogger(); - try { -#if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); -#endif - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; - - pid_t pid = -1; - - if (options.cloneFlags) { - #ifdef __linux__ - // Not supported, since then we don't know when to free the stack. - assert(!(options.cloneFlags & CLONE_VM)); - - size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - Finally freeStack([&]() { munmap(stack, stackSize); }); - - pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); - #else - throw Error("clone flags are only supported on Linux"); - #endif - } else - pid = doFork(options.allowVfork, wrapper); - - if (pid == -1) throw SysError("unable to fork"); - - return pid; -} - - std::vector stringsToCharPtrs(const Strings & ss) { std::vector res; @@ -1168,211 +38,6 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input, bool isInteractive) -{ - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); - - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; -} - -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); - - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - // vfork implies that the environment of the main process and the fork will - // be shared (technically this is undefined, but in practice that's the - // case), so we can't use it if we alter the environment - processOptions.allowVfork = !options.environment; - - std::optional>> resumeLoggerDefer; - if (options.isInteractive) { - logger->pause(); - resumeLoggerDefer.emplace( - []() { - logger->resume(); - } - ); - } - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program); - - restoreProcessContext(); - - if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing '%1%'", options.program); - }, processOptions); - - out.writeSide.close(); - - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); - - - if (source) { - in.readSide.close(); - writerThread = std::thread([&]() { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); - - /* Wait for the child to finish. */ - int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); - - if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); -} - - -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - -////////////////////////////////////////////////////////////////////// - - -std::atomic _isInterrupted = false; - -static thread_local bool interruptThrown = false; -thread_local std::function interruptCheck; - -void setInterruptThrown() -{ - interruptThrown = true; -} - -void _interrupted() -{ - /* Block user interrupts while an exception is being handled. - Throwing an exception while another exception is being handled - kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { - interruptThrown = true; - throw Interrupted("interrupted by the user"); - } -} - ////////////////////////////////////////////////////////////////////// @@ -1438,32 +103,6 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites) } -std::string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return fmt("failed with exit code %1%", WEXITSTATUS(status)); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return fmt("failed due to signal %1% (%2%)", sig, description); -#else - return fmt("failed due to signal %1%", sig); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; -} - - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; -} - - bool hasPrefix(std::string_view s, std::string_view prefix) { return s.compare(0, prefix.size(), prefix) == 0; @@ -1511,82 +150,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -bool shouldANSI() -{ - return isatty(STDERR_FILENO) - && getEnv("TERM").value_or("dumb") != "dumb" - && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); -} - -std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } - - else if (*i == '\r' || *i == '\a') - // do nothing for now - i++; - - else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; - } - } - - return t; -} - constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -1703,386 +266,9 @@ std::pair getLine(std::string_view s) } -////////////////////////////////////////////////////////////////////// - -static Sync> windowSize{{0, 0}}; - - -static void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } -} - - -std::pair getWindowSize() -{ - return *windowSize.lock(); -} - - -/* We keep track of interrupt callbacks using integer tokens, so we can iterate - safely without having to lock the data structure while executing arbitrary - functions. - */ -struct InterruptCallbacks { - typedef int64_t Token; - - /* We use unique tokens so that we can't accidentally delete the wrong - handler because of an erroneous double delete. */ - Token nextToken = 0; - - /* Used as a list, see InterruptCallbacks comment. */ - std::map> callbacks; -}; - -static Sync _interruptCallbacks; - -static void signalHandlerThread(sigset_t set) -{ - while (true) { - int signal = 0; - sigwait(&set, &signal); - - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) - triggerInterrupt(); - - else if (signal == SIGWINCH) { - updateWindowSize(); - } - } -} - -void triggerInterrupt() -{ - _isInterrupted = true; - - { - InterruptCallbacks::Token i = 0; - while (true) { - std::function callback; - { - auto interruptCallbacks(_interruptCallbacks.lock()); - auto lb = interruptCallbacks->callbacks.lower_bound(i); - if (lb == interruptCallbacks->callbacks.end()) - break; - - callback = lb->second; - i = lb->first + 1; - } - - try { - callback(); - } catch (...) { - ignoreException(); - } - } - } -} - -static sigset_t savedSignalMask; -static bool savedSignalMaskIsSet = false; - -void setChildSignalMask(sigset_t * sigs) -{ - assert(sigs); // C style function, but think of sigs as a reference - -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE - sigemptyset(&savedSignalMask); - // There's no "assign" or "copy" function, so we rely on (math) idempotence - // of the or operator: a or a = a. - sigorset(&savedSignalMask, sigs, sigs); -#else - // Without sigorset, our best bet is to assume that sigset_t is a type that - // can be assigned directly, such as is the case for a sigset_t defined as - // an integer type. - savedSignalMask = *sigs; -#endif - - savedSignalMaskIsSet = true; -} - -void saveSignalMask() { - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); - - savedSignalMaskIsSet = true; -} - -void startSignalHandlerThread() -{ - updateWindowSize(); - - saveSignalMask(); - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGINT); - sigaddset(&set, SIGTERM); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGPIPE); - sigaddset(&set, SIGWINCH); - if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) - throw SysError("blocking signals"); - - std::thread(signalHandlerThread, set).detach(); -} - -static void restoreSignals() -{ - // If startSignalHandlerThread wasn't called, that means we're not running - // in a proper libmain process, but a process that presumably manages its - // own signal handlers. Such a process should call either - // - initNix(), to be a proper libmain process - // - startSignalHandlerThread(), to resemble libmain regarding signal - // handling only - // - saveSignalMask(), for processes that define their own signal handling - // thread - // TODO: Warn about this? Have a default signal mask? The latter depends on - // whether we should generally inherit signal masks from the caller. - // I don't know what the larger unix ecosystem expects from us here. - if (!savedSignalMaskIsSet) - return; - - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); -} - -#if __linux__ -rlim_t savedStackSize = 0; -#endif - -void setStackSize(size_t stackSize) -{ - #if __linux__ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - savedStackSize = limit.rlim_cur; - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); - } - #endif -} - -#if __linux__ -static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedRoot; -#endif - -void saveMountNamespace() -{ -#if __linux__ - static std::once_flag done; - std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); - if (!fdSavedMountNamespace) - throw SysError("saving parent mount namespace"); - - fdSavedRoot = open("/proc/self/root", O_RDONLY); - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - try { - auto savedCwd = absPath("."); - - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); - - if (fdSavedRoot) { - if (fchdir(fdSavedRoot.get())) - throw SysError("chdir into saved root"); - if (chroot(".")) - throw SysError("chroot into saved root"); - } - - if (chdir(savedCwd.c_str()) == -1) - throw SysError("restoring cwd"); - } catch (Error & e) { - debug(e.msg()); - } -#endif -} - -void unshareFilesystem() -{ -#ifdef __linux__ - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); -#endif -} - -void restoreProcessContext(bool restoreMounts) -{ - restoreSignals(); - if (restoreMounts) { - restoreMountNamespace(); - } - - #if __linux__ - if (savedStackSize) { - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; - setrlimit(RLIMIT_STACK, &limit); - } - } - #endif -} - -/* RAII helper to automatically deregister a callback. */ -struct InterruptCallbackImpl : InterruptCallback -{ - InterruptCallbacks::Token token; - ~InterruptCallbackImpl() override - { - auto interruptCallbacks(_interruptCallbacks.lock()); - interruptCallbacks->callbacks.erase(token); - } -}; - -std::unique_ptr createInterruptCallback(std::function callback) -{ - auto interruptCallbacks(_interruptCallbacks.lock()); - auto token = interruptCallbacks->nextToken++; - interruptCallbacks->callbacks.emplace(token, callback); - - auto res = std::make_unique(); - res->token = token; - - return std::unique_ptr(res.release()); -} - - -AutoCloseFD createUnixDomainSocket() -{ - AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); - return fdSocket; -} - - -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) -{ - auto fdSocket = nix::createUnixDomainSocket(); - - bind(fdSocket.get(), path); - - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing permissions on '%1%'", path); - - if (listen(fdSocket.get(), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); - - return fdSocket; -} - - -void bind(int fd, const std::string & path) -{ - unlink(path.c_str()); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot bind to socket '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - } -} - - -void connect(int fd, const std::string & path) -{ - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot connect to socket at '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - } -} - - std::string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } - -// FIXME: move to libstore/build -void commonChildInit() -{ - logger = makeSimpleLogger(); - - const static std::string pathNullDevice = "/dev/null"; - restoreProcessContext(false); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 75683f8fe..5f730eaf6 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -4,485 +4,18 @@ #include "types.hh" #include "error.hh" #include "logging.hh" -#include "ansicolor.hh" - -#include -#include -#include -#include -#include #include -#include #include #include #include #include -#ifndef HAVE_STRUCT_DIRENT_D_TYPE -#define DT_UNKNOWN 0 -#define DT_REG 1 -#define DT_LNK 2 -#define DT_DIR 3 -#endif - namespace nix { -struct Sink; -struct Source; - void initLibUtil(); -/** - * @return an environment variable. - */ -std::optional getEnv(const std::string & key); - -/** - * @return a non empty environment variable. Returns nullopt if the env - * variable is set to "" - */ -std::optional getEnvNonEmpty(const std::string & key); - -/** - * Get the entire environment. - */ -std::map getEnv(); - -/** - * Clear the environment. - */ -void clearEnv(); - -/** - * @return An absolutized path, resolving paths relative to the - * specified directory, or the current directory otherwise. The path - * is also canonicalised. - */ -Path absPath(Path path, - std::optional dir = {}, - bool resolveSymlinks = false); - -/** - * Canonicalise a path by removing all `.` or `..` components and - * double or trailing slashes. Optionally resolves all symlink - * components such that each component of the resulting path is *not* - * a symbolic link. - */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/** - * @return The directory part of the given canonical path, i.e., - * everything before the final `/`. If the path is the root or an - * immediate child thereof (e.g., `/foo`), this means `/` - * is returned. - */ -Path dirOf(const PathView path); - -/** - * @return the base name of the given canonical path, i.e., everything - * following the final `/` (trailing slashes are removed). - */ -std::string_view baseNameOf(std::string_view path); - -/** - * Perform tilde expansion on a path. - */ -std::string expandTilde(std::string_view path); - -/** - * Check whether 'path' is a descendant of 'dir'. Both paths must be - * canonicalized. - */ -bool isInDir(std::string_view path, std::string_view dir); - -/** - * Check whether 'path' is equal to 'dir' or a descendant of - * 'dir'. Both paths must be canonicalized. - */ -bool isDirOrInDir(std::string_view path, std::string_view dir); - -/** - * Get status of `path`. - */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); - -/** - * @return true iff the given path exists. - */ -bool pathExists(const Path & path); - -/** - * A version of pathExists that returns false on a permission error. - * Useful for inferring default paths across directories that might not - * be readable. - * @return true iff the given path can be accessed and exists - */ -bool pathAccessible(const Path & path); - -/** - * Read the contents (target) of a symbolic link. The result is not - * in any way canonicalised. - */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/** - * Read the contents of a directory. The entries `.` and `..` are - * removed. - */ -struct DirEntry -{ - std::string name; - ino_t ino; - /** - * one of DT_* - */ - unsigned char type; - DirEntry(std::string name, ino_t ino, unsigned char type) - : name(std::move(name)), ino(ino), type(type) { } -}; - -typedef std::vector DirEntries; - -DirEntries readDirectory(const Path & path); - -unsigned char getFileType(const Path & path); - -/** - * Read the contents of a file into a string. - */ -std::string readFile(int fd); -std::string readFile(const Path & path); -void readFile(const Path & path, Sink & sink); - -/** - * Write a string to a file. - */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); - -/** - * Flush a file's parent directory to disk - */ -void syncParent(const Path & path); - -/** - * Read a line from a file descriptor. - */ -std::string readLine(int fd); - -/** - * Write a line to a file descriptor. - */ -void writeLine(int fd, std::string s); - -/** - * Delete a path; i.e., in the case of a directory, it is deleted - * recursively. It's not an error if the path does not exist. The - * second variant returns the number of bytes and blocks freed. - */ -void deletePath(const Path & path); - -void deletePath(const Path & path, uint64_t & bytesFreed); - -std::string getUserName(); - -/** - * @return the given user's home directory from /etc/passwd. - */ -Path getHomeOf(uid_t userId); - -/** - * @return $HOME or the user's home directory from /etc/passwd. - */ -Path getHome(); - -/** - * @return $XDG_CACHE_HOME or $HOME/.cache. - */ -Path getCacheDir(); - -/** - * @return $XDG_CONFIG_HOME or $HOME/.config. - */ -Path getConfigDir(); - -/** - * @return the directories to search for user configuration files - */ -std::vector getConfigDirs(); - -/** - * @return $XDG_DATA_HOME or $HOME/.local/share. - */ -Path getDataDir(); - -/** - * @return the path of the current executable. - */ -std::optional getSelfExe(); - -/** - * @return $XDG_STATE_HOME or $HOME/.local/state. - */ -Path getStateDir(); - -/** - * Create the Nix state directory and return the path to it. - */ -Path createNixStateDir(); - -/** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. - */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) -{ - return createDirs(Path(path)); -} - -/** - * Create a symlink. - */ -void createSymlink(const Path & target, const Path & link); - -/** - * Atomically create or replace a symlink. - */ -void replaceSymlink(const Path & target, const Path & link); - -void renameFile(const Path & src, const Path & dst); - -/** - * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` - * are on a different filesystem. - * - * Beware that this might not be atomic because of the copy that happens behind - * the scenes - */ -void moveFile(const Path & src, const Path & dst); - - -/** - * Wrappers arount read()/write() that read/write exactly the - * requested number of bytes. - */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/** - * Read a file descriptor until EOF occurs. - */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - -/** - * If cgroups are active, attempt to calculate the number of CPUs available. - * If cgroups are unavailable or if cpu.max is set to "max", return 0. - */ -unsigned int getMaxCPU(); - -/** - * Automatic cleanup of resources. - */ - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } - operator PathView() const { return path; } -}; - - -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd); - int get() const; - explicit operator bool() const; - int release(); - void close(); - void fsync(); -}; - - -/** - * Create a temporary directory. - */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/** - * Create a temporary file, returning a file handle and its path. - */ -std::pair createTempFile(const Path & prefix = "nix"); - - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; - - -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid(); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); -}; - - -/** - * Kill all processes running under the specified uid by sending them - * a SIGKILL. - */ -void killUser(uid_t uid); - - -/** - * Fork a process that runs the given function, and return the child - * pid to the caller. - */ -struct ProcessOptions -{ - std::string errorPrefix = ""; - bool dieWithParent = true; - bool runExitHandlers = false; - bool allowVfork = false; - /** - * use clone() with the specified flags (Linux only) - */ - int cloneFlags = 0; -}; - -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - - -/** - * Run a program and return its stdout in a string (i.e., like the - * shell backtick operator). - */ -std::string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional & input = {}, bool isInteractive = false); - -struct RunOptions -{ - Path program; - bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; - std::optional input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; - bool isInteractive = false; -}; - -std::pair runProgram(RunOptions && options); - -void runProgram2(const RunOptions & options); - - -/** - * Change the stack size. - */ -void setStackSize(size_t stackSize); - - -/** - * Restore the original inherited Unix process context (such as signal - * masks, stack size). - - * See startSignalHandlerThread(), saveSignalMask(). - */ -void restoreProcessContext(bool restoreMounts = true); - -/** - * Save the current mount namespace. Ignored if called more than - * once. - */ -void saveMountNamespace(); - -/** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. - */ -void restoreMountNamespace(); - -/** - * Cause this thread to not share any FS attributes with the main - * thread, because this causes setns() in restoreMountNamespace() to - * fail. - */ -void unshareFilesystem(); - - -class ExecError : public Error -{ -public: - int status; - - template - ExecError(int status, const Args & ... args) - : Error(args...), status(status) - { } -}; - /** * Convert a list of strings to a null-terminated vector of `char * *`s. The result must not be accessed beyond the lifetime of the @@ -490,36 +23,6 @@ public: */ std::vector stringsToCharPtrs(const Strings & ss); -/** - * Close all file descriptors except those listed in the given set. - * Good practice in child processes. - */ -void closeMostFDs(const std::set & exceptions); - -/** - * Set the close-on-exec flag for the given file descriptor. - */ -void closeOnExec(int fd); - - -/* User interruption. */ - -extern std::atomic _isInterrupted; - -extern thread_local std::function interruptCheck; - -void setInterruptThrown(); - -void _interrupted(); - -void inline checkInterrupt() -{ - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); -} - -MakeError(Interrupted, BaseError); - MakeError(FormatError, Error); @@ -595,15 +98,6 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/** - * Convert the exit status of a child as returned by wait() into an - * error string. - */ -std::string statusToString(int status); - -bool statusOk(int status); - - /** * Parse a string into an integer. */ @@ -711,23 +205,6 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/** - * Determine whether ANSI escape sequences are appropriate for the - * present output. - */ -bool shouldANSI(); - -/** - * Truncate a string to 'width' printable characters. If 'filterAll' - * is true, all ANSI escape sequences are filtered out. Otherwise, - * some escape sequences (such as colour setting) are copied but not - * included in the character count. Also, tabs are expanded to - * spaces. - */ -std::string filterANSIEscapes(std::string_view s, - bool filterAll = false, - unsigned int width = std::numeric_limits::max()); - /** * Base64 encoding/decoding. @@ -815,61 +292,6 @@ template class Callback; -/** - * Start a thread that handles various signals. Also block those signals - * on the current thread (and thus any threads created by it). - * Saves the signal mask before changing the mask to block those signals. - * See saveSignalMask(). - */ -void startSignalHandlerThread(); - -/** - * Saves the signal mask, which is the signal mask that nix will restore - * before creating child processes. - * See setChildSignalMask() to set an arbitrary signal mask instead of the - * current mask. - */ -void saveSignalMask(); - -/** - * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't - * necessarily match the current thread's mask. - * See saveSignalMask() to set the saved mask to the current mask. - */ -void setChildSignalMask(sigset_t *sigs); - -struct InterruptCallback -{ - virtual ~InterruptCallback() { }; -}; - -/** - * Register a function that gets called on SIGINT (in a non-signal - * context). - */ -std::unique_ptr createInterruptCallback( - std::function callback); - -void triggerInterrupt(); - -/** - * A RAII class that causes the current thread to receive SIGUSR1 when - * the signal handler thread receives SIGINT. That is, this allows - * SIGINT to be multiplexed to multiple threads. - */ -struct ReceiveInterrupts -{ - pthread_t target; - std::unique_ptr callback; - - ReceiveInterrupts() - : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) - { } -}; - - - /** * A RAII helper that increments a counter on construction and * decrements it on destruction. @@ -884,45 +306,6 @@ struct MaintainCount }; -/** - * @return the number of rows and columns of the terminal. - */ -std::pair getWindowSize(); - - -/** - * Used in various places. - */ -typedef std::function PathFilter; - -extern PathFilter defaultPathFilter; - -/** - * Common initialisation performed in child processes. - */ -void commonChildInit(); - -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); - -/** - * Bind a Unix domain socket to a path. - */ -void bind(int fd, const std::string & path); - -/** - * Connect to a Unix domain socket. - */ -void connect(int fd, const std::string & path); - - /** * A Rust/Python-like enumerate() iterator adapter. * diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 60bc08146..75ce12a8c 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -9,12 +9,12 @@ #include +#include "current-process.hh" #include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" #include "derivations.hh" -#include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" #include "eval.hh" diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 4504441fa..79db78236 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -5,7 +5,7 @@ #include "store-api.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr -#include "util.hh" +#include "users.hh" #include "tarball.hh" #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 70af53b28..bb3f1bc6a 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,3 +1,5 @@ +#include "file-system.hh" +#include "signals.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 25068f801..213a20d93 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" @@ -11,7 +12,6 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "user-env.hh" -#include "util.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index d12d70f33..250224e7d 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -1,5 +1,4 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index d40196497..c67409e89 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -6,7 +6,6 @@ #include "attr-path.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "util.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb..2c530999b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -1,5 +1,4 @@ #include "dotgraph.hh" -#include "util.hh" #include "store-api.hh" #include diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 439557658..3e789a2d8 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -1,5 +1,4 @@ #include "graphml.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e4dd94585..123283dfe 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,7 +11,6 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "shared.hh" -#include "util.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index af428018a..373dedf7c 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,11 +1,12 @@ ///@file +#include "signals.hh" +#include "unix-domain-socket.hh" #include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "util.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index b080a3939..38482ed42 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,7 +8,6 @@ #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" -#include "util.hh" #include #include diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 1aa6831d3..59f9e3e5d 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -6,7 +6,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" #include "worker-protocol.hh" using namespace nix; diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 66629fab0..9cbab230b 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e8906a252..38938f09e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -15,6 +15,7 @@ #include "registry.hh" #include "eval-cache.hh" #include "markdown.hh" +#include "users.hh" #include #include diff --git a/src/nix/main.cc b/src/nix/main.cc index d20bc1f8a..b582fc166 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,6 +1,8 @@ #include #include "args/root.hh" +#include "current-process.hh" +#include "namespaces.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index 1465e8cde..ea0a17897 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "run.hh" #include "command-installable-value.hh" #include "common-args.hh" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 45cd2e1a6..a68616355 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,3 +1,4 @@ +#include "signals.hh" #include "command.hh" #include "shared.hh" #include "store-api.hh" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index d238456db..c529c2363 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,3 +1,4 @@ +#include "processes.hh" #include "command.hh" #include "common-args.hh" #include "store-api.hh" diff --git a/src/nix/verify.cc b/src/nix/verify.cc index adaa33c0c..78cb765ce 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -4,6 +4,7 @@ #include "sync.hh" #include "thread-pool.hh" #include "references.hh" +#include "signals.hh" #include From 6472c3bf0d4b529f28f9e50834e1fc3dd101c409 Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Sat, 4 Nov 2023 12:08:00 -0400 Subject: [PATCH 282/402] fix(ssh): extraneous master processes --- src/libstore/ssh.cc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 03b2f0be9..300eb391c 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -132,7 +132,6 @@ Path SSHMaster::startMaster() if (state->sshMaster != -1) return state->socketPath; - state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; Pipe out; @@ -144,7 +143,8 @@ Path SSHMaster::startMaster() logger->pause(); Finally cleanup = [&]() { logger->resume(); }; - bool wasMasterRunning = isMasterRunning(); + if (isMasterRunning()) + return state->socketPath; state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -165,14 +165,13 @@ Path SSHMaster::startMaster() out.writeSide = -1; - if (!wasMasterRunning) { - std::string reply; - try { - reply = readLine(out.readSide.get()); - } catch (EndOfFile & e) { } + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } - if (reply != "started") - throw Error("failed to start SSH master connection to '%s'", host); + if (reply != "started") { + throw Error("failed to start SSH master connection to '%s'", host); } return state->socketPath; From 2fb49759b8307838dd1208d8ce756a60d41e4ebf Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Sat, 4 Nov 2023 12:32:57 -0400 Subject: [PATCH 283/402] fix(ssh): log first line of stdout Spent a while debugging why `nix-copy-closure` wasn't working anymore and it was my shell RC printing something I added for debug. Hopefully this can save someone else some time. --- src/libstore/ssh.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 300eb391c..5c8d6a504 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -114,8 +114,10 @@ std::unique_ptr SSHMaster::startCommand(const std::string reply = readLine(out.readSide.get()); } catch (EndOfFile & e) { } - if (reply != "started") + if (reply != "started") { + printTalkative("SSH stdout first line: %s", reply); throw Error("failed to start SSH connection to '%s'", host); + } } conn->out = std::move(out.readSide); @@ -171,6 +173,7 @@ Path SSHMaster::startMaster() } catch (EndOfFile & e) { } if (reply != "started") { + printTalkative("SSH master stdout first line: %s", reply); throw Error("failed to start SSH master connection to '%s'", host); } From 0b0d1b521449e7a66e7fa33ca7afe292d88aa14b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Oct 2023 19:39:21 -0400 Subject: [PATCH 284/402] Add comparison functions for `NarInfo` We will need these for tests. --- src/libstore/nar-info.cc | 9 +++++++++ src/libstore/nar-info.hh | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ee2ddfd81..2b77c6ab7 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,6 +4,15 @@ namespace nix { +GENERATE_CMP_EXT( + , + NarInfo, + me->url, + me->compression, + me->fileHash, + me->fileSize, + static_cast(*me)); + NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 5dbdafac3..1b3551106 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -24,6 +24,8 @@ struct NarInfo : ValidPathInfo NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); + DECLARE_CMP(NarInfo); + std::string to_string(const Store & store) const; }; From 07ac53732b8989758c264d4e847c94a5d28072cf Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sun, 5 Nov 2023 15:27:25 +0100 Subject: [PATCH 285/402] Fix moves in appendOrSet --- src/libutil/config-impl.hh | 10 +++++----- src/libutil/config.cc | 20 +++++++++----------- src/libutil/config.hh | 2 +- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b9639e761..9f69e8444 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -45,13 +45,13 @@ bool BaseSetting::isAppendable() return trait::appendable; } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); -template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); +template<> void BaseSetting::appendOrSet(Strings newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append); template -void BaseSetting::appendOrSet(T && newValue, bool append) +void BaseSetting::appendOrSet(T newValue, bool append) { static_assert( !trait::appendable, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 0bf36c987..5b510b69e 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -301,10 +301,11 @@ template<> Strings BaseSetting::parse(const std::string & str) const return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) +template<> void BaseSetting::appendOrSet(Strings newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) value.push_back(std::move(s)); + value.insert(value.end(), std::make_move_iterator(newValue.begin()), + std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -317,11 +318,10 @@ template<> StringSet BaseSetting::parse(const std::string & str) cons return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -342,11 +342,10 @@ template<> std::set BaseSetting void BaseSetting>::appendOrSet(std::set && newValue, bool append) +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting>::to_string() const @@ -369,11 +368,10 @@ template<> StringMap BaseSetting::parse(const std::string & str) cons return res; } -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append) { if (!append) value.clear(); - for (auto && [k, v] : std::move(newValue)) - value.emplace(std::move(k), std::move(v)); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 38c3ce0c4..d9441fb63 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -247,7 +247,7 @@ protected: * * @param append Whether to append or overwrite. */ - virtual void appendOrSet(T && newValue, bool append); + virtual void appendOrSet(T newValue, bool append); public: From ad385f9ec44f8d845e994764c45876042c715946 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sun, 5 Nov 2023 15:27:48 +0100 Subject: [PATCH 286/402] Minor improvements --- src/libutil/config.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index d9441fb63..3f2522c38 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -150,7 +150,7 @@ public: AbstractSetting * setting; }; - typedef std::map Settings; + using Settings = std::map; private: @@ -316,7 +316,7 @@ std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) } template -bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == (const T &) v2; } +bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == static_cast(v2); } template class Setting : public BaseSetting @@ -329,7 +329,7 @@ public: const std::set & aliases = {}, const bool documentDefault = true, std::optional experimentalFeature = std::nullopt) - : BaseSetting(def, documentDefault, name, description, aliases, experimentalFeature) + : BaseSetting(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) { options->addSetting(this); } From f404e9b3b362a054219797df02bbe277de249f80 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sun, 5 Nov 2023 16:12:20 +0100 Subject: [PATCH 287/402] Make toJSONObject const --- src/libutil/abstract-setting-to-json.hh | 2 +- src/libutil/config.cc | 2 +- src/libutil/config.hh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index d506dfb74..eea687d8a 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -7,7 +7,7 @@ namespace nix { template -std::map BaseSetting::toJSONObject() +std::map BaseSetting::toJSONObject() const { auto obj = AbstractSetting::toJSONObject(); obj.emplace("value", value); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 5b510b69e..2a5cf6212 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -221,7 +221,7 @@ nlohmann::json AbstractSetting::toJSON() return nlohmann::json(toJSONObject()); } -std::map AbstractSetting::toJSONObject() +std::map AbstractSetting::toJSONObject() const { std::map obj; obj.emplace("description", description); diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 3f2522c38..5d7bd8e0c 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -213,7 +213,7 @@ protected: nlohmann::json toJSON(); - virtual std::map toJSONObject(); + virtual std::map toJSONObject() const; virtual void convertToArg(Args & args, const std::string & category); @@ -306,7 +306,7 @@ public: void convertToArg(Args & args, const std::string & category) override; - std::map toJSONObject() override; + std::map toJSONObject() const override; }; template From a4b7df7bfaee7d27a152be2445886c81881daf94 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 6 Nov 2023 15:47:25 +0100 Subject: [PATCH 288/402] More const, scope reductions, move fixes --- src/libutil/config.cc | 60 +++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 2a5cf6212..8e7901133 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -36,27 +36,25 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { _settings.emplace(setting->name, Config::SettingData{false, setting}); - for (auto & alias : setting->aliases) + for (const auto & alias : setting->aliases) _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; - auto i = unknownSettings.find(setting->name); - if (i != unknownSettings.end()) { - setting->set(i->second); + if (auto i = unknownSettings.find(setting->name); i != unknownSettings.end()) { + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = unknownSettings.find(alias); - if (i != unknownSettings.end()) { + if (auto i = unknownSettings.find(alias); i != unknownSettings.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { - setting->set(i->second); + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; @@ -71,7 +69,7 @@ AbstractConfig::AbstractConfig(StringMap initials) void AbstractConfig::warnUnknownSettings() { - for (auto & s : unknownSettings) + for (const auto & s : unknownSettings) warn("unknown setting '%s'", s.first); } @@ -85,7 +83,7 @@ void AbstractConfig::reapplyUnknownSettings() void Config::getSettings(std::map & res, bool overriddenOnly) { - for (auto & opt : _settings) + for (const auto & opt : _settings) if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -101,8 +99,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string line += contents[pos++]; pos++; - auto hash = line.find('#'); - if (hash != std::string::npos) + if (auto hash = line.find('#'); hash != line.npos) line = std::string(line, 0, hash); auto tokens = tokenizeString>(line); @@ -135,24 +132,24 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - std::string name = tokens[0]; + std::string name = std::move(tokens[0]); auto i = tokens.begin(); advance(i, 2); parsedContents.push_back({ - name, + std::move(name), concatStringsSep(" ", Strings(i, tokens.end())), }); }; // First apply experimental-feature related settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name == "experimental-features" || name == "extra-experimental-features") set(name, value); // Then apply other settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name != "experimental-features" && name != "extra-experimental-features") set(name, value); } @@ -174,7 +171,7 @@ void Config::resetOverridden() nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); - for (auto & s : _settings) + for (const auto & s : _settings) if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; @@ -182,8 +179,8 @@ nlohmann::json Config::toJSON() std::string Config::toKeyValue() { - auto res = std::string(); - for (auto & s : _settings) + std::string res; + for (const auto & s : _settings) if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; @@ -205,7 +202,7 @@ AbstractSetting::AbstractSetting( : name(name) , description(stripIndentation(description)) , aliases(aliases) - , experimentalFeature(experimentalFeature) + , experimentalFeature(std::move(experimentalFeature)) { } @@ -284,14 +281,14 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(true); }}, + .handler = {[this] { override(true); }}, .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(false); }}, + .handler = {[this] { override(false); }}, .experimentalFeature = experimentalFeature, }); } @@ -333,8 +330,7 @@ template<> std::set BaseSetting res; for (auto & s : tokenizeString(str)) { - auto thisXpFeature = parseExperimentalFeature(s); - if (thisXpFeature) + if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) res.insert(thisXpFeature.value()); else warn("unknown experimental feature '%s'", s); @@ -351,7 +347,7 @@ template<> void BaseSetting>::appendOrSet(std::set template<> std::string BaseSetting>::to_string() const { StringSet stringifiedXpFeatures; - for (auto & feature : value) + for (const auto & feature : value) stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); return concatStringsSep(" ", stringifiedXpFeatures); } @@ -359,9 +355,8 @@ template<> std::string BaseSetting>::to_string() c template<> StringMap BaseSetting::parse(const std::string & str) const { StringMap res; - for (auto & s : tokenizeString(str)) { - auto eq = s.find_first_of('='); - if (std::string::npos != eq) + for (const auto & s : tokenizeString(str)) { + if (auto eq = s.find_first_of('='); s.npos != eq) res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } @@ -376,10 +371,9 @@ template<> void BaseSetting::appendOrSet(StringMap newValue, bool app template<> std::string BaseSetting::to_string() const { - Strings kvstrs; - std::transform(value.begin(), value.end(), back_inserter(kvstrs), - [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); - return concatStringsSep(" ", kvstrs); + return std::transform_reduce(value.cbegin(), value.cend(), std::string{}, + [](const auto & l, const auto &r) { return l + " " + r; }, + [](const auto & kvpair){ return kvpair.first + "=" + kvpair.second; }); } template class BaseSetting; @@ -468,7 +462,7 @@ void GlobalConfig::resetOverridden() nlohmann::json GlobalConfig::toJSON() { auto res = nlohmann::json::object(); - for (auto & config : *configRegistrations) + for (const auto & config : *configRegistrations) res.update(config->toJSON()); return res; } @@ -478,7 +472,7 @@ std::string GlobalConfig::toKeyValue() std::string res; std::map settings; globalConfig.getSettings(settings); - for (auto & s : settings) + for (const auto & s : settings) res += fmt("%s = %s\n", s.first, s.second.value); return res; } From 937e02e7b9538fd4500ade184eb4f0a888a9967d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 22 Oct 2023 21:12:54 -0400 Subject: [PATCH 289/402] Shuffle `ValidPathInfo` JSON rendering `Store::pathInfoToJSON` was a rather baroque functions, being full of parameters to support both parsed derivations and `nix path-info`. The common core of each, a simple `dValidPathInfo::toJSON` function, is factored out, but the rest of the logic is just duplicated and then specialized to its use-case (at which point it is no longer that duplicated). This keeps the human oriented CLI logic (which is currently unstable) and the core domain logic (export reference graphs with structured attrs, which is stable), separate, which I think is better. --- src/libstore/nar-info.cc | 46 +++++++++ src/libstore/nar-info.hh | 9 ++ src/libstore/parsed-derivations.cc | 34 ++++++- src/libstore/path-info.cc | 99 +++++++++++++++++++ src/libstore/path-info.hh | 12 +++ src/libstore/store-api.cc | 90 ----------------- src/libstore/store-api.hh | 23 ----- src/libstore/tests/nar-info.cc | 84 ++++++++++++++++ src/libstore/tests/path-info.cc | 79 +++++++++++++++ src/libutil/tests/characterization.hh | 1 + src/nix/path-info.cc | 80 ++++++++++++++- unit-test-data/libstore/nar-info/impure.json | 21 ++++ unit-test-data/libstore/nar-info/pure.json | 11 +++ unit-test-data/libstore/path-info/impure.json | 18 ++++ unit-test-data/libstore/path-info/pure.json | 11 +++ 15 files changed, 499 insertions(+), 119 deletions(-) create mode 100644 src/libstore/tests/nar-info.cc create mode 100644 src/libstore/tests/path-info.cc create mode 100644 unit-test-data/libstore/nar-info/impure.json create mode 100644 unit-test-data/libstore/nar-info/pure.json create mode 100644 unit-test-data/libstore/path-info/impure.json create mode 100644 unit-test-data/libstore/path-info/pure.json diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 2b77c6ab7..a90812ff9 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -134,4 +134,50 @@ std::string NarInfo::to_string(const Store & store) const return res; } +nlohmann::json NarInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat); + + if (includeImpureInfo) { + if (!url.empty()) + jsonObject["url"] = url; + if (fileHash) + jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); + if (fileSize) + jsonObject["downloadSize"] = fileSize; + } + + return jsonObject; +} + +NarInfo NarInfo::fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + NarInfo res { ValidPathInfo::fromJSON(store, json) }; + res.path = path; + + if (json.contains("url")) + res.url = ensureType(valueAt(json, "url"), value_t::string); + + if (json.contains("downloadHash")) + res.fileHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "downloadHash"), value_t::string)), + std::nullopt); + + if (json.contains("downloadSize")) + res.fileSize = ensureType(valueAt(json, "downloadSize"), value_t::number_integer); + + return res; +} + } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 1b3551106..cec65ff70 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -27,6 +27,15 @@ struct NarInfo : ValidPathInfo DECLARE_CMP(NarInfo); std::string to_string(const Store & store) const; + + nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const override; + static NarInfo fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json); }; } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 1d900c272..45629dc7f 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -132,6 +132,36 @@ bool ParsedDerivation::useUidRange() const static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + */ +static nlohmann::json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths) +{ + nlohmann::json::array_t jsonList = nlohmann::json::array(); + + for (auto & storePath : storePaths) { + auto info = store.queryPathInfo(storePath); + + auto & jsonPath = jsonList.emplace_back( + info->toJSON(store, false, HashFormat::Base32)); + + jsonPath["closureSize"] = ({ + uint64_t totalNarSize = 0; + StorePathSet closure; + store.computeFSClosure(info->path, closure, false, false); + for (auto & p : closure) { + auto info = store.queryPathInfo(p); + totalNarSize += info->narSize; + } + totalNarSize; + }); + } + return jsonList; +} + std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); @@ -152,8 +182,8 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s StorePathSet storePaths; for (auto & p : *i) storePaths.insert(store.parseStorePath(p.get())); - json[i.key()] = store.pathInfoToJSON( - store.exportReferences(storePaths, inputPaths), false, true); + json[i.key()] = pathInfoToJSON(store, + store.exportReferences(storePaths, inputPaths)); } } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ab39e71f4..e5d5205f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,5 +1,8 @@ +#include + #include "path-info.hh" #include "store-api.hh" +#include "json-utils.hh" namespace nix { @@ -144,4 +147,100 @@ ValidPathInfo::ValidPathInfo( }, std::move(ca).raw); } + +nlohmann::json ValidPathInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = json::object(); + + jsonObject["path"] = store.printStorePath(path); + jsonObject["valid"] = true; + jsonObject["narHash"] = narHash.to_string(hashFormat, true); + jsonObject["narSize"] = narSize; + + { + auto& jsonRefs = (jsonObject["references"] = json::array()); + for (auto & ref : references) + jsonRefs.emplace_back(store.printStorePath(ref)); + } + + if (ca) + jsonObject["ca"] = renderContentAddress(ca); + + if (includeImpureInfo) { + if (deriver) + jsonObject["deriver"] = store.printStorePath(*deriver); + + if (registrationTime) + jsonObject["registrationTime"] = registrationTime; + + if (ultimate) + jsonObject["ultimate"] = ultimate; + + if (!sigs.empty()) { + for (auto & sig : sigs) + jsonObject["signatures"].push_back(sig); + } + } + + return jsonObject; +} + +ValidPathInfo ValidPathInfo::fromJSON( + const Store & store, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + ValidPathInfo res { + StorePath(StorePath::dummy), + Hash(Hash::dummy), + }; + + ensureType(json, value_t::object); + res.path = store.parseStorePath( + static_cast( + ensureType(valueAt(json, "path"), value_t::string))); + res.narHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "narHash"), value_t::string)), + std::nullopt); + res.narSize = ensureType(valueAt(json, "narSize"), value_t::number_integer); + + try { + auto & references = ensureType(valueAt(json, "references"), value_t::array); + for (auto & input : references) + res.references.insert(store.parseStorePath(static_cast +(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'references'"); + throw; + } + + if (json.contains("ca")) + res.ca = ContentAddress::parse( + static_cast( + ensureType(valueAt(json, "ca"), value_t::string))); + + if (json.contains("deriver")) + res.deriver = store.parseStorePath( + static_cast( + ensureType(valueAt(json, "deriver"), value_t::string))); + + if (json.contains("registrationTime")) + res.registrationTime = ensureType(valueAt(json, "registrationTime"), value_t::number_integer); + + if (json.contains("ultimate")) + res.ultimate = ensureType(valueAt(json, "ultimate"), value_t::boolean); + + if (json.contains("signatures")) + res.sigs = valueAt(json, "signatures"); + + return res; +} + } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index c4c4a6366..feeda6c27 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -125,6 +125,18 @@ struct ValidPathInfo : UnkeyedValidPathInfo { Strings shortRefs() const; + /** + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + */ + virtual nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const; + static ValidPathInfo fromJSON( + const Store & store, + const nlohmann::json & json); + ValidPathInfo(const ValidPathInfo & other) = default; ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c9ebb6c14..0f88d9b92 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -951,96 +951,6 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor return paths; } -json Store::pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat, - AllowInvalidFlag allowInvalid) -{ - json::array_t jsonList = json::array(); - - for (auto & storePath : storePaths) { - auto& jsonPath = jsonList.emplace_back(json::object()); - - try { - auto info = queryPathInfo(storePath); - - jsonPath["path"] = printStorePath(info->path); - jsonPath["valid"] = true; - jsonPath["narHash"] = info->narHash.to_string(hashFormat, true); - jsonPath["narSize"] = info->narSize; - - { - auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) - jsonRefs.emplace_back(printStorePath(ref)); - } - - if (info->ca) - jsonPath["ca"] = renderContentAddress(info->ca); - - std::pair closureSizes; - - if (showClosureSize) { - closureSizes = getClosureSize(info->path); - jsonPath["closureSize"] = closureSizes.first; - } - - if (includeImpureInfo) { - - if (info->deriver) - jsonPath["deriver"] = printStorePath(*info->deriver); - - if (info->registrationTime) - jsonPath["registrationTime"] = info->registrationTime; - - if (info->ultimate) - jsonPath["ultimate"] = info->ultimate; - - if (!info->sigs.empty()) { - for (auto & sig : info->sigs) - jsonPath["signatures"].push_back(sig); - } - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - - if (narInfo) { - if (!narInfo->url.empty()) - jsonPath["url"] = narInfo->url; - if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true); - if (narInfo->fileSize) - jsonPath["downloadSize"] = narInfo->fileSize; - if (showClosureSize) - jsonPath["closureDownloadSize"] = closureSizes.second; - } - } - - } catch (InvalidPath &) { - jsonPath["path"] = printStorePath(storePath); - jsonPath["valid"] = false; - } - } - return jsonList; -} - - -std::pair Store::getClosureSize(const StorePath & storePath) -{ - uint64_t totalNarSize = 0, totalDownloadSize = 0; - StorePathSet closure; - computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) { - auto info = queryPathInfo(p); - totalNarSize += info->narSize; - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - if (narInfo) - totalDownloadSize += narInfo->fileSize; - } - return {totalNarSize, totalDownloadSize}; -} - const Store::Stats & Store::getStats() { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6aa317e3d..32ad2aa44 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -80,7 +80,6 @@ typedef std::map OutputPathMap; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; -enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; /** * Magic header of exportPath() output (obsolete). @@ -665,28 +664,6 @@ public: std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /** - * Write a JSON representation of store path metadata, such as the - * hash and the references. - * - * @param includeImpureInfo If true, variable elements such as the - * registration time are included. - * - * @param showClosureSize If true, the closure size of each path is - * included. - */ - nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat = HashFormat::Base32, - AllowInvalidFlag allowInvalid = DisallowInvalid); - - /** - * @return the size of the closure of the specified path, that is, - * the sum of the size of the NAR serialisation of each path in the - * closure. - */ - std::pair getClosureSize(const StorePath & storePath); - /** * Optimise the disk space usage of the Nix store by hard-linking files * with the same contents. diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc new file mode 100644 index 000000000..cb92f3a28 --- /dev/null +++ b/src/libstore/tests/nar-info.cc @@ -0,0 +1,84 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class NarInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/libstore/nar-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { + NarInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + + info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; + info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); + info.fileSize = 4029176; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + NarInfo got = NarInfo::fromJSON( \ + *store, \ + expected.path, \ + encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makeNarInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libstore/tests/path-info.cc b/src/libstore/tests/path-info.cc new file mode 100644 index 000000000..fbee751c6 --- /dev/null +++ b/src/libstore/tests/path-info.cc @@ -0,0 +1,79 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class PathInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/libstore/path-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { + ValidPathInfo info { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + ValidPathInfo got = ValidPathInfo::fromJSON( \ + *store, \ + encoded); \ + auto expected = makePathInfo(*store, PURE); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makePathInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh index 6698c5239..6eb513d68 100644 --- a/src/libutil/tests/characterization.hh +++ b/src/libutil/tests/characterization.hh @@ -4,6 +4,7 @@ #include #include "types.hh" +#include "environment-variables.hh" namespace nix { diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index c16864d30..b4bdd15ba 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -9,6 +9,74 @@ #include using namespace nix; +using nlohmann::json; + +/** + * @return the total size of a set of store objects (specified by path), + * that is, the sum of the size of the NAR serialisation of each object + * in the set. + */ +static uint64_t getStoreObjectsTotalSize(Store & store, const StorePathSet & closure) +{ + uint64_t totalNarSize = 0; + for (auto & p : closure) { + totalNarSize += store.queryPathInfo(p)->narSize; + } + return totalNarSize; +} + + +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ +static json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths, + bool showClosureSize) +{ + json::array_t jsonList = json::array(); + + for (auto & storePath : storePaths) { + try { + auto info = store.queryPathInfo(storePath); + + auto & jsonPath = jsonList.emplace_back( + info->toJSON(store, true, HashFormat::SRI)); + + if (showClosureSize) { + StorePathSet closure; + store.computeFSClosure(storePath, closure, false, false); + + jsonPath["closureSize"] = getStoreObjectsTotalSize(store, closure); + + if (auto * narInfo = dynamic_cast(&*info)) { + uint64_t totalDownloadSize = 0; + for (auto & p : closure) { + auto depInfo = store.queryPathInfo(p); + if (auto * depNarInfo = dynamic_cast(&*depInfo)) + totalDownloadSize += depNarInfo->fileSize; + else + throw Error("Missing .narinfo for dep %s of %s", + store.printStorePath(p), + store.printStorePath(storePath)); + } + jsonPath["closureDownloadSize"] = totalDownloadSize; + } + } + + } catch (InvalidPath &) { + auto & jsonPath = jsonList.emplace_back(json::object()); + jsonPath["path"] = store.printStorePath(storePath); + jsonPath["valid"] = false; + } + } + return jsonList; +} + struct CmdPathInfo : StorePathsCommand, MixJSON { @@ -87,10 +155,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - std::cout << store->pathInfoToJSON( + std::cout << pathInfoToJSON( + *store, // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, HashFormat::SRI, AllowInvalid).dump(); + showClosureSize).dump(); } else { @@ -107,8 +176,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (showSize) printSize(info->narSize); - if (showClosureSize) - printSize(store->getClosureSize(info->path).first); + if (showClosureSize) { + StorePathSet closure; + store->computeFSClosure(storePath, closure, false, false); + printSize(getStoreObjectsTotalSize(*store, closure)); + } if (showSigs) { std::cout << '\t'; diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json new file mode 100644 index 000000000..093f25025 --- /dev/null +++ b/unit-test-data/libstore/nar-info/impure.json @@ -0,0 +1,21 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "valid": true +} diff --git a/unit-test-data/libstore/nar-info/pure.json b/unit-test-data/libstore/nar-info/pure.json new file mode 100644 index 000000000..62005d414 --- /dev/null +++ b/unit-test-data/libstore/nar-info/pure.json @@ -0,0 +1,11 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "valid": true +} diff --git a/unit-test-data/libstore/path-info/impure.json b/unit-test-data/libstore/path-info/impure.json new file mode 100644 index 000000000..c477c768c --- /dev/null +++ b/unit-test-data/libstore/path-info/impure.json @@ -0,0 +1,18 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true, + "valid": true +} diff --git a/unit-test-data/libstore/path-info/pure.json b/unit-test-data/libstore/path-info/pure.json new file mode 100644 index 000000000..62005d414 --- /dev/null +++ b/unit-test-data/libstore/path-info/pure.json @@ -0,0 +1,11 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "valid": true +} From a7212e169b7204f80ea67f60c855d05b72b5d4f7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Oct 2023 20:01:36 -0400 Subject: [PATCH 290/402] Include `compression` in the `NarInfo` JSON format It was forgotten before. --- src/libstore/nar-info.cc | 5 +++++ src/libstore/tests/nar-info.cc | 1 + unit-test-data/libstore/nar-info/impure.json | 1 + 3 files changed, 7 insertions(+) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index a90812ff9..708cc7341 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -146,6 +146,8 @@ nlohmann::json NarInfo::toJSON( if (includeImpureInfo) { if (!url.empty()) jsonObject["url"] = url; + if (!compression.empty()) + jsonObject["compression"] = compression; if (fileHash) jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); if (fileSize) @@ -168,6 +170,9 @@ NarInfo NarInfo::fromJSON( if (json.contains("url")) res.url = ensureType(valueAt(json, "url"), value_t::string); + if (json.contains("compression")) + res.compression = ensureType(valueAt(json, "compression"), value_t::string); + if (json.contains("downloadHash")) res.fileHash = Hash::parseAny( static_cast( diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc index cb92f3a28..c5b21d56b 100644 --- a/src/libstore/tests/nar-info.cc +++ b/src/libstore/tests/nar-info.cc @@ -48,6 +48,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { info.sigs = { "asdf", "qwer" }; info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; + info.compression = "xz"; info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); info.fileSize = 4029176; } diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json index 093f25025..3f16667c9 100644 --- a/unit-test-data/libstore/nar-info/impure.json +++ b/unit-test-data/libstore/nar-info/impure.json @@ -1,5 +1,6 @@ { "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "compression": "xz", "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "downloadSize": 4029176, From cc46ea163024254d0b74646e1b38b19896d40040 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 22 Oct 2023 21:12:54 -0400 Subject: [PATCH 291/402] Make `nix path-info --json` return an object not array Before it returned a list of JSON objects with store object information, including the path in each object. Now, it maps the paths to JSON objects with the metadata sans path. This matches how `nix derivation show` works. Quite hillariously, none of our existing functional tests caught this change to `path-info --json` though they did use it. So just new functional tests need to be added. --- doc/manual/src/release-notes/rl-next.md | 36 ++++++++++++++++++- src/libstore/nar-info.cc | 8 +++-- src/libstore/parsed-derivations.cc | 5 +++ src/libstore/path-info.cc | 12 ++----- src/libstore/path-info.hh | 24 ++++++------- src/libstore/tests/path-info.cc | 6 ++-- src/nix/path-info.cc | 19 +++++----- src/nix/path-info.md | 12 +++---- tests/functional/local.mk | 1 + tests/functional/path-info.sh | 23 ++++++++++++ unit-test-data/libstore/nar-info/impure.json | 4 +-- unit-test-data/libstore/nar-info/pure.json | 4 +-- unit-test-data/libstore/path-info/impure.json | 4 +-- unit-test-data/libstore/path-info/pure.json | 4 +-- 14 files changed, 108 insertions(+), 54 deletions(-) create mode 100644 tests/functional/path-info.sh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 8cd69f8fd..85e180e37 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -28,5 +28,39 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. - + - Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + +- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) + (experimental) now returns a JSON map rather than JSON list. + The `path` field of each object has instead become the key in th outer map, since it is unique. + The `valid` field also goes away because we just use null instead. + + - Old way: + + ```json5 + [ + { + "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", + "valid": true, + // ... + }, + { + "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", + "valid": false + } + ] + ``` + + - New way + + ```json5 + { + "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { + // ... + }, + "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, + } + ``` + + This makes it match `nix derivation show`, which also maps store paths to information. diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 708cc7341..ae2223fb0 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -164,8 +164,12 @@ NarInfo NarInfo::fromJSON( { using nlohmann::detail::value_t; - NarInfo res { ValidPathInfo::fromJSON(store, json) }; - res.path = path; + NarInfo res { + ValidPathInfo { + path, + UnkeyedValidPathInfo::fromJSON(store, json), + } + }; if (json.contains("url")) res.url = ensureType(valueAt(json, "url"), value_t::string); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 45629dc7f..73e55a96c 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -148,6 +148,11 @@ static nlohmann::json pathInfoToJSON( auto & jsonPath = jsonList.emplace_back( info->toJSON(store, false, HashFormat::Base32)); + // Add the path to the object whose metadata we are including. + jsonPath["path"] = store.printStorePath(storePath); + + jsonPath["valid"] = true; + jsonPath["closureSize"] = ({ uint64_t totalNarSize = 0; StorePathSet closure; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index e5d5205f4..2d7dc972f 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -148,7 +148,7 @@ ValidPathInfo::ValidPathInfo( } -nlohmann::json ValidPathInfo::toJSON( +nlohmann::json UnkeyedValidPathInfo::toJSON( const Store & store, bool includeImpureInfo, HashFormat hashFormat) const @@ -157,8 +157,6 @@ nlohmann::json ValidPathInfo::toJSON( auto jsonObject = json::object(); - jsonObject["path"] = store.printStorePath(path); - jsonObject["valid"] = true; jsonObject["narHash"] = narHash.to_string(hashFormat, true); jsonObject["narSize"] = narSize; @@ -190,21 +188,17 @@ nlohmann::json ValidPathInfo::toJSON( return jsonObject; } -ValidPathInfo ValidPathInfo::fromJSON( +UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( const Store & store, const nlohmann::json & json) { using nlohmann::detail::value_t; - ValidPathInfo res { - StorePath(StorePath::dummy), + UnkeyedValidPathInfo res { Hash(Hash::dummy), }; ensureType(json, value_t::object); - res.path = store.parseStorePath( - static_cast( - ensureType(valueAt(json, "path"), value_t::string))); res.narHash = Hash::parseAny( static_cast( ensureType(valueAt(json, "narHash"), value_t::string)), diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index feeda6c27..077abc7e1 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -78,6 +78,18 @@ struct UnkeyedValidPathInfo DECLARE_CMP(UnkeyedValidPathInfo); virtual ~UnkeyedValidPathInfo() { } + + /** + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + */ + virtual nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const; + static UnkeyedValidPathInfo fromJSON( + const Store & store, + const nlohmann::json & json); }; struct ValidPathInfo : UnkeyedValidPathInfo { @@ -125,18 +137,6 @@ struct ValidPathInfo : UnkeyedValidPathInfo { Strings shortRefs() const; - /** - * @param includeImpureInfo If true, variable elements such as the - * registration time are included. - */ - virtual nlohmann::json toJSON( - const Store & store, - bool includeImpureInfo, - HashFormat hashFormat) const; - static ValidPathInfo fromJSON( - const Store & store, - const nlohmann::json & json); - ValidPathInfo(const ValidPathInfo & other) = default; ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; diff --git a/src/libstore/tests/path-info.cc b/src/libstore/tests/path-info.cc index fbee751c6..49bf623bd 100644 --- a/src/libstore/tests/path-info.cc +++ b/src/libstore/tests/path-info.cc @@ -19,8 +19,8 @@ class PathInfoTest : public CharacterizationTest, public LibStoreTest } }; -static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { - ValidPathInfo info { +static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { + UnkeyedValidPathInfo info = ValidPathInfo { store, "foo", FixedOutputInfo { @@ -54,7 +54,7 @@ static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ readTest(#STEM, [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ - ValidPathInfo got = ValidPathInfo::fromJSON( \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON( \ *store, \ encoded); \ auto expected = makePathInfo(*store, PURE); \ diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index b4bdd15ba..23198a120 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -38,20 +38,21 @@ static json pathInfoToJSON( const StorePathSet & storePaths, bool showClosureSize) { - json::array_t jsonList = json::array(); + json::object_t jsonAllObjects = json::object(); for (auto & storePath : storePaths) { + json jsonObject; + try { auto info = store.queryPathInfo(storePath); - auto & jsonPath = jsonList.emplace_back( - info->toJSON(store, true, HashFormat::SRI)); + jsonObject = info->toJSON(store, true, HashFormat::SRI); if (showClosureSize) { StorePathSet closure; store.computeFSClosure(storePath, closure, false, false); - jsonPath["closureSize"] = getStoreObjectsTotalSize(store, closure); + jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure); if (auto * narInfo = dynamic_cast(&*info)) { uint64_t totalDownloadSize = 0; @@ -64,17 +65,17 @@ static json pathInfoToJSON( store.printStorePath(p), store.printStorePath(storePath)); } - jsonPath["closureDownloadSize"] = totalDownloadSize; + jsonObject["closureDownloadSize"] = totalDownloadSize; } } } catch (InvalidPath &) { - auto & jsonPath = jsonList.emplace_back(json::object()); - jsonPath["path"] = store.printStorePath(storePath); - jsonPath["valid"] = false; + jsonObject = nullptr; } + + jsonAllObjects[store.printStorePath(storePath)] = std::move(jsonObject); } - return jsonList; + return jsonAllObjects; } diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 2dda866d0..4594854eb 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -43,7 +43,7 @@ R""( command): ```console - # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + # nix path-info --json --all | jq -r 'to_entries | sort_by(.value.registrationTime) | .[-11:-1][] | .key' ``` * Show the size of the entire Nix store: @@ -58,13 +58,13 @@ R""( ```console # nix path-info --json --all --closure-size \ - | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + | jq 'map_values(.closureSize | select(. < 1e9)) | to_entries | sort_by(.value)' [ …, - [ - "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", - 5887562256 - ] + { + .key = "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + .value = 5887562256, + } ] ``` diff --git a/tests/functional/local.mk b/tests/functional/local.mk index fe0d0c4ed..21dabca88 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -120,6 +120,7 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ + path-info.sh \ toString-path.sh \ read-only-store.sh \ nested-sandboxing.sh \ diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh new file mode 100644 index 000000000..763935eb7 --- /dev/null +++ b/tests/functional/path-info.sh @@ -0,0 +1,23 @@ +source common.sh + +echo foo > $TEST_ROOT/foo +foo=$(nix store add-file $TEST_ROOT/foo) + +echo bar > $TEST_ROOT/bar +bar=$(nix store add-file $TEST_ROOT/bar) + +echo baz > $TEST_ROOT/baz +baz=$(nix store add-file $TEST_ROOT/baz) +nix-store --delete "$baz" + +diff --unified --color=always \ + <(nix path-info --json "$foo" "$bar" "$baz" | + jq --sort-keys 'map_values(.narHash)') \ + <(jq --sort-keys <<-EOF + { + "$foo": "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=", + "$bar": "sha256-9fhYGu9fqxcQC2Kc81qh2RMo1QcLBUBo8U+pPn+jthQ=", + "$baz": null + } +EOF + ) diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json index 3f16667c9..bb9791a6a 100644 --- a/unit-test-data/libstore/nar-info/impure.json +++ b/unit-test-data/libstore/nar-info/impure.json @@ -6,7 +6,6 @@ "downloadSize": 4029176, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, - "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" @@ -17,6 +16,5 @@ "qwer" ], "ultimate": true, - "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", - "valid": true + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" } diff --git a/unit-test-data/libstore/nar-info/pure.json b/unit-test-data/libstore/nar-info/pure.json index 62005d414..955baec31 100644 --- a/unit-test-data/libstore/nar-info/pure.json +++ b/unit-test-data/libstore/nar-info/pure.json @@ -2,10 +2,8 @@ "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, - "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ], - "valid": true + ] } diff --git a/unit-test-data/libstore/path-info/impure.json b/unit-test-data/libstore/path-info/impure.json index c477c768c..0c452cc49 100644 --- a/unit-test-data/libstore/path-info/impure.json +++ b/unit-test-data/libstore/path-info/impure.json @@ -3,7 +3,6 @@ "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, - "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" @@ -13,6 +12,5 @@ "asdf", "qwer" ], - "ultimate": true, - "valid": true + "ultimate": true } diff --git a/unit-test-data/libstore/path-info/pure.json b/unit-test-data/libstore/path-info/pure.json index 62005d414..955baec31 100644 --- a/unit-test-data/libstore/path-info/pure.json +++ b/unit-test-data/libstore/path-info/pure.json @@ -2,10 +2,8 @@ "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, - "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ], - "valid": true + ] } From 61d6fe059e959455e156c1d57bb91155d363e983 Mon Sep 17 00:00:00 2001 From: Mel Zuser Date: Mon, 6 Nov 2023 14:13:40 -0500 Subject: [PATCH 292/402] Fix `boost::bad_format_string` exception in `builtins.addErrorContext` (#9291) * Fix boost::bad_format_string exception in builtins.addErrorContext The message passed to addTrace was incorrectly being used as a format string and this this would cause an exception when the string contained a '%', which can be hit in places where arbitrary file paths are interpolated. * add test --- src/libexpr/primops.cc | 2 +- tests/functional/lang.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 36340d0f9..8d3a18526 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -825,7 +825,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); - e.addTrace(nullptr, message, true); + e.addTrace(nullptr, hintfmt(message), true); throw; } } diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index c3acef5ee..12df32c87 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -23,6 +23,7 @@ nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2> nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello %" (throw "Foo")' | grepQuiet 'Hello %' nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' From 867f894289437a96630579592a46a4253151f079 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 15 Sep 2023 10:49:30 -0700 Subject: [PATCH 293/402] Populate $XDG_DATA_DIRS with appropriate folder from Nix profile On non-NixOS systems, the default `nix` install does not populate the `$XDG_DATA_DIRS`. This populates it and enables things like bash-completion and `.desktop` file detection for `nix` profile installed packages. Signed-off-by: Ana Hobden --- scripts/nix-profile-daemon.fish.in | 8 ++++++++ scripts/nix-profile-daemon.sh.in | 8 ++++++++ scripts/nix-profile.fish.in | 8 ++++++++ scripts/nix-profile.sh.in | 8 ++++++++ 4 files changed, 32 insertions(+) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 400696812..e7b394d56 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -19,6 +19,14 @@ set __ETC_PROFILE_NIX_SOURCED 1 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +# Populate bash completions, .desktop files, etc +if test -n "$NIX_SSH_CERT_FILE" + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +else + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +end + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if test -n "$NIX_SSH_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 8cfd3149e..3089cec66 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -30,6 +30,14 @@ fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" +# Populate bash completions, .desktop files, etc +if [ -n "${XDG_DATA_DIRS-}" ]; then + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +else + export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +fi + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 731498c76..fc8fe4e97 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -20,6 +20,14 @@ if test -n "$HOME" && test -n "$USER" # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + # Populate bash completions, .desktop files, etc + if test -n "$NIX_SSH_CERT_FILE" + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + else + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + end + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if test -n "$NIX_SSH_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index c4d60cf37..a0d098588 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -32,6 +32,14 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" + # Populate bash completions, .desktop files, etc + if [ -n "${XDG_DATA_DIRS-}" ]; then + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + else + export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + fi + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt From 896013ec0c0d4633349ff0373bdae626667adc77 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Thu, 21 Sep 2023 09:27:35 -0700 Subject: [PATCH 294/402] Fix bad copy-paste --- scripts/nix-profile-daemon.fish.in | 2 +- scripts/nix-profile.fish.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index e7b394d56..5f5a53141 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -20,7 +20,7 @@ set __ETC_PROFILE_NIX_SOURCED 1 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Populate bash completions, .desktop files, etc -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index fc8fe4e97..2523594f2 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -21,7 +21,7 @@ if test -n "$HOME" && test -n "$USER" set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Populate bash completions, .desktop files, etc - if test -n "$NIX_SSH_CERT_FILE" + if test -n "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else From 150b5aba509d169a50c6ad62100c3ad7bf00242b Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 6 Nov 2023 10:07:53 -0800 Subject: [PATCH 295/402] Update scripts/nix-profile-daemon.fish.in Co-authored-by: Valentin Gagarin --- scripts/nix-profile-daemon.fish.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 5f5a53141..3fe9e782a 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -22,9 +22,9 @@ set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profi # Populate bash completions, .desktop files, etc if test -n "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default - set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share" else - set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:/nix/var/nix/profiles/default/share" end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. From c60eba3276d7417a7f51ef606e5b9ca580cf5e5b Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Mon, 6 Nov 2023 21:43:18 +0100 Subject: [PATCH 296/402] Add release note on XDG_DATA_DIRS change Follow-up to https://github.com/NixOS/nix/pull/8985 --- doc/manual/src/release-notes/rl-next.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 85e180e37..73ba03fc4 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -64,3 +64,8 @@ ``` This makes it match `nix derivation show`, which also maps store paths to information. + +- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) + [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. + This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) + (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). From 9fec62a10044629ad4758ec95f9b1e67d7aefff5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 22:21:07 +0000 Subject: [PATCH 297/402] build(deps): bump zeebe-io/backport-action from 2.0.0 to 2.1.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.0.0 to 2.1.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v2.0.0...v2.1.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 312c211dd..893f4a56f 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.0.0 + uses: zeebe-io/backport-action@v2.1.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From b733f4ab29cec07cf17e1fe6580c9d2f8a4362a0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 7 Nov 2023 01:12:39 +0100 Subject: [PATCH 298/402] maintainers: refine the mission statement phrasing setting a direction falls short of what we're already doing: guide contributors. the direction aspect is still important, as that is the authoritative part. guidance is the supportive part. --- maintainers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 5be4f9d04..ee97c1195 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -2,7 +2,7 @@ ## Motivation -The team's main responsibility is to set a direction for the development of Nix and ensure that the code is in good shape. +The team's main responsibility is to guide and direct the development of Nix and ensure that the code is in good shape. We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs. From 1362a0a55aaddccef5a525e3b1179239d650bb07 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Mon, 6 Nov 2023 23:16:05 +0100 Subject: [PATCH 299/402] Fix logic for default XDG_DATA_DIRS value The [POSIX test manpage](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/test.html) as well as the [fish test manpage](https://fishshell.com/docs/current/cmds/test.html#operators-for-text-strings) specify that `-z` will be "True if the length of string string is zero; otherwise, false." The `-n` was likely a mixup and not caught during testing of https://github.com/NixOS/nix/pull/8985 due to a lack of missing conflicting entries in `XDG_DATA_DIRS`. --- scripts/nix-profile-daemon.fish.in | 2 +- scripts/nix-profile-daemon.sh.in | 2 +- scripts/nix-profile.fish.in | 2 +- scripts/nix-profile.sh.in | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 3fe9e782a..c23aa64f0 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -20,7 +20,7 @@ set __ETC_PROFILE_NIX_SOURCED 1 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Populate bash completions, .desktop files, etc -if test -n "$XDG_DATA_DIRS" +if test -z "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 3089cec66..c63db4648 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -31,7 +31,7 @@ fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc -if [ -n "${XDG_DATA_DIRS-}" ]; then +if [ -z "$XDG_DATA_DIRS" ]; then # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 2523594f2..619df52b8 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -21,7 +21,7 @@ if test -n "$HOME" && test -n "$USER" set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Populate bash completions, .desktop files, etc - if test -n "$XDG_DATA_DIRS" + if test -z "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index a0d098588..56e070ae1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -33,7 +33,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc - if [ -n "${XDG_DATA_DIRS-}" ]; then + if [ -z "$XDG_DATA_DIRS" ]; then # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else From 74210c12feccc6c6b717c5f39c28d7ce86614e60 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 28 Aug 2021 16:26:53 -0400 Subject: [PATCH 300/402] Shellbang support with flakes Enables shebang usage of nix shell. All arguments with `#! nix` get added to the nix invocation. This implementation does NOT set any additional arguments other than placing the script path itself as the first argument such that the interpreter can utilize it. Example below: ``` #!/usr/bin/env nix #! nix shell --quiet #! nix nixpkgs#bash #! nix nixpkgs#shellcheck #! nix nixpkgs#hello #! nix --ignore-environment --command bash # shellcheck shell=bash set -eu shellcheck "$0" || exit 1 function main { hello echo 0:"$0" 1:"$1" 2:"$2" } "$@" ``` fix: include programName usage EDIT: For posterity I've changed shellwords to shellwords2 in order not to interfere with other changes during a rebase. shellwords2 is removed in a later commit. -- roberth --- src/libutil/args.cc | 37 +++++++++++++++++++++++++ src/libutil/args.hh | 8 +++++- src/libutil/util.cc | 45 +++++++++++++++++++++++++++++++ src/libutil/util.hh | 11 +++++--- src/nix/main.cc | 2 +- tests/functional/flakes/common.sh | 5 ++-- tests/functional/flakes/flakes.sh | 15 ++++++++++- 7 files changed, 114 insertions(+), 9 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 0b65519a3..7106491fd 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -6,6 +6,7 @@ #include "users.hh" #include "json-utils.hh" +#include #include namespace nix { @@ -78,6 +79,12 @@ std::optional RootArgs::needsCompletion(std::string_view s) } void RootArgs::parseCmdline(const Strings & _cmdline) +{ + // Default via 5.1.2.2.1 in C standard + Args::parseCmdline("", _cmdline); +} + +void Args::parseCmdline(const std::string & programName, const Strings & _cmdline) { Strings pendingArgs; bool dashDash = false; @@ -93,6 +100,36 @@ void RootArgs::parseCmdline(const Strings & _cmdline) } bool argsSeen = false; + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + Strings savedArgs; + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + if (isNixCommand && cmdline.size() > 0) { + auto script = *cmdline.begin(); + try { + auto lines = tokenizeString(readFile(script), "\n"); + if (std::regex_search(lines.front(), std::regex("^#!"))) { + lines.pop_front(); + for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) + savedArgs.push_back(*pos); + cmdline.clear(); + + for (auto line : lines) { + line = chomp(line); + + std::smatch match; + if (std::regex_match(line, match, std::regex("^#!\\s*nix\\s(.*)$"))) + for (const auto & word : shellwords(match[1].str())) + cmdline.push_back(word); + } + cmdline.push_back(script); + for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) + cmdline.push_back(*pos); + } + } catch (SysError &) { } + } for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 45fd678e7..1d056678d 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -27,8 +27,14 @@ class Args public: /** - * Return a short one-line description of the command. + * Parse the command line with argv0, throwing a UsageError if something + goes wrong. */ + void parseCmdline(const std::string & argv0, const Strings & cmdline); + + /** + * Return a short one-line description of the command. + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ee7a22849..6ca1dbd7a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -5,6 +5,8 @@ #include #include #include +#include + namespace nix { @@ -136,6 +138,49 @@ std::string shellEscape(const std::string_view s) return r; } +/* Recreate the effect of the perl shellwords function, breaking up a + * string into arguments like a shell word, including escapes + */ +std::vector shellwords2(const std::string & s) +{ + std::regex whitespace("^(\\s+).*"); + auto begin = s.cbegin(); + std::vector res; + std::string cur; + enum state { + sBegin, + sQuote + }; + state st = sBegin; + auto it = begin; + for (; it != s.cend(); ++it) { + if (st == sBegin) { + std::smatch match; + if (regex_search(it, s.cend(), match, whitespace)) { + cur.append(begin, it); + res.push_back(cur); + cur.clear(); + it = match[1].second; + begin = it; + } + } + switch (*it) { + case '"': + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sQuote : sBegin; + break; + case '\\': + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + break; + } + } + cur.append(begin, it); + if (!cur.empty()) res.push_back(cur); + return res; +} void ignoreException(Verbosity lvl) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 5f730eaf6..bcd0c1769 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -189,10 +189,13 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/** - * Exception handling in destructors: print an error message, then - * ignore the exception. - */ +/* Recreate the effect of the perl shellwords function, breaking up a + * string into arguments like a shell word, including escapes */ +std::vector shellwords2(const std::string & s); + + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); diff --git a/src/nix/main.cc b/src/nix/main.cc index b582fc166..16fb50806 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -428,7 +428,7 @@ void mainWrapped(int argc, char * * argv) }); try { - args.parseCmdline(argvToStrings(argc, argv)); + args.parseCmdline(programName, argvToStrings(argc, argv)); } catch (UsageError &) { if (!args.helpRequested && !args.completions) throw; } diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 8aed296e6..fc45cf7bf 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -11,6 +11,7 @@ writeSimpleFlake() { outputs = inputs: rec { packages.$system = rec { foo = import ./simple.nix; + fooScript = (import ./shell.nix {}).foo; default = foo; }; packages.someOtherSystem = rec { @@ -24,13 +25,13 @@ writeSimpleFlake() { } EOF - cp ../simple.nix ../simple.builder.sh ../config.nix $flakeDir/ + cp ../simple.nix ../shell.nix ../simple.builder.sh ../config.nix $flakeDir/ } createSimpleGitFlake() { local flakeDir="$1" writeSimpleFlake $flakeDir - git -C $flakeDir add flake.nix simple.nix simple.builder.sh config.nix + git -C $flakeDir add flake.nix simple.nix shell.nix simple.builder.sh config.nix git -C $flakeDir commit -m 'Initial' } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index b0038935c..c4b18a21b 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -66,7 +66,17 @@ cat > "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < Date: Mon, 14 Nov 2022 17:04:19 +0100 Subject: [PATCH 301/402] src/libutil/util.hh: Formatting --- src/libutil/util.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bcd0c1769..b7d3ac504 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -190,7 +190,7 @@ std::string shellEscape(const std::string_view s); /* Recreate the effect of the perl shellwords function, breaking up a - * string into arguments like a shell word, including escapes */ + string into arguments like a shell word, including escapes. */ std::vector shellwords2(const std::string & s); From eea5a003d99094d8488fd0d1ecd97f98d3573133 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 14 Nov 2022 19:40:01 -0500 Subject: [PATCH 302/402] fix: test to ensure arguments are passed --- tests/functional/flakes/flakes.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index c4b18a21b..e7dffde07 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -71,8 +71,9 @@ cat > "$nonFlakeDir/shebang.sh" < Date: Mon, 14 Nov 2022 23:58:58 -0500 Subject: [PATCH 303/402] doc: shebang release notes, docs, tests fix: release notes --- doc/manual/src/release-notes/rl-next.md | 44 +++++++++ src/nix/shell.md | 117 ++++++++++++++++++++++++ tests/functional/flakes/flakes.sh | 13 +++ 3 files changed, 174 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 73ba03fc4..93d4f432b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,5 +1,49 @@ # Release X.Y (202?-??-??) +- The experimental nix command is now a `#!-interpreter` by appending the + contents of any `#! nix` lines and the script's location to a single call. + Some examples: + ``` + #!/usr/bin/env nix + #! nix shell --file "" hello --command bash + + hello | cowsay + ``` + or with flakes: + ``` + #!/usr/bin/env nix + #! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + + hello | cowsay + ``` + or + ```bash + #! /usr/bin/env nix + #! nix shell --impure --expr + #! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" + #! nix --command bash + + terraform "$@" + ``` + or + ``` + #!/usr/bin/env nix + //! ```cargo + //! [dependencies] + //! time = "0.1.25" + //! ``` + /* + #!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script + */ + fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); + } + // vim: ft=rust + ``` - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/shell.md b/src/nix/shell.md index f36919575..b0bfa1609 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -51,4 +51,121 @@ R""( provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. +# Use as a `#!`-interpreter + +You can use `nix` as a script interpreter to allow scripts written +in arbitrary languages to obtain their own dependencies via Nix. This is +done by starting the script with the following lines: + +```bash +#! /usr/bin/env nix +#! nix shell installables --command real-interpreter +``` + +where *real-interpreter* is the “real” script interpreter that will be +invoked by `nix shell` after it has obtained the dependencies and +initialised the environment, and *installables* are the attribute names of +the dependencies in Nixpkgs. + +The lines starting with `#! nix` specify options (see above). Note that you +cannot write `#! /usr/bin/env nix shell -i ...` because many operating systems +only allow one argument in `#!` lines. + +For example, here is a Python script that depends on Python and the +`prettytable` package: + +```python +#! /usr/bin/env nix +#! nix shell github:tomberek/-#python3With.prettytable --command python + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t +``` + +Similarly, the following is a Perl script that specifies that it +requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: + +```perl +#! /usr/bin/env nix +#! nix shell github:tomberek/-#perlWith.HTMLTokeParserSimple.LWP --command perl -x + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} +``` + +Sometimes you need to pass a simple Nix expression to customize a +package like Terraform: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr +#! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix --command bash + +terraform "$@" +``` + +> **Note** +> +> You must use double quotes (`"`) when passing a simple Nix expression +> in a nix shell shebang. + +Finally, using the merging of multiple nix shell shebangs the following +Haskell script uses a specific branch of Nixpkgs/NixOS (the 21.11 stable +branch): + +```haskell +#!/usr/bin/env nix +#!nix shell --override-input nixpkgs github:NixOS/nixpkgs/nixos-21.11 +#!nix github:tomberek/-#haskellWith.download-curl.tagsoup --command runghc + +import Network.Curl.Download +import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) + +-- Fetch nixos.org and print all hrefs. +main = do + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp + let tags' = map (fromAttrib "href") tags + mapM_ putStrLn $ filter (/= "") tags' +``` + +If you want to be even more precise, you can specify a specific revision +of Nixpkgs: + + #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d + +The examples above all used `-p` to get dependencies from Nixpkgs. You +can also use a Nix expression to build your own dependencies. For +example, the Python example could have been written as: + +```python +#! /usr/bin/env nix +#! nix shell --impure --file deps.nix -i python +``` + +where the file `deps.nix` in the same directory as the `#!`-script +contains: + +```nix +with import {}; +python3.withPackages (ps: with ps; [ prettytable ]) +``` + + + + )"" diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index e7dffde07..f27925493 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,6 +80,18 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' +cat > $nonFlakeDir/shebang-perl.sh < Date: Wed, 1 Sep 2021 02:19:51 -0400 Subject: [PATCH 304/402] Read file incrementally --- src/libutil/args.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 7106491fd..80216e7ad 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -6,6 +6,8 @@ #include "users.hh" #include "json-utils.hh" +#include +#include #include #include @@ -109,14 +111,17 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin if (isNixCommand && cmdline.size() > 0) { auto script = *cmdline.begin(); try { - auto lines = tokenizeString(readFile(script), "\n"); - if (std::regex_search(lines.front(), std::regex("^#!"))) { - lines.pop_front(); + std::ifstream stream(script); + char shebang[3]={0,0,0}; + stream.get(shebang,3); + if (strncmp(shebang,"#!",2) == 0){ for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) savedArgs.push_back(*pos); cmdline.clear(); - for (auto line : lines) { + std::string line; + std::getline(stream,line); + while (std::getline(stream,line) && !line.empty()){ line = chomp(line); std::smatch match; From 06f3583b1c860b24f2f704f216f4db8fd1dcae9c Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 26 Nov 2022 09:06:39 -0500 Subject: [PATCH 305/402] feat: break out of shebang processing for non-comments --- src/libutil/args.cc | 3 ++- tests/functional/flakes/flakes.sh | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 80216e7ad..d90374adc 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -121,7 +121,8 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin std::string line; std::getline(stream,line); - while (std::getline(stream,line) && !line.empty()){ + std::string commentChars("#/\\%@*-"); + while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); std::smatch match; diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index f27925493..28b5e4e0f 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,17 +80,17 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' -cat > $nonFlakeDir/shebang-perl.sh < $nonFlakeDir/shebang-comments.sh < Date: Tue, 3 Jan 2023 05:55:06 -0500 Subject: [PATCH 306/402] doc: remove reference to nix-shell --- src/nix/shell.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/nix/shell.md b/src/nix/shell.md index b0bfa1609..7e0e5f213 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -148,9 +148,8 @@ of Nixpkgs: #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d -The examples above all used `-p` to get dependencies from Nixpkgs. You -can also use a Nix expression to build your own dependencies. For -example, the Python example could have been written as: +You can also use a Nix expression to build your own dependencies. For example, +the Python example could have been written as: ```python #! /usr/bin/env nix @@ -166,6 +165,4 @@ python3.withPackages (ps: with ps; [ prettytable ]) ``` - - )"" From bbeddf06027424dc08742c1d54bf2fdc85ff6e8e Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 12 May 2023 07:44:25 -0400 Subject: [PATCH 307/402] fix: refactor parseCmdline interface --- src/libutil/args.cc | 9 ++++----- src/libutil/args.hh | 2 +- src/nix/main.cc | 5 ++++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index d90374adc..481ed33ff 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -83,10 +83,10 @@ std::optional RootArgs::needsCompletion(std::string_view s) void RootArgs::parseCmdline(const Strings & _cmdline) { // Default via 5.1.2.2.1 in C standard - Args::parseCmdline("", _cmdline); + Args::parseCmdline(_cmdline, false); } -void Args::parseCmdline(const std::string & programName, const Strings & _cmdline) +void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -107,8 +107,7 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". Strings savedArgs; - auto isNixCommand = std::regex_search(programName, std::regex("nix$")); - if (isNixCommand && cmdline.size() > 0) { + if (allowShebang){ auto script = *cmdline.begin(); try { std::ifstream stream(script); @@ -121,7 +120,7 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin std::string line; std::getline(stream,line); - std::string commentChars("#/\\%@*-"); + static const std::string commentChars("#/\\%@*-"); while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 1d056678d..e753dcaf6 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -30,7 +30,7 @@ public: * Parse the command line with argv0, throwing a UsageError if something goes wrong. */ - void parseCmdline(const std::string & argv0, const Strings & cmdline); + void parseCmdline(const Strings & _cmdline, bool allowShebang); /** * Return a short one-line description of the command. diff --git a/src/nix/main.cc b/src/nix/main.cc index 16fb50806..73641f6d2 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -428,7 +429,9 @@ void mainWrapped(int argc, char * * argv) }); try { - args.parseCmdline(programName, argvToStrings(argc, argv)); + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + auto allowShebang = isNixCommand && argc > 1; + args.parseCmdline(argvToStrings(argc, argv),allowShebang); } catch (UsageError &) { if (!args.helpRequested && !args.completions) throw; } From cc68ed8ff7b9e3898308a39dfdad2660bacc153f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:42:49 +0200 Subject: [PATCH 308/402] libcmd: lookupFileArg(): add baseDir This will allow a different base directory to be used, matching a shebang script location instead of the working directory. --- src/libcmd/common-eval-args.cc | 4 ++-- src/libcmd/common-eval-args.hh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 91fa881b1..401acc38e 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -164,7 +164,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -SourcePath lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( @@ -185,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) } else - return state.rootPath(CanonPath::fromCwd(s)); + return state.rootPath(CanonPath(s, baseDir)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 6359b2579..4b403d936 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -2,6 +2,7 @@ ///@file #include "args.hh" +#include "canon-path.hh" #include "common-args.hh" #include "search-path.hh" @@ -28,6 +29,6 @@ private: std::map autoArgs; }; -SourcePath lookupFileArg(EvalState & state, std::string_view s); +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd()); } From 20ff61ab252fc1d2bd69987f51a000739b24c670 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:46:37 +0200 Subject: [PATCH 309/402] nix: Reserve shebang line syntax and only parse double backtick quotes Being restrictive about syntax leaves opportunity to improve the syntax and functionality later. --- doc/manual/src/release-notes/rl-next.md | 11 +- src/libutil/args.cc | 152 +++++++++++++++++++++++- src/libutil/util.cc | 43 ------- src/libutil/util.hh | 5 - src/nix/shell.md | 8 +- tests/functional/flakes/flakes.sh | 16 ++- 6 files changed, 177 insertions(+), 58 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 93d4f432b..4bff3c685 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,10 +2,13 @@ - The experimental nix command is now a `#!-interpreter` by appending the contents of any `#! nix` lines and the script's location to a single call. + + Verbatim strings may be passed in double backtick (```` `` ````) quotes. + Some examples: ``` #!/usr/bin/env nix - #! nix shell --file "" hello --command bash + #! nix shell --file ```` hello --command bash hello | cowsay ``` @@ -19,8 +22,10 @@ or ```bash #! /usr/bin/env nix - #! nix shell --impure --expr - #! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" + #! nix shell --impure --expr `` + #! nix with (import (builtins.getFlake "nixpkgs") {}); + #! nix terraform.withPlugins (plugins: [ plugins.openstack ]) + #! nix `` #! nix --command bash terraform "$@" diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 481ed33ff..ab6e0e266 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -86,6 +86,147 @@ void RootArgs::parseCmdline(const Strings & _cmdline) Args::parseCmdline(_cmdline, false); } +/** + * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` + * + * Except we can't recursively reference the Parser typedef, so we have to write a class. + */ +struct Parser { + std::string_view remaining; + + /** + * @brief Parse the next character(s) + * + * @param r + * @return std::shared_ptr + */ + virtual void operator()(std::shared_ptr & state, Strings & r) = 0; + + Parser(std::string_view s) : remaining(s) {}; +}; + +struct ParseQuoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. + */ + std::string acc; + + ParseQuoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override; +}; + + +struct ParseUnquoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. Empty string is not representable in + * unquoted syntax, so we use it for the initial state. + */ + std::string acc; + + ParseUnquoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override { + if (remaining.empty()) { + if (!acc.empty()) + r.push_back(acc); + state = nullptr; // done + return; + } + switch (remaining[0]) { + case ' ': case '\t': case '\n': case '\r': + if (!acc.empty()) + r.push_back(acc); + state = std::make_shared(ParseUnquoted(remaining.substr(1))); + return; + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseQuoted(remaining.substr(2))); + return; + } + else + throw Error("single backtick is not a supported syntax in the nix shebang."); + + // reserved characters + // meaning to be determined, or may be reserved indefinitely so that + // #!nix syntax looks unambiguous + case '$': + case '*': + case '~': + case '<': + case '>': + case '|': + case ';': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\'': + case '"': + case '\\': + throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?"); + + case '#': + if (acc.empty()) { + throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?"); + } else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); + } +}; + +void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { + if (remaining.empty()) { + throw Error("unterminated quoted string in nix shebang"); + } + switch (remaining[0]) { + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseUnquoted(remaining.substr(2))); + r.push_back(acc); + return; + } + else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); +} + +static Strings parseShebangContent(std::string_view s) { + Strings result; + std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); + + // trampoline == iterated strategy pattern + while (parserState) { + auto currentState = parserState; + (*currentState)(parserState, result); + } + + return result; +} + void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; @@ -121,13 +262,18 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) std::string line; std::getline(stream,line); static const std::string commentChars("#/\\%@*-"); + std::string shebangContent; while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix\\s(.*)$"))) - for (const auto & word : shellwords(match[1].str())) - cmdline.push_back(word); + // We match one space after `nix` so that we preserve indentation. + // No space is necessary for an empty line. An empty line has basically no effect. + if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) + shebangContent += match[2].str() + "\n"; + } + for (const auto & word : parseShebangContent(shebangContent)) { + cmdline.push_back(word); } cmdline.push_back(script); for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6ca1dbd7a..5bb3f374b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -138,49 +138,6 @@ std::string shellEscape(const std::string_view s) return r; } -/* Recreate the effect of the perl shellwords function, breaking up a - * string into arguments like a shell word, including escapes - */ -std::vector shellwords2(const std::string & s) -{ - std::regex whitespace("^(\\s+).*"); - auto begin = s.cbegin(); - std::vector res; - std::string cur; - enum state { - sBegin, - sQuote - }; - state st = sBegin; - auto it = begin; - for (; it != s.cend(); ++it) { - if (st == sBegin) { - std::smatch match; - if (regex_search(it, s.cend(), match, whitespace)) { - cur.append(begin, it); - res.push_back(cur); - cur.clear(); - it = match[1].second; - begin = it; - } - } - switch (*it) { - case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; - break; - case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; - break; - } - } - cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); - return res; -} void ignoreException(Verbosity lvl) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b7d3ac504..27faa4d6d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -189,11 +189,6 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/* Recreate the effect of the perl shellwords function, breaking up a - string into arguments like a shell word, including escapes. */ -std::vector shellwords2(const std::string & s); - - /* Exception handling in destructors: print an error message, then ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); diff --git a/src/nix/shell.md b/src/nix/shell.md index 7e0e5f213..7c315fb3f 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -109,8 +109,10 @@ package like Terraform: ```bash #! /usr/bin/env nix -#! nix shell --impure --expr -#! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake ''nixpkgs'') {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` #! nix --command bash terraform "$@" @@ -118,7 +120,7 @@ terraform "$@" > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use double backticks (```` `` ````) when passing a simple Nix expression > in a nix shell shebang. Finally, using the merging of multiple nix shell shebangs the following diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 28b5e4e0f..a0a34ffa9 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,6 +80,7 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' +# this also tests a fairly trivial double backtick quoted string, ``--command`` cat > $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-reject.sh <&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 198bc22e3b856bf2a86225c2ce5b3a7394e3ac0c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:54:54 +0200 Subject: [PATCH 310/402] nix: Add command baseDir to parse --expr relative to shebang script --- doc/manual/src/release-notes/rl-next.md | 1 + src/libcmd/installables.cc | 3 ++- src/libutil/args.cc | 9 +++++++++ src/libutil/args.hh | 20 ++++++++++++++++++++ tests/functional/flakes/flakes.sh | 19 +++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 4bff3c685..28b6d75f5 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,6 +4,7 @@ contents of any `#! nix` lines and the script's location to a single call. Verbatim strings may be passed in double backtick (```` `` ````) quotes. + `--expr` resolves relative paths based on the shebang script location. Some examples: ``` diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index e7f58556f..528643dc5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -445,7 +445,8 @@ Installables SourceExprCommand::parseInstallables( else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { - auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd())); + CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); + auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ab6e0e266..0012b3f47 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -276,6 +276,7 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) cmdline.push_back(word); } cmdline.push_back(script); + commandBaseDir = dirOf(script); for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) cmdline.push_back(*pos); } @@ -336,6 +337,14 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) d.completer(*completions, d.n, d.prefix); } +Path Args::getCommandBaseDir() const +{ + if (parent) + return parent->getCommandBaseDir(); + else + return commandBaseDir; +} + bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index e753dcaf6..9c942606e 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -24,6 +24,16 @@ class AddCompletions; class Args { + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + public: /** @@ -44,6 +54,16 @@ public: */ virtual std::string doc() { return ""; } + /** + * @brief Get the base directory for the command. + * + * @return Generally the working directory, but in case of a shebang + * interpreter, returns the directory of the script. + * + * This only returns the correct value after parseCmdline() has run. + */ + Path getCommandBaseDir() const; + protected: /** diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index a0a34ffa9..76f3495dd 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -105,6 +105,24 @@ foo EOF chmod +x $nonFlakeDir/shebang-reject.sh +cat > $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -552,4 +570,5 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock [[ $($nonFlakeDir/shebang.sh) = "foo" ]] [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] +[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 466271568be7d3bcf0151dc7e09899775ac31f13 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:56:04 +0200 Subject: [PATCH 311/402] nix: Parse --file relative to shebang script --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libcmd/installables.cc | 5 +++-- tests/functional/flakes/flakes.sh | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 28b6d75f5..7eae5d96e 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,7 +4,7 @@ contents of any `#! nix` lines and the script's location to a single call. Verbatim strings may be passed in double backtick (```` `` ````) quotes. - `--expr` resolves relative paths based on the shebang script location. + `--file` and `--expr` resolve relative paths based on the script location. Some examples: ``` diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 528643dc5..d897a01c4 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -442,8 +442,9 @@ Installables SourceExprCommand::parseInstallables( auto e = state->parseStdin(); state->eval(e, *vFile); } - else if (file) - state->evalFile(lookupFileArg(*state, *file), *vFile); + else if (file) { + state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile); + } else { CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); auto e = state->parseExprFromString(*expr, state->rootPath(dir)); diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 76f3495dd..ccf1699f9 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -123,6 +123,25 @@ echo "$@" EOF chmod +x $nonFlakeDir/shebang-inline-expr.sh +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -571,4 +590,5 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] [[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 51bb69535b76060582f91e5c044d5752d8e3998b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:57:36 +0200 Subject: [PATCH 312/402] nix/installables.cc: Use getCommandBaseDir() where possible These usages of the working directory are perhaps unlikely to interact with shebangs, but the code is more consistent this way, and we're less likely to miss usages that do interact. --- src/libcmd/installables.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index d897a01c4..f840865d2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -88,7 +88,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath("."), true)); + parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -130,7 +130,7 @@ MixFlakeOptions::MixFlakeOptions() auto evalState = getEvalState(); auto flake = flake::lockFlake( *evalState, - parseFlakeRef(flakeRef, absPath(".")), + parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes @@ -294,7 +294,7 @@ void completeFlakeRefWithFragment( prefixRoot = "."; } auto flakeRefS = std::string(prefix.substr(0, hash)); - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(getCommandBaseDir())); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); @@ -482,7 +482,7 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -756,7 +756,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( expandTilde(i), - absPath(".")).first); + absPath(getCommandBaseDir())).first); return res; } @@ -778,7 +778,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() return { parseFlakeRefWithFragment( expandTilde(_installable), - absPath(".")).first + absPath(getCommandBaseDir())).first }; } From e91fd837ee997cc1879cc9035158260f3dc7cf67 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 16:16:51 +0200 Subject: [PATCH 313/402] Move shebang docs from rl-next to nix.md --- doc/manual/src/release-notes/rl-next.md | 48 ------------------- src/nix/nix.md | 61 +++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7eae5d96e..608699270 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,54 +2,6 @@ - The experimental nix command is now a `#!-interpreter` by appending the contents of any `#! nix` lines and the script's location to a single call. - - Verbatim strings may be passed in double backtick (```` `` ````) quotes. - `--file` and `--expr` resolve relative paths based on the script location. - - Some examples: - ``` - #!/usr/bin/env nix - #! nix shell --file ```` hello --command bash - - hello | cowsay - ``` - or with flakes: - ``` - #!/usr/bin/env nix - #! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash - - hello | cowsay - ``` - or - ```bash - #! /usr/bin/env nix - #! nix shell --impure --expr `` - #! nix with (import (builtins.getFlake "nixpkgs") {}); - #! nix terraform.withPlugins (plugins: [ plugins.openstack ]) - #! nix `` - #! nix --command bash - - terraform "$@" - ``` - or - ``` - #!/usr/bin/env nix - //! ```cargo - //! [dependencies] - //! time = "0.1.25" - //! ``` - /* - #!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script - */ - fn main() { - for argument in std::env::args().skip(1) { - println!("{}", argument); - }; - println!("{}", std::env::var("HOME").expect("")); - println!("{}", time::now().rfc822z()); - } - // vim: ft=rust - ``` - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/nix.md b/src/nix/nix.md index 6e7e8a649..5bf82a8bf 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -238,4 +238,65 @@ operate are determined as follows: Most `nix` subcommands operate on a *Nix store*. These are documented in [`nix help-stores`](./nix3-help-stores.md). +# Shebang interpreter + +The `nix` command can be used as a `#!` interpreter. +Arguments to Nix can be passed on subsequent lines in the script. + +Verbatim strings may be passed in double backtick (```` `` ````) quotes. + +`--file` and `--expr` resolve relative paths based on the script location. + +Examples: + +``` +#!/usr/bin/env nix +#! nix shell --file ```` hello cowsay --command bash + +hello | cowsay +``` + +or with **flakes**: + +``` +#!/usr/bin/env nix +#! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + +hello | cowsay +``` + +or with an **expression**: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake "nixpkgs") {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters. + +``` +#!/usr/bin/env nix +//! ```cargo +//! [dependencies] +//! time = "0.1.25" +//! ``` +/* +#!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script +*/ +fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); +} +// vim: ft=rust +``` + )"" From ffd414eb756dcb3c64348551d5dbaf674c0d4900 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 18:38:54 +0200 Subject: [PATCH 314/402] Fix nix shebang interaction with #8131 overhaul completions --- src/libcmd/installables.cc | 4 +++- src/libutil/args.cc | 19 ++++++++----------- src/libutil/args.hh | 17 +---------------- src/libutil/args/root.hh | 14 +++++++++++++- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f840865d2..1c6103020 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -294,7 +294,9 @@ void completeFlakeRefWithFragment( prefixRoot = "."; } auto flakeRefS = std::string(prefix.substr(0, hash)); - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(getCommandBaseDir())); + + // TODO: ideally this would use the command base directory instead of assuming ".". + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 0012b3f47..5ba1e5c55 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -80,12 +80,6 @@ std::optional RootArgs::needsCompletion(std::string_view s) return {}; } -void RootArgs::parseCmdline(const Strings & _cmdline) -{ - // Default via 5.1.2.2.1 in C standard - Args::parseCmdline(_cmdline, false); -} - /** * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` * @@ -227,7 +221,7 @@ static Strings parseShebangContent(std::string_view s) { return result; } -void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) +void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -339,10 +333,13 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) Path Args::getCommandBaseDir() const { - if (parent) - return parent->getCommandBaseDir(); - else - return commandBaseDir; + assert(parent); + return parent->getCommandBaseDir(); +} + +Path RootArgs::getCommandBaseDir() const +{ + return commandBaseDir; } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 9c942606e..30a44cd10 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -24,24 +24,9 @@ class AddCompletions; class Args { - /** - * @brief The command's "working directory", but only set when top level. - * - * Use getCommandBaseDir() to get the directory regardless of whether this - * is a top-level command or subcommand. - * - * @see getCommandBaseDir() - */ - Path commandBaseDir = "."; public: - /** - * Parse the command line with argv0, throwing a UsageError if something - goes wrong. - */ - void parseCmdline(const Strings & _cmdline, bool allowShebang); - /** * Return a short one-line description of the command. */ @@ -62,7 +47,7 @@ public: * * This only returns the correct value after parseCmdline() has run. */ - Path getCommandBaseDir() const; + virtual Path getCommandBaseDir() const; protected: diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh index bb98732a1..5c55c37a5 100644 --- a/src/libutil/args/root.hh +++ b/src/libutil/args/root.hh @@ -29,14 +29,26 @@ struct Completions final : AddCompletions */ class RootArgs : virtual public Args { + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + public: /** Parse the command line, throwing a UsageError if something goes * wrong. */ - void parseCmdline(const Strings & cmdline); + void parseCmdline(const Strings & cmdline, bool allowShebang = false); std::shared_ptr completions; + Path getCommandBaseDir() const override; + protected: friend class Args; From 589d3387769b18de9c8d42035eea7ac1e21c6fde Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 6 Nov 2023 18:19:14 +0100 Subject: [PATCH 315/402] parseShebangs: Make strings with backtick sequences representable --- src/libutil/args.cc | 32 ++++++++++++- src/libutil/args.hh | 2 + src/libutil/tests/args.cc | 94 +++++++++++++++++++++++++++++++++++++++ src/nix/nix.md | 4 +- 4 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 src/libutil/tests/args.cc diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 5ba1e5c55..4359c5e8e 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -189,12 +189,40 @@ void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { throw Error("unterminated quoted string in nix shebang"); } switch (remaining[0]) { + case ' ': + if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`') + || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) { + // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present. + state = std::make_shared(ParseUnquoted(remaining.substr(3))); + r.push_back(acc); + return; + } + else { + // just a normal space + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } case '`': - if (remaining.size() > 1 && remaining[1] == '`') { + // exactly two backticks mark the end of a quoted string + if ((remaining.size() == 2 && remaining[1] == '`') + || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) { state = std::make_shared(ParseUnquoted(remaining.substr(2))); r.push_back(acc); return; } + + // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim + else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') { + // ignore "escape" backtick + remaining = remaining.substr(1); + // add the rest + while (remaining.size() > 0 && remaining[0] == '`') { + acc += '`'; + remaining = remaining.substr(1); + } + return; + } else { acc += remaining[0]; remaining = remaining.substr(1); @@ -208,7 +236,7 @@ void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { assert(false); } -static Strings parseShebangContent(std::string_view s) { +Strings parseShebangContent(std::string_view s) { Strings result; std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 30a44cd10..7af82b178 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -409,4 +409,6 @@ public: virtual void add(std::string completion, std::string description = "") = 0; }; +Strings parseShebangContent(std::string_view s); + } diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc new file mode 100644 index 000000000..e7a16b0be --- /dev/null +++ b/src/libutil/tests/args.cc @@ -0,0 +1,94 @@ +#include "../args.hh" +#include + +#include + +namespace nix { + + TEST(parseShebangContent, basic) { + std::list r = parseShebangContent("hi there"); + ASSERT_EQ(r.size(), 2); + auto i = r.begin(); + ASSERT_EQ(*i++, "hi"); + ASSERT_EQ(*i++, "there"); + } + + TEST(parseShebangContent, empty) { + std::list r = parseShebangContent(""); + ASSERT_EQ(r.size(), 0); + } + + TEST(parseShebangContent, doubleBacktick) { + std::list r = parseShebangContent("``\"ain't that nice\"``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "\"ain't that nice\""); + } + + TEST(parseShebangContent, doubleBacktickEmpty) { + std::list r = parseShebangContent("````"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, ""); + } + + TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) { + std::list r = parseShebangContent("``# I'm markdown section about `coolFunction` ``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) { + std::list r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "`"); + } + + TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "` "); + } + + TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "``"); + } + + TEST(parseShebangContent, increasingQuotes) { + std::list r = parseShebangContent("```` ``` `` ````` `` `````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 4); + ASSERT_EQ(*i++, ""); + ASSERT_EQ(*i++, "`"); + ASSERT_EQ(*i++, "``"); + ASSERT_EQ(*i++, "```"); + } + +} \ No newline at end of file diff --git a/src/nix/nix.md b/src/nix/nix.md index 5bf82a8bf..eb150f03b 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -243,7 +243,9 @@ in [`nix help-stores`](./nix3-help-stores.md). The `nix` command can be used as a `#!` interpreter. Arguments to Nix can be passed on subsequent lines in the script. -Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Sequences of _n_ backticks of 3 or longer are parsed as _n-1_ literal backticks. +A single space before the closing ```` `` ```` is ignored if present. `--file` and `--expr` resolve relative paths based on the script location. From ab69dc4da3ce5dc270e11b460c5b99f549bcf5d3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 6 Nov 2023 19:15:36 +0100 Subject: [PATCH 316/402] Test parseShebangContent round trip --- src/libutil/tests/args.cc | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc index e7a16b0be..bea74a8c8 100644 --- a/src/libutil/tests/args.cc +++ b/src/libutil/tests/args.cc @@ -1,7 +1,9 @@ #include "../args.hh" +#include "libutil/fs-sink.hh" #include #include +#include namespace nix { @@ -91,4 +93,76 @@ namespace nix { ASSERT_EQ(*i++, "```"); } + +#ifndef COVERAGE + +// quick and dirty +static inline std::string escape(std::string_view s_) { + + std::string_view s = s_; + std::string r = "``"; + + // make a guess to allocate ahead of time + r.reserve( + // plain chars + s.size() + // quotes + + 5 + // some "escape" backticks + + s.size() / 8); + + while (!s.empty()) { + if (s[0] == '`' && s.size() >= 2 && s[1] == '`') { + // escape it + r += "`"; + while (!s.empty() && s[0] == '`') { + r += "`"; + s = s.substr(1); + } + } else { + r += s[0]; + s = s.substr(1); + } + } + + if (!r.empty() + && ( + r[r.size() - 1] == '`' + || r[r.size() - 1] == ' ' + )) { + r += " "; + } + + r += "``"; + + return r; +}; + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_single, + (const std::string & orig)) +{ + auto escaped = escape(orig); + // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl; + auto ss = parseShebangContent(escaped); + RC_ASSERT(ss.size() == 1); + RC_ASSERT(*ss.begin() == orig); +} + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_two, + (const std::string & one, const std::string & two)) +{ + auto ss = parseShebangContent(escape(one) + " " + escape(two)); + RC_ASSERT(ss.size() == 2); + auto i = ss.begin(); + RC_ASSERT(*i++ == one); + RC_ASSERT(*i++ == two); +} + + +#endif + } \ No newline at end of file From c0c7c4b6cd1aefaa65fc11fcdc8df7e608960825 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 7 Nov 2023 21:16:18 +0100 Subject: [PATCH 317/402] Link to shebang interpreter docs from release notes --- doc/manual/src/release-notes/rl-next.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 608699270..da81ed83b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,7 +1,8 @@ # Release X.Y (202?-??-??) -- The experimental nix command is now a `#!-interpreter` by appending the - contents of any `#! nix` lines and the script's location to a single call. +- The experimental nix command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) + by appending the contents of any `#! nix` lines and the script's location to a single call. + - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). From 6a47629530469b84d33444119e43c61effa88aa4 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Nov 2023 13:38:52 +0100 Subject: [PATCH 318/402] Fix initialization of struct members (wrong order) --- src/libstore/nar-accessor.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 02993680f..58740b685 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -60,12 +60,22 @@ struct NarAccessor : public SourceAccessor void createDirectory(const Path & path) override { - createMember(path, {Type::tDirectory, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tDirectory, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void createRegularFile(const Path & path) override { - createMember(path, {Type::tRegular, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tRegular, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void closeRegularFile() override From 77dceb2844276217bff321d80f601297f3581530 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Nov 2023 13:39:10 +0100 Subject: [PATCH 319/402] Drop obsolete assert and cast --- src/libstore/nar-accessor.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 58740b685..cfbbbd80b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -88,9 +88,8 @@ struct NarAccessor : public SourceAccessor void preallocateContents(uint64_t size) override { - assert(size <= std::numeric_limits::max()); auto & st = parents.top()->stat; - st.fileSize = (uint64_t) size; + st.fileSize = size; st.narOffset = pos; } From c581143e0c6721fba455e6616e7c6f07e47000b1 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Nov 2023 13:39:30 +0100 Subject: [PATCH 320/402] Use structured binding for json iteration --- src/libstore/nar-accessor.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index cfbbbd80b..1a4936736 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -137,9 +137,8 @@ struct NarAccessor : public SourceAccessor if (type == "directory") { member.stat = {.type = Type::tDirectory}; - for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { - std::string name = i.key(); - recurse(member.children[name], i.value()); + for (const auto &[name, function] : v["entries"].items()) { + recurse(member.children[name], function); } } else if (type == "regular") { member.stat = { From df8bfe84cca62c89417d676af2c6fbe3bcf23412 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Nov 2023 13:40:21 +0100 Subject: [PATCH 321/402] Fix consts and casts --- src/libstore/nar-accessor.cc | 6 +++--- src/libstore/nar-accessor.hh | 2 +- src/libstore/nar-info.cc | 4 ++-- src/libstore/nar-info.hh | 4 ++-- src/libstore/worker-protocol.cc | 8 ++++---- src/libstore/worker-protocol.hh | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 1a4936736..15b05fe25 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -161,7 +161,7 @@ struct NarAccessor : public SourceAccessor { NarMember * current = &root; - for (auto & i : path) { + for (const auto & i : path) { if (current->stat.type != Type::tDirectory) return nullptr; auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; @@ -194,7 +194,7 @@ struct NarAccessor : public SourceAccessor throw Error("path '%1%' inside NAR file is not a directory", path); DirEntries res; - for (auto & child : i.children) + for (const auto & child : i.children) res.insert_or_assign(child.first, std::nullopt); return res; @@ -259,7 +259,7 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & [name, type] : accessor->readDirectory(path)) { + for (const auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { res2[name] = listNar(accessor, path + name, true); } else diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 433774524..0043897c6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -25,7 +25,7 @@ ref makeNarAccessor(Source & source); * readFile() method of the accessor to get the contents of files * inside the NAR. */ -typedef std::function GetNarBytes; +using GetNarBytes = std::function; ref makeLazyNarAccessor( const std::string & listing, diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ae2223fb0..1060a6c8b 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -38,12 +38,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt("expecting ':'"); + if (colon == s.npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt("expecting '\\n'"); + if (eol == s.npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index cec65ff70..fd538a7cd 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,10 +17,10 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + NarInfo(const Store & store, std::string name, ContentAddressWithReferences ca, Hash narHash) : ValidPathInfo(store, std::move(name), std::move(ca), narHash) { } - NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } + NarInfo(StorePath path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 1d202f8d1..7118558b1 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -31,14 +31,14 @@ std::optional WorkerProto::Serialise>::r void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) - conn.to << (uint8_t)0; + conn.to << uint8_t{0}; else { switch (*optTrusted) { case Trusted: - conn.to << (uint8_t)1; + conn.to << uint8_t{1}; break; case NotTrusted: - conn.to << (uint8_t)2; + conn.to << uint8_t{2}; break; default: assert(false); @@ -101,7 +101,7 @@ void WorkerProto::Serialise::write(const Store & store, Worker BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { BuildResult res; - res.status = (BuildResult::Status) readInt(conn.from); + res.status = static_cast(readInt(conn.from)); conn.from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.from diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index dcd54ad16..25d544ba7 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -171,7 +171,7 @@ enum struct WorkerProto::Op : uint64_t */ inline Sink & operator << (Sink & sink, WorkerProto::Op op) { - return sink << (uint64_t) op; + return sink << static_cast(op); } /** @@ -181,7 +181,7 @@ inline Sink & operator << (Sink & sink, WorkerProto::Op op) */ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) { - return s << (uint64_t) op; + return s << static_cast(op); } /** From d854e8696b549de15ac9960736a39302d7846ece Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Nov 2023 15:57:43 -0400 Subject: [PATCH 322/402] Specify the size of the experimental feature array in a more robust way See doc comment for details. --- src/libutil/experimental-features.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 47edca3a5..6b9427423 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,19 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +/** + * If two different PRs both add an experimental feature, and we just + * used a number for this, we *woudln't* get merge conflict and the + * counter will be incremented once instead of twice, causing a build + * failure. + * + * By instead defining this instead as 1 + the bottom experimental + * feature, we either have no issue at all if few features are not added + * at the end of the list, or a proper merge conflict if they are. + */ +constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); + +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", From f0adb72c238aa6f21c2f07fe2e434a3adcea975d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 Nov 2023 23:08:05 -0500 Subject: [PATCH 323/402] Mark `fetchTree` as unstable again As discussed in our last meeting, we need a bit more time, but we are "time boxing" the work left to do to ensure there is not unbounded delay. Rather than putting it back underneath `flakes`, though, put it underneath its own `fetch-tree` experimental feature (which `flakes` includes/implies). This signals our commitment to the plan to stabilize it first without waiting to go through the rest of Flakes, and also will give users a "release candidate" when we get closer to stabilization. This reverts commit 4112dd1fc93c9ff03a5a4e8be773c45ebefbbd1f. --- doc/manual/src/release-notes/rl-next.md | 3 ++- src/libexpr/primops/fetchTree.cc | 1 + src/libutil/config.cc | 8 ++++++-- src/libutil/experimental-features.cc | 15 +++++++++++++++ src/libutil/experimental-features.hh | 1 + tests/functional/config.sh | 3 ++- 6 files changed, 27 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index da81ed83b..addb7de71 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -17,7 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now marked as stable. +- `builtins.fetchTree` is now unstable under its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes. - The interface for creating and updating lock files has been overhauled: diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3717b9022..8031bf809 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -228,6 +228,7 @@ static RegisterPrimOp primop_fetchTree({ ``` )", .fun = prim_fetchTree, + .experimentalFeature = Xp::FetchTree, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e7901133..eddc4a588 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -330,9 +330,13 @@ template<> std::set BaseSetting res; for (auto & s : tokenizeString(str)) { - if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) + if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { res.insert(thisXpFeature.value()); - else + // FIXME: Replace this hack with a proper notion of + // experimental feature implications/dependencies. + if (thisXpFeature.value() == Xp::Flakes) + res.insert(Xp::FetchTree); + } else warn("unknown experimental feature '%s'", s); } return res; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 6b9427423..b0edbe185 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -74,6 +74,21 @@ constexpr std::array xpFeatureDetails flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. )", }, + { + .tag = Xp::FetchTree, + .name = "fetch-tree", + .description = R"( + Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. + + `fetchTree` exposes a larger suite of fetching functionality in a more systematic way. + The same fetching functionality is always used for for + [`flakes`](#xp-feature-flakes). + + This built-in was previously guarded by the `flakes` experimental feature because of that overlap, + but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. + Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation. + )", + }, { .tag = Xp::NixCommand, .name = "nix-command", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index f005cc9ee..f580fd030 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -20,6 +20,7 @@ enum struct ExperimentalFeature CaDerivations, ImpureDerivations, Flakes, + FetchTree, NixCommand, RecursiveNix, NoUrlLiterals, diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 723f575ed..0780c55d0 100644 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -50,7 +50,8 @@ exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] -[[ $exp_features == "flakes nix-command" ]] +# flakes implies fetch-tree +[[ $exp_features == "fetch-tree flakes nix-command" ]] # Test that it's possible to retrieve a single setting's value val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) From 12953b942c7752568070e0b703b448dd8f16f21b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 9 Nov 2023 07:08:56 +0100 Subject: [PATCH 324/402] Fixup docs --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libutil/config.cc | 2 -- src/libutil/experimental-features.cc | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index addb7de71..1e6ad6922 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -17,7 +17,7 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now unstable under its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes. - The interface for creating and updating lock files has been overhauled: diff --git a/src/libutil/config.cc b/src/libutil/config.cc index eddc4a588..96a0a4df8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -332,8 +332,6 @@ template<> std::set BaseSetting(str)) { if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { res.insert(thisXpFeature.value()); - // FIXME: Replace this hack with a proper notion of - // experimental feature implications/dependencies. if (thisXpFeature.value() == Xp::Flakes) res.insert(Xp::FetchTree); } else diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b0edbe185..88fb55713 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -80,9 +80,8 @@ constexpr std::array xpFeatureDetails .description = R"( Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. - `fetchTree` exposes a larger suite of fetching functionality in a more systematic way. - The same fetching functionality is always used for for - [`flakes`](#xp-feature-flakes). + `fetchTree` exposes a large suite of fetching functionality in a more systematic way. + The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. This built-in was previously guarded by the `flakes` experimental feature because of that overlap, but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. From 1d5a48240cd3c5b81939b0562141772323550d99 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 9 Nov 2023 23:10:42 -0500 Subject: [PATCH 325/402] `.editorconfig`: Also affect Perl FFI `xs` file This way `perl/lib/Nix/Store.xs` is affected. --- .editorconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 887ecadba..86360e658 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,7 +17,7 @@ indent_style = space indent_size = 2 # Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl}] +[*.{hpp,cc,hh,sh,pl,xs}] indent_style = space indent_size = 4 From 3d9d5dc18977d21a04299f4a37b366f9a1d32051 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 3 Nov 2023 00:57:19 -0400 Subject: [PATCH 326/402] Create `MemorySink` This is for writing to a `MemorySourceAccessor`. --- src/libutil/memory-source-accessor.cc | 56 +++++++++++++++++++++++++++ src/libutil/memory-source-accessor.hh | 25 ++++++++++++ 2 files changed, 81 insertions(+) diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index f34f6c091..78a4dd298 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -121,4 +121,60 @@ CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) return path; } + +using File = MemorySourceAccessor::File; + +void MemorySink::createDirectory(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Directory { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + + if (!std::holds_alternative(f->raw)) + throw Error("file '%s' is not a directory", path); +}; + +void MemorySink::createRegularFile(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (!(r = std::get_if(&f->raw))) + throw Error("file '%s' is not a regular file", path); +} + +void MemorySink::closeRegularFile() +{ + r = nullptr; +} + +void MemorySink::isExecutable() +{ + assert(r); + r->executable = true; +} + +void MemorySink::preallocateContents(uint64_t len) +{ + assert(r); + r->contents.reserve(len); +} + +void MemorySink::receiveContents(std::string_view data) +{ + assert(r); + r->contents += data; +} + +void MemorySink::createSymlink(const Path & path, const std::string & target) +{ + auto * f = dst.open(CanonPath{path}, File { File::Symlink { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * s = std::get_if(&f->raw)) + s->target = target; + else + throw Error("file '%s' is not a symbolic link", path); +} + } diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh index 014fa8098..b908f3713 100644 --- a/src/libutil/memory-source-accessor.hh +++ b/src/libutil/memory-source-accessor.hh @@ -1,4 +1,5 @@ #include "source-accessor.hh" +#include "fs-sink.hh" #include "variant-wrapper.hh" namespace nix { @@ -71,4 +72,28 @@ struct MemorySourceAccessor : virtual SourceAccessor CanonPath addFile(CanonPath path, std::string && contents); }; +/** + * Write to a `MemorySourceAccessor` at the given path + */ +struct MemorySink : ParseSink +{ + MemorySourceAccessor & dst; + + MemorySink(MemorySourceAccessor & dst) : dst(dst) { } + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; + + void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + MemorySourceAccessor::File::Regular * r; +}; + } From 9afa697ab61ea6bbbb0d88e629b62606681cc744 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 Nov 2023 00:30:55 -0500 Subject: [PATCH 327/402] Refactor bash test build system a bit The basic idea here is to separate a few intertwined notions: 1. Not all "run bash tests" are "install tests" 2. Not all "run bash tests" use `tests/functional/init.sh`, or any pre-test initialization at all. This will used in the next commit when we have a test that check unit test golden master data. Also, move our custom `PS4` from the test to the test runner, as it is part of how we want to display the tests, not the test themselves. Co-authored-by: Robert Hensing --- doc/manual/src/contributing/testing.md | 10 ++++---- mk/common-test.sh | 24 ++++++++++++++----- mk/debug-test.sh | 5 +++- mk/lib.mk | 7 +++--- mk/run-test.sh | 5 +++- mk/tests.mk | 21 +++++++++------- .../common/vars-and-functions.sh.in | 2 +- 7 files changed, 48 insertions(+), 26 deletions(-) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 3d75ebe7b..0b45b88a3 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -133,17 +133,17 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh -+ foo +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ++(${testName}.sh:1) foo output from foo -+ bar ++(${testName}.sh:2) bar output from bar ... ``` @@ -175,7 +175,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ... + gdb blash blub GNU gdb (GDB) 12.1 diff --git a/mk/common-test.sh b/mk/common-test.sh index 7ab25febf..00ccd1584 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,15 +1,27 @@ -test_dir=tests/functional +# Remove overall test dir (at most one of the two should match) and +# remove file extension. +test_name=$(echo -n "$test" | sed \ + -e "s|^unit-test-data/||" \ + -e "s|^tests/functional/||" \ + -e "s|\.sh$||" \ + ) -test=$(echo -n "$test" | sed -e "s|^$test_dir/||") - -TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') +TESTS_ENVIRONMENT=( + "TEST_NAME=$test_name" + 'NIX_REMOTE=' + 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' +) : ${BASH:=/usr/bin/env bash} +run () { + cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) +} + init_test () { - cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null + run "$init" 2>/dev/null > /dev/null } run_test_proper () { - cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) + run "$test" } diff --git a/mk/debug-test.sh b/mk/debug-test.sh index b5b628ecd..52482c01e 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,9 +3,12 @@ set -eu -o pipefail test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -(init_test) +if [ -n "$init" ]; then + (init_test) +fi run_test_proper diff --git a/mk/lib.mk b/mk/lib.mk index e86a7f1a4..49abe9862 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -122,14 +122,15 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) +install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-install-test,$(test))) \ + $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-install-test-group,$(test-group))) \ + $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-install-test,$(test))) \ + $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval $(test-group).test-group: $(test).test))) $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/run-test.sh b/mk/run-test.sh index 1a1d65930..da9c5a473 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,6 +8,7 @@ yellow="" normal="" test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -21,7 +22,9 @@ if [ -t 1 ]; then fi run_test () { - (init_test 2>/dev/null > /dev/null) + if [ -n "$init" ]; then + (init_test 2>/dev/null > /dev/null) + fi log="$(run_test_proper 2>&1)" && status=0 || status=$? } diff --git a/mk/tests.mk b/mk/tests.mk index ec8128bdf..bac9b704a 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -2,19 +2,22 @@ test-deps = -define run-install-test +define run-bash - .PHONY: $1.test - $1.test: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null - - .PHONY: $1.test-debug - $1.test-debug: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null + .PHONY: $1 + $1: $2 + @env BASH=$(bash) $(bash) $3 < /dev/null endef -define run-install-test-group +define run-test + + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + +endef + +define run-test-group .PHONY: $1.test-group diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in index 967d6be54..848988af9 100644 --- a/tests/functional/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' +set +x export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} export NIX_STORE_DIR From 20b95d622336cf982082d7daf3075339f6edce70 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Nov 2023 15:35:38 -0400 Subject: [PATCH 328/402] Git object hashing in libutil This is the core functionality but just unit-tested and not yet made part of the store layer. This is because there is some tech debt around (a) repeated boilerplate hashing objects (b) better integration of the new `SourceAccessor` type that needs to be cleaned up first. Part of RFC 133 Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera Co-authored-by: Robert Hensing Co-authored-by: Florian Klink --- src/libutil/experimental-features.cc | 8 + src/libutil/experimental-features.hh | 1 + src/libutil/git.cc | 263 +++++++++++++++++- src/libutil/git.hh | 141 +++++++++- src/libutil/serialise.cc | 4 + src/libutil/serialise.hh | 1 + src/libutil/tests/git.cc | 249 +++++++++++++++-- src/libutil/tests/local.mk | 4 + unit-test-data/libutil/git/check-data.sh | 31 +++ .../libutil/git/hello-world-blob.bin | Bin 0 -> 24 bytes unit-test-data/libutil/git/hello-world.bin | Bin 0 -> 16 bytes unit-test-data/libutil/git/tree.bin | Bin 0 -> 100 bytes unit-test-data/libutil/git/tree.txt | 3 + 13 files changed, 667 insertions(+), 38 deletions(-) create mode 100644 unit-test-data/libutil/git/check-data.sh create mode 100644 unit-test-data/libutil/git/hello-world-blob.bin create mode 100644 unit-test-data/libutil/git/hello-world.bin create mode 100644 unit-test-data/libutil/git/tree.bin create mode 100644 unit-test-data/libutil/git/tree.txt diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 88fb55713..ac4d189e1 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -96,6 +96,14 @@ constexpr std::array xpFeatureDetails [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", }, + { + .tag = Xp::GitHashing, + .name = "git-hashing", + .description = R"( + Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm. + These store objects will not be understandable by older versions of Nix. + )", + }, { .tag = Xp::RecursiveNix, .name = "recursive-nix", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index f580fd030..c355b8081 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -22,6 +22,7 @@ enum struct ExperimentalFeature Flakes, FetchTree, NixCommand, + GitHashing, RecursiveNix, NoUrlLiterals, FetchClosure, diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f35c2fdb7..a4bd60096 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,9 +1,263 @@ -#include "git.hh" - +#include +#include +#include +#include #include +#include // for strcasecmp + +#include "signals.hh" +#include "config.hh" +#include "hash.hh" +#include "posix-source-accessor.hh" + +#include "git.hh" +#include "serialise.hh" + +namespace nix::git { + +using namespace nix; +using namespace std::string_literals; + +std::optional decodeMode(RawMode m) { + switch (m) { + case (RawMode) Mode::Directory: + case (RawMode) Mode::Executable: + case (RawMode) Mode::Regular: + case (RawMode) Mode::Symlink: + return (Mode) m; + default: + return std::nullopt; + } +} + + +static std::string getStringUntil(Source & source, char byte) +{ + std::string s; + char n[1]; + source(std::string_view { n, 1 }); + while (*n != byte) { + s += *n; + source(std::string_view { n, 1 }); + } + return s; +} + + +static std::string getString(Source & source, int n) +{ + std::string v; + v.resize(n); + source(v); + return v; +} + + +void parse( + ParseSink & sink, + const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = getString(source, 5); + + if (type == "blob ") { + sink.createRegularFile(sinkPath); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::string buf; + buf.reserve(65536); + + while (left) { + checkInterrupt(); + buf.resize(std::min((unsigned long long)buf.capacity(), left)); + source(buf); + sink.receiveContents(buf); + left -= buf.size(); + } + } else if (type == "tree ") { + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(sinkPath); + + while (left) { + std::string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + RawMode rawMode = std::stoi(perms, 0, 8); + auto modeOpt = decodeMode(rawMode); + if (!modeOpt) + throw Error("Unknown Git permission: %o", perms); + auto mode = std::move(*modeOpt); + + std::string name = getStringUntil(source, '\0'); + left -= name.size(); + left -= 1; + + std::string hashs = getString(source, 20); + left -= 20; + + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + hook(name, TreeEntry { + .mode = mode, + .hash = hash, + }); + + if (mode == Mode::Executable) + sink.isExecutable(); + } + } else throw Error("input doesn't look like a Git object"); +} + + +std::optional convertMode(SourceAccessor::Type type) +{ + switch (type) { + case SourceAccessor::tSymlink: return Mode::Symlink; + case SourceAccessor::tRegular: return Mode::Regular; + case SourceAccessor::tDirectory: return Mode::Directory; + case SourceAccessor::tMisc: return std::nullopt; + default: abort(); + } +} + + +void restore(ParseSink & sink, Source & source, std::function hook) +{ + parse(sink, "", source, [&](Path name, TreeEntry entry) { + auto [accessor, from] = hook(entry.hash); + auto stat = accessor->lstat(from); + auto gotOpt = convertMode(stat.type); + if (!gotOpt) + throw Error("file '%s' (git hash %s) has an unsupported type", + from, + entry.hash.to_string(HashFormat::Base16, false)); + auto & got = *gotOpt; + if (got != entry.mode) + throw Error("git mode of file '%s' (git hash %s) is %o but expected %o", + from, + entry.hash.to_string(HashFormat::Base16, false), + (RawMode) got, + (RawMode) entry.mode); + copyRecursive( + *accessor, from, + sink, name); + }); +} + + +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + auto s = fmt("blob %d\0"s, std::to_string(size)); + sink(s); +} + + +void dumpTree(const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + std::string v1; + + for (auto & [name, entry] : entries) { + auto name2 = name; + if (entry.mode == Mode::Directory) { + assert(name2.back() == '/'); + name2.pop_back(); + } + v1 += fmt("%o %s\0"s, static_cast(entry.mode), name2); + std::copy(entry.hash.hash, entry.hash.hash + entry.hash.hashSize, std::back_inserter(v1)); + } + + { + auto s = fmt("tree %d\0"s, v1.size()); + sink(s); + } + + sink(v1); +} + + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter, + const ExperimentalFeatureSettings & xpSettings) +{ + auto st = accessor.lstat(path); + + switch (st.type) { + case SourceAccessor::tRegular: + { + accessor.readFile(path, sink, [&](uint64_t size) { + dumpBlobPrefix(size, sink, xpSettings); + }); + return st.isExecutable + ? Mode::Executable + : Mode::Regular; + } + + case SourceAccessor::tDirectory: + { + Tree entries; + for (auto & [name, _] : accessor.readDirectory(path)) { + auto child = path + name; + if (!filter(child.abs())) continue; + + auto entry = hook(child); + + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += "/"; + + entries.insert_or_assign(std::move(name2), std::move(entry)); + } + dumpTree(entries, sink, xpSettings); + return Mode::Directory; + } + + case SourceAccessor::tSymlink: + case SourceAccessor::tMisc: + default: + throw Error("file '%1%' has an unsupported type", path); + } +} + + +TreeEntry dumpHash( + HashType ht, + SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) +{ + std::function hook; + hook = [&](const CanonPath & path) -> TreeEntry { + auto hashSink = HashSink(ht); + auto mode = dump(accessor, path, hashSink, hook, filter); + auto hash = hashSink.finish().first; + return { + .mode = mode, + .hash = hash, + }; + }; + + return hook(path); +} -namespace nix { -namespace git { std::optional parseLsRemoteLine(std::string_view line) { @@ -22,4 +276,3 @@ std::optional parseLsRemoteLine(std::string_view line) } } -} diff --git a/src/libutil/git.hh b/src/libutil/git.hh index bf2b9a286..303460072 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -5,9 +5,127 @@ #include #include -namespace nix { +#include "types.hh" +#include "serialise.hh" +#include "hash.hh" +#include "source-accessor.hh" +#include "fs-sink.hh" -namespace git { +namespace nix::git { + +using RawMode = uint32_t; + +enum struct Mode : RawMode { + Directory = 0040000, + Executable = 0100755, + Regular = 0100644, + Symlink = 0120000, +}; + +std::optional decodeMode(RawMode m); + +/** + * An anonymous Git tree object entry (no name part). + */ +struct TreeEntry +{ + Mode mode; + Hash hash; + + GENERATE_CMP(TreeEntry, me->mode, me->hash); +}; + +/** + * A Git tree object, fully decoded and stored in memory. + * + * Directory names must end in a `/` for sake of sorting. See + * https://github.com/mirage/irmin/issues/352 + */ +using Tree = std::map; + +/** + * Callback for processing a child hash with `parse` + * + * The function should + * + * 1. Obtain the file system objects denoted by `gitHash` + * + * 2. Ensure they match `mode` + * + * 3. Feed them into the same sink `parse` was called with + * + * Implementations may seek to memoize resources (bandwidth, storage, + * etc.) for the same Git hash. + */ +using SinkHook = void(const Path & name, TreeEntry entry); + +void parse( + ParseSink & sink, const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Assists with writing a `SinkHook` step (2). + */ +std::optional convertMode(SourceAccessor::Type type); + +/** + * Simplified version of `SinkHook` for `restore`. + * + * Given a `Hash`, return a `SourceAccessor` and `CanonPath` pointing to + * the file system object with that path. + */ +using RestoreHook = std::pair(Hash); + +/** + * Wrapper around `parse` and `RestoreSink` + */ +void restore(ParseSink & sink, Source & source, std::function hook); + +/** + * Dumps a single file to a sink + * + * @param xpSettings for testing purposes + */ +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Dumps a representation of a git tree to a sink + */ +void dumpTree( + const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Callback for processing a child with `dump` + * + * The function should return the Git hash and mode of the file at the + * given path in the accessor passed to `dump`. + * + * Note that if the child is a directory, its child in must also be so + * processed in order to compute this information. + */ +using DumpHook = TreeEntry(const CanonPath & path); + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter = defaultPathFilter, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Recursively dumps path, hashing as we go. + * + * A smaller wrapper around `dump`. + */ +TreeEntry dumpHash( + HashType ht, + SourceAccessor & accessor, const CanonPath & path, + PathFilter & filter = defaultPathFilter); /** * A line from the output of `git ls-remote --symref`. @@ -16,15 +134,17 @@ namespace git { * * - Symbolic references of the form * - * ref: {target} {reference} - * - * where {target} is itself a reference and {reference} is optional + * ``` + * ref: {target} {reference} + * ``` + * where {target} is itself a reference and {reference} is optional * * - Object references of the form * - * {target} {reference} - * - * where {target} is a commit id and {reference} is mandatory + * ``` + * {target} {reference} + * ``` + * where {target} is a commit id and {reference} is mandatory */ struct LsRemoteRefLine { enum struct Kind { @@ -36,8 +156,9 @@ struct LsRemoteRefLine { std::optional reference; }; +/** + * Parse an `LsRemoteRefLine` + */ std::optional parseLsRemoteLine(std::string_view line); } - -} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 725ddbb8d..d7950b11b 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len) } } +void Source::operator () (std::string_view data) +{ + (*this)((char *)data.data(), data.size()); +} void Source::drainInto(Sink & sink) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 9e07226bf..3f57ce88b 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -73,6 +73,7 @@ struct Source * an error if it is not going to be available. */ void operator () (char * data, size_t len); + void operator () (std::string_view data); /** * Store up to ‘len’ in the buffer pointed to by ‘data’, and diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc index 5b5715fc2..2842ea4d0 100644 --- a/src/libutil/tests/git.cc +++ b/src/libutil/tests/git.cc @@ -1,33 +1,236 @@ -#include "git.hh" #include +#include "git.hh" +#include "memory-source-accessor.hh" + +#include "tests/characterization.hh" + namespace nix { - TEST(GitLsRemote, parseSymrefLineWithReference) { - auto line = "ref: refs/head/main HEAD"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, "HEAD"); +using namespace git; + +class GitTest : public CharacterizationTest +{ + Path unitTestData = getUnitTestData() + "/libutil/git"; + +public: + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; } - TEST(GitLsRemote, parseSymrefLineWithNoReference) { - auto line = "ref: refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, std::nullopt); - } + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; - TEST(GitLsRemote, parseObjectRefLine) { - auto line = "abc123 refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object); - ASSERT_EQ(res->target, "abc123"); - ASSERT_EQ(res->reference, "refs/head/main"); +private: + + void SetUp() override + { + mockXpSettings.set("experimental-features", "git-hashing"); } +}; + +TEST(GitMode, gitMode_directory) { + Mode m = Mode::Directory; + RawMode r = 0040000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_executable) { + Mode m = Mode::Executable; + RawMode r = 0100755; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_regular) { + Mode m = Mode::Regular; + RawMode r = 0100644; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_symlink) { + Mode m = Mode::Symlink; + RawMode r = 0120000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST_F(GitTest, blob_read) { + readTest("hello-world-blob.bin", [&](const auto & encoded) { + StringSource in { encoded }; + StringSink out; + RegularFileSink out2 { out }; + parse(out2, "", in, [](auto &, auto) {}, mockXpSettings); + + auto expected = readFile(goldenMaster("hello-world.bin")); + + ASSERT_EQ(out.s, expected); + }); } +TEST_F(GitTest, blob_write) { + writeTest("hello-world-blob.bin", [&]() { + auto decoded = readFile(goldenMaster("hello-world.bin")); + StringSink s; + dumpBlobPrefix(decoded.size(), s, mockXpSettings); + s(decoded); + return s.s; + }); +} + +/** + * This data is for "shallow" tree tests. However, we use "real" hashes + * so that we can check our test data in the corresponding functional + * test (`git-hashing/unit-test-data`). + */ +const static Tree tree = { + { + "Foo", + { + .mode = Mode::Regular, + // hello world with special chars from above + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + }, + }, + { + "bAr", + { + .mode = Mode::Executable, + // ditto + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + }, + }, + { + "baZ/", + { + .mode = Mode::Directory, + // Empty directory hash + .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1), + }, + }, +}; + +TEST_F(GitTest, tree_read) { + readTest("tree.bin", [&](const auto & encoded) { + StringSource in { encoded }; + NullParseSink out; + Tree got; + parse(out, "", in, [&](auto & name, auto entry) { + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += '/'; + got.insert_or_assign(name2, std::move(entry)); + }, mockXpSettings); + + ASSERT_EQ(got, tree); + }); +} + +TEST_F(GitTest, tree_write) { + writeTest("tree.bin", [&]() { + StringSink s; + dumpTree(tree, s, mockXpSettings); + return s.s; + }); +} + +TEST_F(GitTest, both_roundrip) { + using File = MemorySourceAccessor::File; + + MemorySourceAccessor files; + files.root = File::Directory { + .contents { + { + "foo", + File::Regular { + .contents = "hello\n\0\n\tworld!", + }, + }, + { + "bar", + File::Directory { + .contents = { + { + "baz", + File::Regular { + .executable = true, + .contents = "good day,\n\0\n\tworld!", + }, + }, + }, + }, + }, + }, + }; + + std::map cas; + + std::function dumpHook; + dumpHook = [&](const CanonPath & path) { + StringSink s; + HashSink hashSink { htSHA1 }; + TeeSink s2 { s, hashSink }; + auto mode = dump( + files, path, s2, dumpHook, + defaultPathFilter, mockXpSettings); + auto hash = hashSink.finish().first; + cas.insert_or_assign(hash, std::move(s.s)); + return TreeEntry { + .mode = mode, + .hash = hash, + }; + }; + + auto root = dumpHook(CanonPath::root); + + MemorySourceAccessor files2; + + MemorySink sinkFiles2 { files2 }; + + std::function mkSinkHook; + mkSinkHook = [&](const Path prefix, const Hash & hash) { + StringSource in { cas[hash] }; + parse(sinkFiles2, prefix, in, [&](const Path & name, const auto & entry) { + mkSinkHook(prefix + "/" + name, entry.hash); + }, mockXpSettings); + }; + + mkSinkHook("", root.hash); + + ASSERT_EQ(files, files2); +} + +TEST(GitLsRemote, parseSymrefLineWithReference) { + auto line = "ref: refs/head/main HEAD"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, "HEAD"); +} + +TEST(GitLsRemote, parseSymrefLineWithNoReference) { + auto line = "ref: refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, std::nullopt); +} + +TEST(GitLsRemote, parseObjectRefLine) { + auto line = "abc123 refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Object); + ASSERT_EQ(res->target, "abc123"); + ASSERT_EQ(res->reference, "refs/head/main"); +} + +} diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 167915439..5a970c0f2 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -27,3 +27,7 @@ libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: unit-test-data/libutil/git/check-data.sh.test + +$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh)) diff --git a/unit-test-data/libutil/git/check-data.sh b/unit-test-data/libutil/git/check-data.sh new file mode 100644 index 000000000..68b705c95 --- /dev/null +++ b/unit-test-data/libutil/git/check-data.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data +mkdir -p $TEST_ROOT + +repo="$TEST_ROOT/scratch" +git init "$repo" + +git -C "$repo" config user.email "you@example.com" +git -C "$repo" config user.name "Your Name" + +# `-w` to write for tree test +freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin") +encodingHash=$(sha1sum -b < "./hello-world-blob.bin" | head -c 40) + +# If the hashes match, then `hello-world-blob.bin` must be the encoding +# of `hello-world.bin`. +[[ "$encodingHash" == "$freshlyAddedHash" ]] + +# Create empty directory object for tree test +echo -n | git -C "$repo" hash-object -w -t tree --stdin + +# Relies on both child hashes already existing in the git store +freshlyAddedHash=$(git -C "$repo" mktree < "./tree.txt") +encodingHash=$(sha1sum -b < "./tree.bin" | head -c 40) + +# If the hashes match, then `tree.bin` must be the encoding of the +# directory denoted by `tree.txt` interpreted as git directory listing. +[[ "$encodingHash" == "$freshlyAddedHash" ]] diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/unit-test-data/libutil/git/hello-world-blob.bin new file mode 100644 index 0000000000000000000000000000000000000000..255f5df55ccedb2dae5f541d516896ffffcdb526 GIT binary patch literal 24 fcmYew$xl)+G-L2c&B@7E;9}t7R0z*6%1HqLQkDjQ literal 0 HcmV?d00001 diff --git a/unit-test-data/libutil/git/hello-world.bin b/unit-test-data/libutil/git/hello-world.bin new file mode 100644 index 0000000000000000000000000000000000000000..63ddb340119baf8492d2da53af47e8c7cfcd5eb2 GIT binary patch literal 16 XcmeZB&B@7E;9}t7R0z*6%1HqLBqsz~ literal 0 HcmV?d00001 diff --git a/unit-test-data/libutil/git/tree.bin b/unit-test-data/libutil/git/tree.bin new file mode 100644 index 0000000000000000000000000000000000000000..5256ec140702fef5f88bd5750caf7cd57c03e5ac GIT binary patch literal 100 zcmXRZN=;R;G-5C`FfcPQQE(Ikg(Sx! ktkNb1K%kJ67{%b-6no6+bl%Pd2~WL$T$|MK`<*8X01f;sp#T5? literal 0 HcmV?d00001 diff --git a/unit-test-data/libutil/git/tree.txt b/unit-test-data/libutil/git/tree.txt new file mode 100644 index 000000000..be3d02920 --- /dev/null +++ b/unit-test-data/libutil/git/tree.txt @@ -0,0 +1,3 @@ +100644 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 Foo +100755 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 bAr +040000 tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 baZ From fd5a4a846752873331b6549f0778181dc4ecc2f3 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 10 Nov 2023 12:12:13 -0500 Subject: [PATCH 329/402] nix upgrade-nix: make the source URL an option This new option enables organizations to more easily manage their Nix fleet's deployment, and ensure a consistent and planned rollout of Nix upgrades. --- src/libstore/globals.hh | 10 ++++++++++ src/nix/upgrade-nix.cc | 7 +++---- src/nix/upgrade-nix.md | 6 ++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8e034f5a9..27caf42c4 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1084,6 +1084,16 @@ public: true, // document default Xp::ConfigurableImpureEnv }; + + Setting upgradeNixStorePathUrl{ + this, + "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix", + "upgrade-nix-store-path-url", + R"( + Used by `nix upgrade-nix`, the URL of the file that contains the + store paths of the latest Nix release. + )" + }; }; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index c529c2363..4c7a74e16 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -14,7 +14,6 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { Path profileDir; - std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; CmdUpgradeNix() { @@ -30,7 +29,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand .longName = "nix-store-paths-url", .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, - .handler = {&storePathsUrl} + .handler = {&(std::string&) settings.upgradeNixStorePathUrl} }); } @@ -44,7 +43,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand std::string description() override { - return "upgrade Nix to the stable version declared in Nixpkgs"; + return "upgrade Nix to the latest stable version"; } std::string doc() override @@ -145,7 +144,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = FileTransferRequest(storePathsUrl); + auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); auto state = std::make_unique(SearchPath{}, store); diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index cce88c397..3a3bf61b9 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -16,8 +16,10 @@ R""( # Description -This command upgrades Nix to the stable version declared in Nixpkgs. -This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) +This command upgrades Nix to the stable version. + +By default, the latest stable version is defined by Nixpkgs, in +[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) and updated manually. It may not always be the latest tagged release. By default, it locates the directory containing the `nix` binary in the `$PATH` From 0be84c83b242b6e6a22400727752072b298e7cab Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 6 Nov 2023 10:29:37 -0500 Subject: [PATCH 330/402] key and cat: no need for progressBar otherwise the output will be invisible in common terminal configurations --- src/nix/cat.cc | 2 ++ src/nix/sigs.cc | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 6e5a736f2..4df086d4f 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "nar-accessor.hh" +#include "progress-bar.hh" using namespace nix; @@ -13,6 +14,7 @@ struct MixCat : virtual Args auto st = accessor->lstat(CanonPath(path)); if (st.type != SourceAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); + stopProgressBar(); writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a68616355..39555c9ea 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -3,6 +3,7 @@ #include "shared.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "progress-bar.hh" #include @@ -174,6 +175,7 @@ struct CmdKeyGenerateSecret : Command if (!keyName) throw UsageError("required argument '--key-name' is missing"); + stopProgressBar(); writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string()); } }; @@ -195,6 +197,7 @@ struct CmdKeyConvertSecretToPublic : Command void run() override { SecretKey secretKey(drainFD(STDIN_FILENO)); + stopProgressBar(); writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string()); } }; From 742a63b98f2008161fd00bdbbd39b8f1b14f6443 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:01:50 +0000 Subject: [PATCH 331/402] build(deps): bump zeebe-io/backport-action from 2.1.0 to 2.1.1 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.1.0 to 2.1.1. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v2.1.0...v2.1.1) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 893f4a56f..975c90b91 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.1.0 + uses: zeebe-io/backport-action@v2.1.1 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From ad99c8950b86b8f354f5c72efe690d3cba045d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20St=C3=BChrk?= Date: Mon, 13 Nov 2023 23:19:27 +0100 Subject: [PATCH 332/402] Update comment to reflect bind mounts are now used for store in chroot --- src/libstore/build/local-derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index adb011e30..a9f930773 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -652,8 +652,8 @@ void LocalDerivationGoal::startBuilder() #if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ + so that the build outputs can be moved efficiently from the + chroot to their final location. */ chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); From 4944cdb94d03742176cc7881f126e981c0e7e21c Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Tue, 14 Nov 2023 19:59:48 +0530 Subject: [PATCH 333/402] nar dump-path command renamed to nar pack --- src/nix/dump-path.cc | 10 +++++++++- src/nix/nar-dump-path.md | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index c4edc894b..0850d4c1c 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -61,4 +61,12 @@ struct CmdDumpPath2 : Command } }; -static auto rDumpPath2 = registerCommand2({"nar", "dump-path"}); +struct CmdNarDumpPath : CmdDumpPath2 { + void run() override { + warn("'nix nar dump-path' is a deprecated alias for 'nix nar pack'"); + CmdDumpPath2::run(); + } +}; + +static auto rCmdNarPack = registerCommand2({"nar", "pack"}); +static auto rCmdNarDumpPath = registerCommand2({"nar", "dump-path"}); diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index 26191ad25..29eaacfdb 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -5,7 +5,7 @@ R""( * To serialise directory `foo` as a NAR: ```console - # nix nar dump-path ./foo > foo.nar + # nix nar pack ./foo > foo.nar ``` # Description @@ -15,3 +15,4 @@ This command generates a NAR file containing the serialisation of symbolic links. The NAR is written to standard output. )"" + From e07e3c106a9ac0537210e62286c4e696573e9f6f Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Tue, 14 Nov 2023 20:02:33 +0530 Subject: [PATCH 334/402] code cleanup --- src/nix/nar-dump-path.md | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index 29eaacfdb..de82202de 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -15,4 +15,3 @@ This command generates a NAR file containing the serialisation of symbolic links. The NAR is written to standard output. )"" - From 9c7749e13508996eb9df83b1692664cc8cdbf952 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Nov 2023 11:42:25 -0500 Subject: [PATCH 335/402] Fix makefile bug confusing `libnixutil-test` exe vs lib The `-exe` variant is the program, the unsuffixed variant is the library. The corrected usage matches `libnixstore-test`. --- src/libutil/tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 5a970c0f2..c8b8557cb 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -1,6 +1,6 @@ check: libutil-tests_RUN -programs += libutil-tests +programs += libutil-tests-exe libutil-tests-exe_NAME = libnixutil-tests From 70b396649c127760e4b123da41451aa7456bc68d Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 15 Nov 2023 00:03:44 +0100 Subject: [PATCH 336/402] doc: logical implication is right-associative nix-repl> bools = [ false true ] nix-repl> combinations = builtins.concatMap (a: builtins.concatMap (b: map (c: { inherit a b c; }) bools) bools) bools nix-repl> builtins.all ({ a, b, c }: (a -> b -> c) == (a -> (b -> c))) combinations true nix-repl> builtins.all ({ a, b, c }: (a -> b -> c) == ((a -> b) -> c)) combinations false --- doc/manual/src/language/operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index cc825b4cf..e9cbb5c92 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -25,7 +25,7 @@ | Inequality | *expr* `!=` *expr* | none | 11 | | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | -| [Logical implication] | *bool* `->` *bool* | none | 14 | +| [Logical implication] | *bool* `->` *bool* | right | 14 | [string]: ./values.md#type-string [path]: ./values.md#type-path From 84128461b68f6274f1cbf309fd019959132f3c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 15 Nov 2023 09:23:26 +0100 Subject: [PATCH 337/402] Add a new `nix store add` command Deprecate `nix store add-file` and `nix store add-path`, and replace them with a single `nix store add` command. --- doc/manual/src/release-notes/rl-next.md | 2 + src/nix/add-file.md | 28 ---------- src/nix/add-to-store.cc | 70 +++++++++++++++++-------- src/nix/{add-path.md => add.md} | 2 +- tests/functional/add.sh | 17 ++++++ 5 files changed, 68 insertions(+), 51 deletions(-) delete mode 100644 src/nix/add-file.md rename src/nix/{add-path.md => add.md} (94%) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 1e6ad6922..422f1fce8 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -73,3 +73,5 @@ [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + +- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. diff --git a/src/nix/add-file.md b/src/nix/add-file.md deleted file mode 100644 index ed237a035..000000000 --- a/src/nix/add-file.md +++ /dev/null @@ -1,28 +0,0 @@ -R""( - -# Description - -Copy the regular file *path* to the Nix store, and print the resulting -store path on standard output. - -> **Warning** -> -> The resulting store path is not registered as a garbage -> collector root, so it could be deleted before you have a -> chance to register it. - -# Examples - -Add a regular file to the store: - -```console -# echo foo > bar - -# nix store add-file ./bar -/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - -# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar -foo -``` - -)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 39e5cc99d..f9d487ada 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -5,11 +5,22 @@ using namespace nix; +static FileIngestionMethod parseIngestionMethod(std::string_view input) +{ + if (input == "flat") { + return FileIngestionMethod::Flat; + } else if (input == "nar") { + return FileIngestionMethod::Recursive; + } else { + throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`"); + } +} + struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -23,6 +34,23 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "mode", + .shortName = 'n', + .description = R"( + How to compute the hash of the input. + One of: + + - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + + - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + )", + .labels = {"hash-mode"}, + .handler = {[this](std::string s) { + this->ingestionMethod = parseIngestionMethod(s); + }}, + }); } void run(ref store) override @@ -62,6 +90,22 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; +struct CmdAdd : CmdAddToStore +{ + + std::string description() override + { + return "Add a file or directory to the Nix store"; + } + + std::string doc() override + { + return + #include "add.md" + ; + } +}; + struct CmdAddFile : CmdAddToStore { CmdAddFile() @@ -71,36 +115,18 @@ struct CmdAddFile : CmdAddToStore std::string description() override { - return "add a regular file to the Nix store"; - } - - std::string doc() override - { - return - #include "add-file.md" - ; + return "Deprecated. Use [`nix store add --mode flat`](@docroot@/command-ref/new-cli/nix3-store-add.md) instead."; } }; struct CmdAddPath : CmdAddToStore { - CmdAddPath() - { - ingestionMethod = FileIngestionMethod::Recursive; - } - std::string description() override { - return "add a path to the Nix store"; - } - - std::string doc() override - { - return - #include "add-path.md" - ; + return "Deprecated alias to [`nix store add`](@docroot@/command-ref/new-cli/nix3-store-add.md)."; } }; static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); +static auto rCmdAdd = registerCommand2({"store", "add"}); diff --git a/src/nix/add-path.md b/src/nix/add.md similarity index 94% rename from src/nix/add-path.md rename to src/nix/add.md index 87473611d..d38cd21d8 100644 --- a/src/nix/add-path.md +++ b/src/nix/add.md @@ -19,7 +19,7 @@ Add a directory to the store: # mkdir dir # echo foo > dir/bar -# nix store add-path ./dir +# nix store add ./dir /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir # cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar diff --git a/tests/functional/add.sh b/tests/functional/add.sh index 5c3eed793..d0fedcb25 100644 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -26,3 +26,20 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" + +#### New style commands + +clearStore + +( + path1=$(nix store add ./dummy) + path2=$(nix store add --mode nar ./dummy) + path3=$(nix store add-path ./dummy) + [[ "$path1" == "$path2" ]] + [[ "$path1" == "$path3" ]] +) +( + path1=$(nix store add --mode flat ./dummy) + path2=$(nix store add-file ./dummy) + [[ "$path1" == "$path2" ]] +) From 5196613e8290a9ee81f1b9d88e7bc61cc3f64d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 18 Nov 2022 11:13:32 +0100 Subject: [PATCH 338/402] Use boost small vectors instead of VLAs in the primops VLAs are a dangerous feature, and their usage triggers an undefined behavior since theire size can be zero in some cases. So replace them with `boost::small_vector`s which fit the same goal but are safer. It's also incidentally consistently 1% faster on the benchmarks. --- src/libexpr/primops.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8d3a18526..e7587506a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -29,7 +29,6 @@ #include - namespace nix { @@ -2729,8 +2728,8 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - Value * res[args[1]->listSize()]; - unsigned int found = 0; + boost::container::small_vector res(args[1]->listSize()); + size_t found = 0; for (auto v2 : args[1]->listItems()) { state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); @@ -3064,9 +3063,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; - unsigned int k = 0; + boost::container::small_vector vs(args[1]->listSize()); + size_t k = 0; bool same = true; for (unsigned int n = 0; n < args[1]->listSize(); ++n) { @@ -3450,7 +3448,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - Value lists[nrLists]; + boost::container::small_vector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From ba3cb4a04949e043669299da5497bea27b944598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 18 Nov 2022 13:02:06 +0100 Subject: [PATCH 339/402] Remove all the occurences of VLAs There's generally no strict reason for using them, and they are somewhat fishy, so let's avoid them. --- src/libexpr/eval.cc | 13 ++++++++----- src/libstore/derivations.cc | 9 ++++----- src/libstore/gc.cc | 9 ++------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dfe81cbf7..d853b104b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -41,6 +41,7 @@ #include #include #include +#include #endif @@ -1691,7 +1692,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[arity]; + assert(arity < 64); + Value * vArgs[64]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1748,11 +1750,12 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) Value vFun; fun->eval(state, env, vFun); - Value * vArgs[args.size()]; + + boost::container::small_vector vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs, v, pos); + state.callFunction(vFun, args.size(), vArgs.data(), v, pos); } @@ -1991,8 +1994,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - Value values[es->size()]; - Value * vTmpP = values; + boost::container::small_vector values(es->size()); + Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1fecd1c97..6d9c8b9d6 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -151,11 +151,10 @@ StorePath writeDerivation(Store & store, /* Read string `s' from stream `str'. */ static void expect(std::istream & str, std::string_view s) { - char s2[s.size()]; - str.read(s2, s.size()); - std::string_view s2View { s2, s.size() }; - if (s2View != s) - throw FormatError("expected string '%s', got '%s'", s, s2View); + for (auto & c : s) { + if (str.get() != c) + throw FormatError("expected string '%1%'", s); + } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8d05ae4bd..ddec43fdc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -330,9 +330,7 @@ typedef std::unordered_map> UncheckedRoots static void readProcLink(const std::string & file, UncheckedRoots & roots) { - /* 64 is the starting buffer size gnu readlink uses... */ - auto bufsiz = ssize_t{64}; -try_again: + constexpr auto bufsiz = PATH_MAX; char buf[bufsiz]; auto res = readlink(file.c_str(), buf, bufsiz); if (res == -1) { @@ -341,10 +339,7 @@ try_again: throw SysError("reading symlink"); } if (res == bufsiz) { - if (SSIZE_MAX / 2 < bufsiz) - throw Error("stupidly long symlink"); - bufsiz *= 2; - goto try_again; + throw Error("stupidly long symlink"); } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] From 0daccb1121dfd5e98db3e41ba992b1b2c413dfc8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 11:10:25 +0100 Subject: [PATCH 340/402] libexpr: Check primop arity earlier --- src/libexpr/eval.cc | 20 ++++++++++++++++++-- src/libexpr/eval.hh | 12 ++++++++++++ src/libexpr/tests/value/print.cc | 3 ++- src/libexpr/value.hh | 8 +------- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d853b104b..1425eab97 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -723,6 +723,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) } +void PrimOp::check() +{ + if (arity > maxPrimOpArity) { + throw Error("primop arity must not exceed %1%", maxPrimOpArity); + } +} + + +void Value::mkPrimOp(PrimOp * p) +{ + p->check(); + clearValue(); + internalType = tPrimOp; + primOp = p; +} + + Value * EvalState::addPrimOp(PrimOp && primOp) { /* Hack to make constants lazy: turn them into a application of @@ -1692,8 +1709,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - assert(arity < 64); - Value * vArgs[64]; + Value * vArgs[maxPrimOpArity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 048dff42b..5ee6359a8 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -18,6 +18,12 @@ namespace nix { +/** + * We put a limit on primop arity because it lets us use a fixed size array on + * the stack. 16 is already an impractical number of arguments. Use an attrset + * argument for such overly complicated functions. + */ +constexpr size_t maxPrimOpArity = 64; class Store; class EvalState; @@ -71,6 +77,12 @@ struct PrimOp * Optional experimental for this to be gated on. */ std::optional experimentalFeature; + + /** + * Validity check to be performed by functions that introduce primops, + * such as RegisterPrimOp() and Value::mkPrimOp(). + */ + void check(); }; /** diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc index 5e96e12ec..a4f6fc014 100644 --- a/src/libexpr/tests/value/print.cc +++ b/src/libexpr/tests/value/print.cc @@ -114,7 +114,8 @@ TEST_F(ValuePrintingTests, vLambda) TEST_F(ValuePrintingTests, vPrimOp) { Value vPrimOp; - vPrimOp.mkPrimOp(nullptr); + PrimOp primOp{}; + vPrimOp.mkPrimOp(&primOp); test(vPrimOp, ""); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 622e613ea..191cc30ba 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -354,13 +354,7 @@ public: // Value will be overridden anyways } - inline void mkPrimOp(PrimOp * p) - { - clearValue(); - internalType = tPrimOp; - primOp = p; - } - + void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) { From 12c91a823e80b5e0a14a0abb0f34a6633b14bbfe Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 11:27:31 +0100 Subject: [PATCH 341/402] maxPrimOpArity: 64 -> 8 This makes stack usage significantly more compact, allowing larger amounts of data to be processed on the same stack. PrimOp functions with more than 8 positional (curried) arguments should use an attrset instead. --- src/libexpr/eval.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 5ee6359a8..ce798ed96 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -20,10 +20,10 @@ namespace nix { /** * We put a limit on primop arity because it lets us use a fixed size array on - * the stack. 16 is already an impractical number of arguments. Use an attrset + * the stack. 8 is already an impractical number of arguments. Use an attrset * argument for such overly complicated functions. */ -constexpr size_t maxPrimOpArity = 64; +constexpr size_t maxPrimOpArity = 8; class Store; class EvalState; From 9fa133dde5610dfb0605399ffea83081bda1c6fc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 11:33:10 +0100 Subject: [PATCH 342/402] readProcLink: Replace unnecessary value judgement by actual info --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ddec43fdc..93fa60682 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -339,7 +339,7 @@ static void readProcLink(const std::string & file, UncheckedRoots & roots) throw SysError("reading symlink"); } if (res == bufsiz) { - throw Error("stupidly long symlink"); + throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz)); } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] From 206ece0f41142536a856c62c49bd202282f12db8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 12:18:37 +0100 Subject: [PATCH 343/402] builtins.{any,all}: Use constant errorCtx Clang warned that the expanded code used to have a buffer overflow. Very strange, but also very avoidable. --- src/libexpr/primops.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e7587506a..d104b7180 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3189,10 +3189,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all")); state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all")); + std::string_view errorCtx = any + ? "while evaluating the return value of the function passed to builtins.any" + : "while evaluating the return value of the function passed to builtins.all"; + Value vTmp; for (auto elem : args[1]->listItems()) { state.callFunction(*args[0], *elem, vTmp, pos); - bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all")); + bool res = state.forceBool(vTmp, pos, errorCtx); if (res == any) { v.mkBool(any); return; From 91114a6fa48e2eb9399c23938eb12fdbd4fcda42 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 12:44:10 +0100 Subject: [PATCH 344/402] ExprCall::eval: Heap allocate at arity 5+ --- src/libexpr/eval.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1425eab97..bfbda52ef 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1766,8 +1766,13 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) Value vFun; fun->eval(state, env, vFun); - - boost::container::small_vector vArgs(args.size()); + // Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5} + // 2: over 4000 + // 3: about 300 + // 4: about 60 + // 5: under 10 + // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. + boost::container::small_vector vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); From 898c47384f651f51b3e4b63c271da274db8fca2e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 12:48:37 +0100 Subject: [PATCH 345/402] primops: Err on the side of less stack usage Try to stay away from stack overflows. These small vectors use stack space. Most instances will not need to allocate because in general most things are small, and large things are worth heap allocating. 16 * 3 * word = 384 bytes is still quite a bit, but these functions tend not to be part of deep recursions. --- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bfbda52ef..2fcbf3311 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2015,7 +2015,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - boost::container::small_vector values(es->size()); + boost::container::small_vector values(es->size()); Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d104b7180..7aa212281 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2549,7 +2549,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference can be used to remove them from attrs[0]. */ - boost::container::small_vector names; + boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); @@ -3452,7 +3452,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - boost::container::small_vector lists(nrLists); + boost::container::small_vector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From 1b9813e4e60836ddb1467efd50c572e7579ac945 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 13:04:27 +0100 Subject: [PATCH 346/402] primops: Name stack reservation limits --- src/libexpr/eval.cc | 3 ++- src/libexpr/primops.cc | 6 +++--- src/libexpr/primops.hh | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2fcbf3311..8b0ada517 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,6 +1,7 @@ #include "eval.hh" #include "eval-settings.hh" #include "hash.hh" +#include "primops.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" @@ -2015,7 +2016,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - boost::container::small_vector values(es->size()); + boost::container::small_vector values(es->size()); Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7aa212281..adce95bed 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2728,7 +2728,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - boost::container::small_vector res(args[1]->listSize()); + boost::container::small_vector res(args[1]->listSize()); size_t found = 0; for (auto v2 : args[1]->listItems()) { @@ -3063,7 +3063,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - boost::container::small_vector vs(args[1]->listSize()); + boost::container::small_vector vs(args[1]->listSize()); size_t k = 0; bool same = true; @@ -3452,7 +3452,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - boost::container::small_vector lists(nrLists); + boost::container::small_vector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 930e7f32a..1d5d5710d 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -8,6 +8,22 @@ namespace nix { +/** + * For functions where we do not expect deep recursion, we can use a sizable + * part of the stack a free allocation space. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t nonRecursiveStackReservation = 256; + +/** + * Functions that maybe applied to self-similar inputs, such as concatMap on a + * tree, should reserve a smaller part of the stack for allocation. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t conservativeStackReservation = 16; + struct RegisterPrimOp { typedef std::vector PrimOps; From a96be29db536177fdc284b51a3b2af44a70496e0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 13:13:01 +0100 Subject: [PATCH 347/402] removeAttrs: increase stack reservation to 64 --- src/libexpr/primops.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index adce95bed..e274c3c0c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2549,7 +2549,8 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference can be used to remove them from attrs[0]. */ - boost::container::small_vector names; + // 64: large enough to fit the attributes of a derivation + boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); From 4e27f1947a444a36d6a85f41cbf1afdc70ac6c4c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 13:23:17 +0100 Subject: [PATCH 348/402] libexpr: Reduce nonRecursiveStackReservation 128 is still beyond the point where the allocation overhead is insignificant, but we don't anticipate to overflow for these use cases, so it's fine. --- src/libexpr/primops.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 1d5d5710d..45486608f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -14,7 +14,7 @@ namespace nix { * * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. */ -constexpr size_t nonRecursiveStackReservation = 256; +constexpr size_t nonRecursiveStackReservation = 128; /** * Functions that maybe applied to self-similar inputs, such as concatMap on a From 6c8f4ef3502aa214557541ec00538e41aeced6e3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Nov 2023 10:52:57 -0500 Subject: [PATCH 349/402] Allow installing unit tests Closes #9343 See that issue for motivation. Installing these is disabled by default, but we enable it (and the additional output we want isntall these too so as not to clutter the existing ones) to use in cross builds and dev shells. --- Makefile.config.in | 3 +++ configure.ac | 12 ++++++++++++ flake.nix | 10 ++++++++-- src/libexpr/tests/local.mk | 6 +++++- src/libstore/tests/local.mk | 12 ++++++++++-- src/libutil/tests/local.mk | 12 ++++++++++-- 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 19992fa20..1482db81f 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -28,6 +28,8 @@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ +checkbindir = @checkbindir@ +checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ doc_generate = @doc_generate@ @@ -48,4 +50,5 @@ sysconfdir = @sysconfdir@ system = @system@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_TESTS = @ENABLE_TESTS@ +INSTALL_UNIT_TESTS = @INSTALL_UNIT_TESTS@ internal_api_docs = @internal_api_docs@ diff --git a/configure.ac b/configure.ac index 75ce7d01d..281ba2c32 100644 --- a/configure.ac +++ b/configure.ac @@ -167,6 +167,18 @@ AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) AC_SUBST(ENABLE_TESTS) +AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), + INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) +AC_SUBST(INSTALL_UNIT_TESTS) + +AC_ARG_WITH(check-bin-dir, AS_HELP_STRING([--with-check-bin-dir=PATH],[path to install unit tests for running later (defaults to $libexecdir/nix)]), + checkbindir=$withval, checkbindir=$libexecdir/nix) +AC_SUBST(checkbindir) + +AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to install unit tests for running later (defaults to $libdir)]), + checklibdir=$withval, checklibdir=$libdir) +AC_SUBST(checklibdir) + # Building without API docs is the default as Nix' C++ interfaces are internal and unstable. AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), internal_api_docs=$enableval, internal_api_docs=no) diff --git a/flake.nix b/flake.nix index 51d818423..05ab7b06d 100644 --- a/flake.nix +++ b/flake.nix @@ -164,6 +164,10 @@ testConfigureFlags = [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ]; internalApiDocsConfigureFlags = [ @@ -404,7 +408,8 @@ src = nixSrc; VERSION_SUFFIX = versionSuffix; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps @@ -710,7 +715,8 @@ stdenv.mkDerivation { name = "nix"; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 331a5ead6..6d2a04aaf 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -6,7 +6,11 @@ libexpr-tests_NAME := libnixexpr-tests libexpr-tests_DIR := $(d) -libexpr-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif libexpr-tests_SOURCES := \ $(wildcard $(d)/*.cc) \ diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index 03becc7d1..e9b8b4f99 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -6,7 +6,11 @@ libstore-tests-exe_NAME = libnixstore-tests libstore-tests-exe_DIR := $(d) -libstore-tests-exe_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests-exe_INSTALL_DIR := $(checkbindir) +else + libstore-tests-exe_INSTALL_DIR := +endif libstore-tests-exe_LIBS = libstore-tests @@ -18,7 +22,11 @@ libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) -libstore-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checklibdir) +else + libstore-tests_INSTALL_DIR := +endif libstore-tests_SOURCES := $(wildcard $(d)/*.cc) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index c8b8557cb..e6fc4e364 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -6,7 +6,11 @@ libutil-tests-exe_NAME = libnixutil-tests libutil-tests-exe_DIR := $(d) -libutil-tests-exe_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests-exe_INSTALL_DIR := $(checkbindir) +else + libutil-tests-exe_INSTALL_DIR := +endif libutil-tests-exe_LIBS = libutil-tests @@ -18,7 +22,11 @@ libutil-tests_NAME = libnixutil-tests libutil-tests_DIR := $(d) -libutil-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checklibdir) +else + libutil-tests_INSTALL_DIR := +endif libutil-tests_SOURCES := $(wildcard $(d)/*.cc) From 31ebc6028b3682969d86a7b39ae87131c41cc604 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Nov 2023 16:45:14 +0100 Subject: [PATCH 350/402] Fix symlink handling This restores the symlink handling behaviour prior to 94812cca98fbb157e5f64a15a85a2b852d289feb. Fixes #9298. --- src/libexpr/eval.hh | 2 +- src/libexpr/parser.y | 18 +++++++++++++----- .../lang/eval-okay-symlink-resolution.exp | 1 + .../lang/eval-okay-symlink-resolution.nix | 1 + .../symlink-resolution/foo/lib/default.nix | 1 + .../lang/symlink-resolution/foo/overlays | 1 + .../symlink-resolution/overlays/overlay.nix | 1 + 7 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 tests/functional/lang/eval-okay-symlink-resolution.exp create mode 100644 tests/functional/lang/eval-okay-symlink-resolution.nix create mode 100644 tests/functional/lang/symlink-resolution/foo/lib/default.nix create mode 120000 tests/functional/lang/symlink-resolution/foo/overlays create mode 100644 tests/functional/lang/symlink-resolution/overlays/overlay.nix diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 048dff42b..9257a0e48 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -827,7 +827,7 @@ std::string showType(const Value & v); /** * If `path` refers to a directory, then append "/default.nix". */ -SourcePath resolveExprPath(const SourcePath & path); +SourcePath resolveExprPath(SourcePath path); struct InvalidPathError : EvalError { diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index b86cef217..f6cf1f689 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -686,17 +686,25 @@ Expr * EvalState::parse( } -SourcePath resolveExprPath(const SourcePath & path) +SourcePath resolveExprPath(SourcePath path) { + unsigned int followCount = 0, maxFollow = 1024; + /* If `path' is a symlink, follow it. This is so that relative path references work. */ - auto path2 = path.resolveSymlinks(); + while (true) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); + if (path.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))}; + } /* If `path' refers to a directory, append `/default.nix'. */ - if (path2.lstat().type == InputAccessor::tDirectory) - return path2 + "default.nix"; + if (path.lstat().type == InputAccessor::tDirectory) + return path + "default.nix"; - return path2; + return path; } diff --git a/tests/functional/lang/eval-okay-symlink-resolution.exp b/tests/functional/lang/eval-okay-symlink-resolution.exp new file mode 100644 index 000000000..8b8441b91 --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.exp @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/eval-okay-symlink-resolution.nix b/tests/functional/lang/eval-okay-symlink-resolution.nix new file mode 100644 index 000000000..ffb1818bd --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.nix @@ -0,0 +1 @@ +import symlink-resolution/foo/overlays/overlay.nix diff --git a/tests/functional/lang/symlink-resolution/foo/lib/default.nix b/tests/functional/lang/symlink-resolution/foo/lib/default.nix new file mode 100644 index 000000000..8b8441b91 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/lib/default.nix @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/symlink-resolution/foo/overlays b/tests/functional/lang/symlink-resolution/foo/overlays new file mode 120000 index 000000000..0d44a21c5 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/overlays @@ -0,0 +1 @@ +../overlays \ No newline at end of file diff --git a/tests/functional/lang/symlink-resolution/overlays/overlay.nix b/tests/functional/lang/symlink-resolution/overlays/overlay.nix new file mode 100644 index 000000000..b0368308e --- /dev/null +++ b/tests/functional/lang/symlink-resolution/overlays/overlay.nix @@ -0,0 +1 @@ +import ../lib From 96d67620d551c7143b6682cfff74a2ee2edbe863 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Nov 2023 17:12:06 +0100 Subject: [PATCH 351/402] Fix a broken generated header file dependency https://hydra.nixos.org/build/240882042 --- src/libexpr/local.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index d243b9cec..ed7bf9490 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -43,7 +43,9 @@ $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh + +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh From c81937576928ca494af0be6e7c61f2070be5d353 Mon Sep 17 00:00:00 2001 From: Dominic Shelton Date: Fri, 17 Nov 2023 17:50:17 +1100 Subject: [PATCH 352/402] doc: Add example of inherit in a let expression --- doc/manual/src/language/constructs.md | 54 ++++++++++++++++++++------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index a3590f55d..bd7bd92a5 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -132,6 +132,32 @@ a = src-set.a; b = src-set.b; c = src-set.c; when used while defining local variables in a let-expression or while defining a set. +in a let expression, inherit can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both resolve to `{ names = [ "a" "b" ]; }`. + ## Functions Functions have the following form: @@ -146,65 +172,65 @@ three kinds of patterns: - If a pattern is a single identifier, then the function matches any argument. Example: - + ```nix let negate = x: !x; concat = x: y: x + y; in if negate true then concat "foo" "bar" else "" ``` - + Note that `concat` is a function that takes one argument and returns a function that takes another argument. This allows partial parameterisation (i.e., only filling some of the arguments of a function); e.g., - + ```nix map (concat "foo") [ "bar" "bla" "abc" ] ``` - + evaluates to `[ "foobar" "foobla" "fooabc" ]`. - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a set containing the listed attributes, and binds the values of those attributes to variables in the function body. For example, the function - + ```nix { x, y, z }: z + y + x ``` - + can only be called with a set containing exactly the attributes `x`, `y` and `z`. No other attributes are allowed. If you want to allow additional arguments, you can use an ellipsis (`...`): - + ```nix { x, y, z, ... }: z + y + x ``` - + This works on any set that contains at least the three named attributes. - + It is possible to provide *default values* for attributes, in which case they are allowed to be missing. A default value is specified by writing `name ? e`, where *e* is an arbitrary expression. For example, - + ```nix { x, y ? "foo", z ? "bar" }: z + y + x ``` - + specifies a function that only requires an attribute named `x`, but optionally accepts `y` and `z`. - An `@`-pattern provides a means of referring to the whole value being matched: - + ```nix args@{ x, y, z, ... }: z + y + x + args.a ``` - + but can also be written as: - + ```nix { x, y, z, ... } @ args: z + y + x + args.a ``` From 2eb59c34b531f03a85f67b9246ccaf0ff5fcad23 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:02:02 +0100 Subject: [PATCH 353/402] Value: extract Value::StringWithContext --- src/libexpr/value.hh | 54 +++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 191cc30ba..0f8cef418 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -158,37 +158,39 @@ public: inline bool isPrimOp() const { return internalType == tPrimOp; }; inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like + * + * "--with-freetype2-library=" + freetype + "/lib" + * + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. + + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. + + * For canonicity, the store paths should be in sorted order. + */ + struct StringWithContext { + const char * c_str; + const char * * context; // must be in sorted order + }; + union { NixInt integer; bool boolean; - /** - * Strings in the evaluator carry a so-called `context` which - * is a list of strings representing store paths. This is to - * allow users to write things like - - * "--with-freetype2-library=" + freetype + "/lib" - - * where `freetype` is a derivation (or a source to be copied - * to the store). If we just concatenated the strings without - * keeping track of the referenced store paths, then if the - * string is used as a derivation attribute, the derivation - * will not have the correct dependencies in its inputDrvs and - * inputSrcs. - - * The semantics of the context is as follows: when a string - * with context C is used as a derivation attribute, then the - * derivations in C will be added to the inputDrvs of the - * derivation, and the other store paths in C will be added to - * the inputSrcs of the derivations. - - * For canonicity, the store paths should be in sorted order. - */ - struct { - const char * c_str; - const char * * context; // must be in sorted order - } string; + StringWithContext string; struct { InputAccessor * accessor; From d8ff5cfe8eba34c8b4b5cc53f3b40cd3dfd84224 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:03:37 +0100 Subject: [PATCH 354/402] Value: extract Value::Path --- src/libexpr/value.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 0f8cef418..34f994997 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -185,6 +185,11 @@ public: const char * * context; // must be in sorted order }; + struct Path { + InputAccessor * accessor; + const char * path; + }; + union { NixInt integer; @@ -192,10 +197,7 @@ public: StringWithContext string; - struct { - InputAccessor * accessor; - const char * path; - } _path; + Path _path; Bindings * attrs; struct { From b55203e874f8e4b2fc5289129efba791937c23d0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:06:04 +0100 Subject: [PATCH 355/402] Value: extract Value::ClosureThunk --- src/libexpr/value.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 34f994997..4c51c52e4 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -190,6 +190,11 @@ public: const char * path; }; + struct ClosureThunk { + Env * env; + Expr * expr; + }; + union { NixInt integer; @@ -205,10 +210,7 @@ public: Value * * elems; } bigList; Value * smallList[2]; - struct { - Env * env; - Expr * expr; - } thunk; + ClosureThunk thunk; struct { Value * left, * right; } app; From 6af1d9f7b94da454252b62f0cfff4ce800c5a46b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:06:55 +0100 Subject: [PATCH 356/402] Value: extract Value::FunctionApplicationThunk --- src/libexpr/value.hh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 4c51c52e4..cfb3f5276 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -195,6 +195,10 @@ public: Expr * expr; }; + struct FunctionApplicationThunk { + Value * left, * right; + }; + union { NixInt integer; @@ -211,17 +215,13 @@ public: } bigList; Value * smallList[2]; ClosureThunk thunk; - struct { - Value * left, * right; - } app; + FunctionApplicationThunk app; struct { Env * env; ExprLambda * fun; } lambda; PrimOp * primOp; - struct { - Value * left, * right; - } primOpApp; + FunctionApplicationThunk primOpApp; ExternalValueBase * external; NixFloat fpoint; }; From 7055c6528532fdd3b0ce9b8f5282b002fc011470 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:07:32 +0100 Subject: [PATCH 357/402] Value: extract Value::Lambda --- src/libexpr/value.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index cfb3f5276..93ccdbc2e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -199,6 +199,11 @@ public: Value * left, * right; }; + struct Lambda { + Env * env; + ExprLambda * fun; + }; + union { NixInt integer; @@ -216,10 +221,7 @@ public: Value * smallList[2]; ClosureThunk thunk; FunctionApplicationThunk app; - struct { - Env * env; - ExprLambda * fun; - } lambda; + Lambda lambda; PrimOp * primOp; FunctionApplicationThunk primOpApp; ExternalValueBase * external; From 260c6147625e95e4772ccdee80d6463d242c7b64 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:31:45 +0100 Subject: [PATCH 358/402] Value: use std::span, change use of const **`Value` and `const`** These two deserve some explanation. We'll get to lists later. Values can normally be thought of as immutable, except they are are also the vehicle for call by need, which must be implemented using mutation. This circumstance makes a `const Value` a rather useless thing: - If it's a thunk, you can't evaluate it, except by copying, but that would not be call by need. - If it's not a thunk, you know the type, so the method that acquired it for you should have returned something more specific, such as a `const Bindings &` (which actually does make sense because that's an immutable span of pointers to mutable `Value`s. - If you don't care about the type yet, you might establish the convention that `const Value` means `deepSeq`-ed data, but this is hardly useful and not actually as safe as you would supposedly want to trust it to be - just convention. **Lists** `std::span` is a tuple of pointer and size - just what we need. We don't return them as `const Value`, because considering the first bullet point we discussed before, we'd have to force all the list values, which isn't what we want. So what we end up with is a nice representation of a list in weak head normal form: the spine is immutable, but the items may need some evaluation later. --- src/libexpr/value.hh | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 93ccdbc2e..bcff8ae55 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,7 @@ #include #include +#include #include "symbol-table.hh" #include "value/context.hh" @@ -395,7 +396,13 @@ public: return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } - const Value * const * listElems() const + std::span listItems() const + { + assert(isList()); + return std::span(listElems(), listSize()); + } + + Value * const * listElems() const { return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } @@ -414,34 +421,6 @@ public: */ bool isTrivial() const; - auto listItems() - { - struct ListIterable - { - typedef Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ListIterable { begin, begin + listSize() }; - } - - auto listItems() const - { - struct ConstListIterable - { - typedef const Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ConstListIterable { begin, begin + listSize() }; - } - SourcePath path() const { assert(internalType == tPath); From 121665f3773bc46ca6df0dda6f66b1a86e7d9e72 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 17:51:09 +0100 Subject: [PATCH 359/402] nix-env: Use state.mkList, required for correct stats --- src/nix-env/nix-env.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 213a20d93..ab1d8f713 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -172,7 +172,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v directory). */ else if (st.type == InputAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); - attrs.alloc("_combineChannels").mkList(0); + state.mkList(attrs.alloc("_combineChannels"), 0); StringSet seen; getAllExprs(state, path, seen, attrs); v.mkAttrs(attrs); From 7b0e8c5c2c09146722349d3fd2dd69211d8b8945 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 17 Nov 2023 10:56:23 +0100 Subject: [PATCH 360/402] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/language/constructs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index bd7bd92a5..a82ec5960 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -132,7 +132,7 @@ a = src-set.a; b = src-set.b; c = src-set.c; when used while defining local variables in a let-expression or while defining a set. -in a let expression, inherit can be used to selectively bring specific attributes of a set into scope. For example +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example ```nix @@ -156,7 +156,7 @@ in } ``` -both resolve to `{ names = [ "a" "b" ]; }`. +both evaluate to `{ names = [ "a" "b" ]; }`. ## Functions From f7d59d0dda5e4a793e701bc8fb9136b3ef22948c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Nov 2023 14:21:17 +0100 Subject: [PATCH 361/402] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.19.md | 77 +++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 75 ------------------------ 3 files changed, 78 insertions(+), 75 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.19.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 794f78a07..8dc464abd 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -115,6 +115,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md new file mode 100644 index 000000000..4eecaf929 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -0,0 +1,77 @@ +# Release 2.19 (2023-11-17) + +- The experimental `nix` command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) + by appending the contents of any `#! nix` lines and the script's location into a single call. + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. + +- There is a new flake installable syntax `flakeref#.attrPath` where the "." prefix specifies that `attrPath` is interpreted from the root of the flake outputs, with no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + +- Add a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + +- `nix-shell` shebang lines now support single-quoted arguments. + +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes. + +- The interface for creating and updating lock files has been overhauled: + + - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. + It will *never* update existing inputs. + + - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. + - Passing no arguments will update all inputs of the current flake, just like it already did. + - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` + - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. + + - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. + They are superceded by `nix flake update`. + +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + +- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) + (experimental) now returns a JSON map rather than JSON list. + The `path` field of each object has instead become the key in the outer map, since it is unique. + The `valid` field also goes away because we just use `null` instead. + + - Old way: + + ```json5 + [ + { + "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", + "valid": true, + // ... + }, + { + "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", + "valid": false + } + ] + ``` + + - New way + + ```json5 + { + "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { + // ... + }, + "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, + } + ``` + + This makes it match `nix derivation show`, which also maps store paths to information. + +- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) + [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. + This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) + (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + +- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 422f1fce8..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,77 +1,2 @@ # Release X.Y (202?-??-??) -- The experimental nix command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) - by appending the contents of any `#! nix` lines and the script's location to a single call. - -- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - -- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). - -- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. - -- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. - -- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. - -- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). - -- `nix-shell` shebang lines now support single-quoted arguments. - -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes. - -- The interface for creating and updating lock files has been overhauled: - - - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. - It will *never* update existing inputs. - - - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. - - Passing no arguments will update all inputs of the current flake, just like it already did. - - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` - - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. - - - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. - They are superceded by `nix flake update`. - -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). - -- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) - (experimental) now returns a JSON map rather than JSON list. - The `path` field of each object has instead become the key in th outer map, since it is unique. - The `valid` field also goes away because we just use null instead. - - - Old way: - - ```json5 - [ - { - "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", - "valid": true, - // ... - }, - { - "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", - "valid": false - } - ] - ``` - - - New way - - ```json5 - { - "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { - // ... - }, - "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, - } - ``` - - This makes it match `nix derivation show`, which also maps store paths to information. - -- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) - [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. - This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) - (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). - -- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. From 293ae592576bb9c48975466613fcba6a30d06f5e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Nov 2023 11:26:45 -0500 Subject: [PATCH 362/402] Fix `make check` After 9c7749e13508996eb9df83b1692664cc8cdbf952, `libutil-tests_RUN` doesn't exist. It needs to become `libutil-tests-exe_RUN`. --- src/libutil/tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index e6fc4e364..66886c45f 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -1,4 +1,4 @@ -check: libutil-tests_RUN +check: libutil-tests-exe_RUN programs += libutil-tests-exe From 4a539ac3eac90b2c2f839cae885df89a03240348 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 17 Nov 2023 17:38:08 +0100 Subject: [PATCH 363/402] Fix buildNoGc Fixes https://hydra.nixos.org/build/241067941/nixlog/1 src/libexpr/eval.cc:1776:54: error: variable 'boost::container::small_vector vArgs' has initializer but incomplete type --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8b0ada517..e9b8cacfd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -31,6 +31,7 @@ #include #include +#include #if HAVE_BOEHMGC @@ -42,7 +43,6 @@ #include #include #include -#include #endif From 251fb23aeab8f85afb4f8376c2e6fc3d8b229d23 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 01:46:48 +0100 Subject: [PATCH 364/402] Shebang parser: add virtual destructor Fixes: warning: destructor called on non-final 'nix::ParseUnquoted' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor] --- src/libutil/args.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 4359c5e8e..4480a03f5 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -97,6 +97,8 @@ struct Parser { virtual void operator()(std::shared_ptr & state, Strings & r) = 0; Parser(std::string_view s) : remaining(s) {}; + + virtual ~Parser() { }; }; struct ParseQuoted : public Parser { From 70ddf298e0882075dcb1cf69562c629a195718f7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sun, 19 Nov 2023 04:09:14 +0100 Subject: [PATCH 365/402] doc: Add link to filterSource from path --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e274c3c0c..dda409955 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2374,7 +2374,7 @@ static RegisterPrimOp primop_path({ like `@`. - filter\ - A function of the type expected by `builtins.filterSource`, + A function of the type expected by [`builtins.filterSource`](#builtins-filterSource), with the same semantics. - recursive\ From d5928085d5a542b19cc21b1f299392ee7a0c960b Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 19 Nov 2023 19:57:07 +0100 Subject: [PATCH 366/402] builtins.concatMap: Fix typo in error message --- src/libexpr/primops.cc | 2 +- src/libexpr/tests/error_traces.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dda409955..27f502830 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3459,7 +3459,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap"); + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap"); len += lists[n].listSize(); } diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 139366bcd..81498f65a 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -906,12 +906,12 @@ namespace nix { ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); } From 1d6abec993a371091459d5e23f985c6d69621ce7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Nov 2023 12:35:35 +0100 Subject: [PATCH 367/402] Revert use of boost::container::small_vector in the evaluator It caused random crashes (https://hydra.nixos.org/build/241514506, https://hydra.nixos.org/build/241443330) because the heap allocation done by small_vector in the not-small case is not scanned for GC roots. --- src/libexpr/eval.cc | 11 +++++------ src/libexpr/primops.cc | 7 ++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e9b8cacfd..46a49c891 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -31,7 +31,6 @@ #include #include -#include #if HAVE_BOEHMGC @@ -1710,7 +1709,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[maxPrimOpArity]; + Value * vArgs[arity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1773,11 +1772,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) // 4: about 60 // 5: under 10 // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. - boost::container::small_vector vArgs(args.size()); + Value * vArgs[args.size()]; for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs.data(), v, pos); + state.callFunction(vFun, args.size(), vArgs, v, pos); } @@ -2016,8 +2015,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - boost::container::small_vector values(es->size()); - Value * vTmpP = values.data(); + Value values[es->size()]; + Value * vTmpP = values; for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 27f502830..a8d44d8b7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2729,7 +2729,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - boost::container::small_vector res(args[1]->listSize()); + Value * res[args[1]->listSize()]; size_t found = 0; for (auto v2 : args[1]->listItems()) { @@ -3064,7 +3064,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - boost::container::small_vector vs(args[1]->listSize()); + // FIXME: putting this on the stack is risky. + Value * vs[args[1]->listSize()]; size_t k = 0; bool same = true; @@ -3453,7 +3454,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - boost::container::small_vector lists(nrLists); + Value lists[nrLists]; size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From 5b99c823ef95ba5c642ae105815d5acd4f093aa3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Jan 2022 00:13:54 +0100 Subject: [PATCH 368/402] Mark official release --- doc/manual/src/SUMMARY.md.in | 1 - flake.nix | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8dc464abd..f4d4c1f41 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -114,7 +114,6 @@ - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - - [Release X.Y (202?-??-??)](release-notes/rl-next.md) - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) diff --git a/flake.nix b/flake.nix index 05ab7b06d..fd2dbdc48 100644 --- a/flake.nix +++ b/flake.nix @@ -13,7 +13,7 @@ let inherit (nixpkgs) lib; - officialRelease = false; + officialRelease = true; version = lib.fileContents ./.version + versionSuffix; versionSuffix = From 4cc65f3dd5fc64d941360e59be75b2a48b46eee1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Nov 2023 15:06:04 +0100 Subject: [PATCH 369/402] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ef0f38abe..b8e248f40 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.19.0 +2.19.1 From 2778b218c3e3de1cb9f31d563a8c53a59b03f985 Mon Sep 17 00:00:00 2001 From: DavHau Date: Sun, 19 Nov 2023 20:32:23 +0700 Subject: [PATCH 370/402] fetchTree: clarify docs for shallow flag (cherry picked from commit 796a7eb92d2b0caf75685126adc7460a4c39cfec) --- src/libexpr/primops/fetchTree.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 8031bf809..383ec7c58 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -425,7 +425,8 @@ static RegisterPrimOp primop_fetchGit({ - `shallow` (default: `false`) - A Boolean parameter that specifies whether fetching a shallow clone is allowed. + A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed. + This still performs a full clone of what is available on the remote. - `allRefs` From a5c6ba3edc9d6c04ac10cde3fdb6896db9bffd23 Mon Sep 17 00:00:00 2001 From: roblabla Date: Mon, 20 Nov 2023 15:41:38 +0100 Subject: [PATCH 371/402] Fix bad_format_string error when builder stdout contains % (cherry picked from commit e2b6821ca0147f36bcb9404aab080f80746984c8) --- src/libutil/serialise.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index d7950b11b..f465bd0de 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -448,7 +448,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintformat(fmt("%s", msg)), + .msg = hintfmt(msg), }; auto havePos = readNum(source); assert(havePos == 0); @@ -457,7 +457,7 @@ Error readError(Source & source) havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintformat(fmt("%s", readString(source))) + .hint = hintfmt(readString(source)) }); } return Error(std::move(info)); From e011d94813c1b1c2309997f26978ec9c704cb748 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 21 Nov 2023 14:49:48 +0100 Subject: [PATCH 372/402] Fix "unbound variable" errors in bash Fixes #9414 (cherry picked from commit 64827360be35a3d16e818aa9d8426ca40b2c4dc2) --- scripts/nix-profile-daemon.sh.in | 2 +- scripts/nix-profile.sh.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index c63db4648..d256b24ed 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -31,7 +31,7 @@ fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc -if [ -z "$XDG_DATA_DIRS" ]; then +if [ -z "${XDG_DATA_DIRS-}" ]; then # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 56e070ae1..44bc96e89 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -33,7 +33,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc - if [ -z "$XDG_DATA_DIRS" ]; then + if [ -z "${XDG_DATA_DIRS-}" ]; then # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else From c27f9777f888d670b6bfd05d5a6417d712639891 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Nov 2023 17:30:19 +0100 Subject: [PATCH 373/402] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index b8e248f40..17bdb70fa 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.19.1 +2.19.2 From 914309c35ddb341082756d090507fb7d06f43290 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Nov 2023 23:19:25 -0500 Subject: [PATCH 374/402] Add missing `-lrapidcheck` fixing build with shared lib https://github.com/NixOS/nixpkgs/pull/269064 makes rapidcheck be build as a shared lib, but that broke Nix because the `-lrapidcheck` was missing. This fixes that (and doesn't break Nix what the library is a static archive as today). (cherry picked from commit 46131567da96ffac298b9ec54016b37114b0dfd5) --- src/libexpr/tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 6d2a04aaf..7689a03e0 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -20,4 +20,4 @@ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/l libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers -libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock From 819eda461556c0a886a945c8fdf517cb42f12213 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Sat, 25 Nov 2023 11:26:57 +0800 Subject: [PATCH 375/402] `nix flake update` add deprecation warnings. This builds on #8817, to add additional UX help for people with existing muscle memory (or shell history) with --update-input and tries to gently guide them towards the newly evolved CLI UI. Co-authored-by: Cole Helbling (cherry picked from commit af002985875a4a25a2a59a601a2b7eecdd84ec5c) --- src/libcmd/installables.cc | 24 ++++++++++++++++++++++++ src/nix/flake.cc | 8 +++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1c6103020..68287b445 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -47,6 +47,16 @@ MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; + addFlag({ + .longName = "recreate-lock-file", + .description = "Recreate the flake's lock file from scratch.", + .category = category, + .handler = {[&]() { + lockFlags.recreateLockFile = true; + warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); + }} + }); + addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", @@ -79,6 +89,20 @@ MixFlakeOptions::MixFlakeOptions() .handler = {&lockFlags.commitLockFile, true} }); + addFlag({ + .longName = "update-input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .category = category, + .labels = {"input-path"}, + .handler = {[&](std::string s) { + warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); + lockFlags.inputUpdates.insert(flake::parseInputPath(s)); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + addFlag({ .longName = "override-input", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 38938f09e..e0c67fdfa 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -89,7 +89,13 @@ public: .label="inputs", .optional=true, .handler={[&](std::string inputToUpdate){ - auto inputPath = flake::parseInputPath(inputToUpdate); + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } if (lockFlags.inputUpdates.contains(inputPath)) warn("Input '%s' was specified multiple times. You may have done this by accident."); lockFlags.inputUpdates.insert(inputPath); From 92f3598a16bae38c5af32b12e5c05ec5767f2c14 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 27 Nov 2023 09:18:56 +0100 Subject: [PATCH 376/402] add deprecation warnings in documentation this is hacky, but can serve as a stopgap until we can do it programmatically. (cherry picked from commit 7e08bdefccf8993704e7430cc3e10a042fe650ac) --- src/libcmd/installables.cc | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 68287b445..6e670efea 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -49,7 +49,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", + .description = R"( + Recreate the flake's lock file from scratch. + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .handler = {[&]() { lockFlags.recreateLockFile = true; @@ -73,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = - "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", + .description = R"( + Don't allow lookups in the flake registries. + + > **DEPRECATED** + > + > Use [`--no-use-registries`](#opt-no-use-registries) instead. + )", .category = category, .handler = {[&]() { lockFlags.useRegistries = false; @@ -91,7 +102,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .description = R"( + Update a specific flake input (ignoring its previous entry in the lock file). + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { From 6dfb06d4a35017d93bd19823b8c18785d1d70b99 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 27 Nov 2023 08:33:03 +0100 Subject: [PATCH 377/402] add path based redirects up to now, those were managed outside of this repo, which as unsurprisingly a real hassle to deal with if one wanted to prevent URLs from breaking when moving pages around. this change removes a large part of the friction involved in moving content in the Nix manual. possible next steps for further automation: - check for content that moved and warn if it's not reachable from links that were valid prior to a change - create redirect rules automatically based on this information (cherry picked from commit 2b7016cc56d12e67de9f1f25b18311866a26a5fe) --- doc/manual/_redirects | 30 ++++++++++++++++++++++++++++++ doc/manual/redirects.js | 8 +++++--- 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 doc/manual/_redirects diff --git a/doc/manual/_redirects b/doc/manual/_redirects new file mode 100644 index 000000000..4ea289d86 --- /dev/null +++ b/doc/manual/_redirects @@ -0,0 +1,30 @@ +# redirect rules for paths (server-side) to prevent link rot. +# see ./redirects.js for redirects based on URL fragments (client-side) +# +# concrete user story this supports: +# - user finds URL to the manual for Nix x.y +# - Nix x.z (z > y) is the most recent release +# - updating the version in the URL will show the right thing +# +# format documentation: +# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file +# - https://docs.netlify.com/routing/redirects/redirect-options/ +# +# conventions: +# - always force (!) since this allows re-using file names +# - group related paths to ease readability +# - always append new redirects to the end of the file +# - redirects that should have been there but are missing can be inserted where they belong + +/expressions/expression-language /language/ 301! +/expressions/language-values /language/values 301! +/expressions/language-constructs /language/constructs 301! +/expressions/language-operators /language/operators 301! +/expressions/* /language/:splat 301! + +/package-management/basic-package-mgmt /command-ref/nix-env 301! + +/package-management/channels* /command-ref/nix-channel 301! + +/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! + diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index d1b10109d..3b507adf3 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,9 @@ -// redirect rules for anchors ensure backwards compatibility of URLs. -// this must be done on the client side, as web servers do not see the anchor part of the URL. +// redirect rules for URL fragments (client-side) to prevent link rot. +// this must be done on the client side, as web servers do not see the fragment part of the URL. +// it will only work with JavaScript enabled in the browser, but this is the best we can do here. +// see ./_redirects for path redirects (client-side) -// redirections are declared as follows: +// redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // // IMPORTANT: it must specify the full path with file name and suffix From 94b240113889149957dd22b2a703f6fd4659767d Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 27 Nov 2023 08:56:24 +0000 Subject: [PATCH 378/402] libexpr: add missing dependency on 'flake/call-flake.nix.gen.hh' Without the change build for `eval.o` fails occasionally as: $ make src/libexpr/eval.o GEN Makefile.config GEN src/libexpr/primops/derivation.nix.gen.hh GEN src/libexpr/fetchurl.nix.gen.hh GEN src/libexpr/parser-tab.cc GEN src/libexpr/lexer-tab.cc src/libexpr/lexer.l:314: warning, -s option given but default rule can be matched CXX src/libexpr/eval.o src/libexpr/eval.cc:519:18: fatal error: flake/call-flake.nix.gen.hh: No such file or directory 519 | #include "flake/call-flake.nix.gen.hh" | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ compilation terminated. make: *** [mk/patterns.mk:3: src/libexpr/eval.o] Error 1 Noticed in https://github.com/NixOS/nixpkgs/pull/269439 (cherry picked from commit 75134b7513eb781074969fc8d6d865cc95063444) --- src/libexpr/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed7bf9490..637f998b6 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -47,6 +47,6 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh -$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = From 28f0322307cda8b830dd3951cdb5797e28122a73 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 27 Nov 2023 15:50:45 +0100 Subject: [PATCH 379/402] libexpr/local.mk: Make eval compile deps regular Dependency is now entirely through the eval.cc rule. All gen.hh deps are now there. (cherry picked from commit 68c48756fece5aee77f9b44607afa9248d75e67c) --- src/libexpr/local.mk | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 637f998b6..b37fe6f1d 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -45,8 +45,6 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh - -$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = From 94a7f91236600ffbf2c62357381d1178aac36e42 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Nov 2023 17:18:00 +0100 Subject: [PATCH 380/402] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 17bdb70fa..6ef6ef5ac 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.19.2 +2.19.3 From f01baf5f0657ce58c9c14d7ab5de910f58e423cb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 11:38:47 +0100 Subject: [PATCH 381/402] flake.nix: Update nixpkgs: release-23.05 -> nixos-23.05-small MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9eb24edd6a0027fed010ccfe300a9734d029983c' (2023-11-01) → 'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18) (cherry picked from commit fe4f573d49a5c47cf9ffd0bd3fe8868104550818) --- flake.lock | 8 ++++---- flake.nix | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 991cef1ee..825166717 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1698876495, - "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=", + "lastModified": 1700342017, + "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c", + "rev": "decdf666c833a325cb4417041a90681499e06a41", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.05", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index fd2dbdc48..b76beefc6 100644 --- a/flake.nix +++ b/flake.nix @@ -1,9 +1,7 @@ { description = "The purely functional package manager"; - # FIXME go back to nixos-23.05-small once - # https://github.com/NixOS/nixpkgs/pull/264875 is included. - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 5656f8c8c7c26eebc3ae9ac5ae512d10f814b069 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 16:18:27 +0100 Subject: [PATCH 382/402] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18) → 'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-23) (cherry picked from commit c5d49ec7ab7b9fb33f0336a909ac837e208be807) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 825166717..d11b468c5 100644 --- a/flake.lock +++ b/flake.lock @@ -34,11 +34,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1700342017, - "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=", + "lastModified": 1700748986, + "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "decdf666c833a325cb4417041a90681499e06a41", + "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0", "type": "github" }, "original": { From a61e42adb528b3d40ce43e07c79368d779a8b624 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Aug 2023 10:20:28 -0400 Subject: [PATCH 383/402] Move tests to separate directories, and document Today, with the tests inside a `tests` intermingled with the corresponding library's source code, we have a few problems: - We have to be careful that wildcards don't end up with tests being built as part of Nix proper, or test headers being installed as part of Nix proper. - Tests in libraries but not executables is not right: - It means each executable runs the previous unit tests again, because it needs the libraries. - It doesn't work right on Windows, which doesn't want you to load a DLL just for the side global variable . It could be made to work with the dlopen equivalent, but that's gross! This reorg solves these problems. There is a remaining problem which is that sibbling headers (like `hash.hh` the test header vs `hash.hh` the main `libnixutil` header) end up shadowing each other. This PR doesn't solve that. That is left as future work for a future PR. Co-authored-by: Valentin Gagarin (cherry picked from commit 91b6833686a6a6d9eac7f3f66393ec89ef1d3b57) --- .gitignore | 6 +- Makefile | 10 ++- doc/internal-api/doxygen.cfg.in | 12 ++- doc/manual/src/contributing/testing.md | 67 ++++++++++---- flake.nix | 2 +- mk/common-test.sh | 2 +- mk/programs.mk | 2 +- src/libexpr/tests/local.mk | 23 ----- src/libstore/tests/local.mk | 37 -------- src/libutil/tests/local.mk | 41 --------- tests/unit/libexpr-support/local.mk | 23 +++++ .../unit/libexpr-support}/tests/libexpr.hh | 0 .../libexpr-support/tests/value/context.cc | 30 +++++++ .../libexpr-support}/tests/value/context.hh | 2 +- .../unit/libexpr}/derived-path.cc | 0 .../unit/libexpr}/error_traces.cc | 0 .../tests => tests/unit/libexpr}/flakeref.cc | 0 .../tests => tests/unit/libexpr}/json.cc | 0 tests/unit/libexpr/local.mk | 36 ++++++++ .../tests => tests/unit/libexpr}/primops.cc | 0 .../unit/libexpr}/search-path.cc | 0 .../tests => tests/unit/libexpr}/trivial.cc | 0 .../unit/libexpr}/value/context.cc | 30 ------- .../unit/libexpr}/value/print.cc | 0 tests/unit/libstore-support/local.mk | 21 +++++ .../libstore-support/tests/derived-path.cc | 57 ++++++++++++ .../libstore-support}/tests/derived-path.hh | 0 .../unit/libstore-support}/tests/libstore.hh | 0 .../libstore-support/tests/outputs-spec.cc | 24 +++++ .../libstore-support}/tests/outputs-spec.hh | 2 +- tests/unit/libstore-support/tests/path.cc | 82 ++++++++++++++++++ .../unit/libstore-support}/tests/path.hh | 3 + .../unit/libstore-support}/tests/protocol.hh | 2 +- .../unit/libstore}/common-protocol.cc | 0 .../data}/common-protocol/content-address.bin | Bin .../data}/common-protocol/drv-output.bin | Bin .../optional-content-address.bin | Bin .../common-protocol/optional-store-path.bin | Bin .../data}/common-protocol/realisation.bin | Bin .../libstore/data}/common-protocol/set.bin | Bin .../data}/common-protocol/store-path.bin | Bin .../libstore/data}/common-protocol/string.bin | Bin .../libstore/data}/common-protocol/vector.bin | Bin .../derivation/bad-old-version-dyn-deps.drv | 0 .../libstore/data}/derivation/bad-version.drv | 0 .../data}/derivation/dynDerivationDeps.drv | 0 .../data}/derivation/dynDerivationDeps.json | 0 .../data}/derivation/output-caFixedFlat.json | 0 .../data}/derivation/output-caFixedNAR.json | 0 .../data}/derivation/output-caFixedText.json | 0 .../data}/derivation/output-caFloating.json | 0 .../data}/derivation/output-deferred.json | 0 .../data}/derivation/output-impure.json | 0 .../derivation/output-inputAddressed.json | 0 .../unit/libstore/data}/derivation/simple.drv | 0 .../libstore/data}/derivation/simple.json | 0 .../unit/libstore/data}/nar-info/impure.json | 0 .../unit/libstore/data}/nar-info/pure.json | 0 .../unit/libstore/data}/path-info/impure.json | 0 .../unit/libstore/data}/path-info/pure.json | 0 .../data}/serve-protocol/build-result-2.2.bin | Bin .../data}/serve-protocol/build-result-2.3.bin | Bin .../data}/serve-protocol/build-result-2.6.bin | Bin .../data}/serve-protocol/content-address.bin | Bin .../data}/serve-protocol/drv-output.bin | Bin .../optional-content-address.bin | Bin .../serve-protocol/optional-store-path.bin | Bin .../data}/serve-protocol/realisation.bin | Bin .../libstore/data}/serve-protocol/set.bin | Bin .../data}/serve-protocol/store-path.bin | Bin .../libstore/data}/serve-protocol/string.bin | Bin .../libstore/data}/serve-protocol/vector.bin | Bin .../worker-protocol/build-result-1.27.bin | Bin .../worker-protocol/build-result-1.28.bin | Bin .../worker-protocol/build-result-1.29.bin | Bin .../data}/worker-protocol/content-address.bin | Bin .../worker-protocol/derived-path-1.29.bin | Bin .../worker-protocol/derived-path-1.30.bin | Bin .../data}/worker-protocol/drv-output.bin | Bin .../keyed-build-result-1.29.bin | Bin .../optional-content-address.bin | Bin .../worker-protocol/optional-store-path.bin | Bin .../worker-protocol/optional-trusted-flag.bin | Bin .../data}/worker-protocol/realisation.bin | Bin .../libstore/data}/worker-protocol/set.bin | Bin .../data}/worker-protocol/store-path.bin | Bin .../libstore/data}/worker-protocol/string.bin | Bin .../unkeyed-valid-path-info-1.15.bin | Bin .../worker-protocol/valid-path-info-1.15.bin | Bin .../worker-protocol/valid-path-info-1.16.bin | Bin .../libstore/data}/worker-protocol/vector.bin | Bin .../unit/libstore}/derivation.cc | 2 +- .../unit/libstore}/derived-path.cc | 53 ----------- .../unit/libstore}/downstream-placeholder.cc | 0 tests/unit/libstore/local.mk | 31 +++++++ .../tests => tests/unit/libstore}/machines.cc | 4 +- .../unit/libstore}/nar-info-disk-cache.cc | 0 .../tests => tests/unit/libstore}/nar-info.cc | 2 +- .../unit/libstore}/outputs-spec.cc | 27 +----- .../unit/libstore}/path-info.cc | 2 +- .../tests => tests/unit/libstore}/path.cc | 73 ---------------- .../unit/libstore}/references.cc | 0 .../unit/libstore}/serve-protocol.cc | 0 .../libstore}/test-data/machines.bad_format | 0 .../unit/libstore}/test-data/machines.valid | 0 .../unit/libstore}/worker-protocol.cc | 0 tests/unit/libutil-support/local.mk | 19 ++++ .../tests/characterization.hh | 4 +- tests/unit/libutil-support/tests/hash.cc | 20 +++++ .../unit/libutil-support}/tests/hash.hh | 0 .../tests => tests/unit/libutil}/args.cc | 6 +- .../unit/libutil}/canon-path.cc | 0 .../unit/libutil}/chunked-vector.cc | 0 .../tests => tests/unit/libutil}/closure.cc | 0 .../unit/libutil}/compression.cc | 0 .../tests => tests/unit/libutil}/config.cc | 0 .../unit/libutil/data}/git/check-data.sh | 2 +- .../libutil/data}/git/hello-world-blob.bin | Bin .../unit/libutil/data}/git/hello-world.bin | Bin .../unit/libutil/data}/git/tree.bin | Bin .../unit/libutil/data}/git/tree.txt | 0 .../tests => tests/unit/libutil}/git.cc | 6 +- .../tests => tests/unit/libutil}/hash.cc | 20 +---- .../tests => tests/unit/libutil}/hilite.cc | 0 tests/unit/libutil/local.mk | 31 +++++++ .../tests => tests/unit/libutil}/logging.cc | 0 .../tests => tests/unit/libutil}/lru-cache.cc | 0 .../tests => tests/unit/libutil}/pool.cc | 0 .../unit/libutil}/references.cc | 0 .../unit/libutil}/suggestions.cc | 0 .../tests => tests/unit/libutil}/tests.cc | 0 .../tests => tests/unit/libutil}/url.cc | 0 .../unit/libutil}/xml-writer.cc | 0 133 files changed, 464 insertions(+), 352 deletions(-) delete mode 100644 src/libexpr/tests/local.mk delete mode 100644 src/libstore/tests/local.mk delete mode 100644 src/libutil/tests/local.mk create mode 100644 tests/unit/libexpr-support/local.mk rename {src/libexpr => tests/unit/libexpr-support}/tests/libexpr.hh (100%) create mode 100644 tests/unit/libexpr-support/tests/value/context.cc rename {src/libexpr => tests/unit/libexpr-support}/tests/value/context.hh (95%) rename {src/libexpr/tests => tests/unit/libexpr}/derived-path.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/error_traces.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/flakeref.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/json.cc (100%) create mode 100644 tests/unit/libexpr/local.mk rename {src/libexpr/tests => tests/unit/libexpr}/primops.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/search-path.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/trivial.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/value/context.cc (83%) rename {src/libexpr/tests => tests/unit/libexpr}/value/print.cc (100%) create mode 100644 tests/unit/libstore-support/local.mk create mode 100644 tests/unit/libstore-support/tests/derived-path.cc rename {src/libstore => tests/unit/libstore-support}/tests/derived-path.hh (100%) rename {src/libstore => tests/unit/libstore-support}/tests/libstore.hh (100%) create mode 100644 tests/unit/libstore-support/tests/outputs-spec.cc rename {src/libstore => tests/unit/libstore-support}/tests/outputs-spec.hh (89%) create mode 100644 tests/unit/libstore-support/tests/path.cc rename {src/libstore => tests/unit/libstore-support}/tests/path.hh (82%) rename {src/libstore => tests/unit/libstore-support}/tests/protocol.hh (96%) rename {src/libstore/tests => tests/unit/libstore}/common-protocol.cc (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/vector.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-old-version-dyn-deps.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-version.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedFlat.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedNAR.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedText.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFloating.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-deferred.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-inputAddressed.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/pure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/pure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.2.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.3.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.6.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/vector.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.27.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.28.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.30.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/keyed-build-result-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-trusted-flag.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/unkeyed-valid-path-info-1.15.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.15.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.16.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/vector.bin (100%) rename {src/libstore/tests => tests/unit/libstore}/derivation.cc (99%) rename {src/libstore/tests => tests/unit/libstore}/derived-path.cc (66%) rename {src/libstore/tests => tests/unit/libstore}/downstream-placeholder.cc (100%) create mode 100644 tests/unit/libstore/local.mk rename {src/libstore/tests => tests/unit/libstore}/machines.cc (97%) rename {src/libstore/tests => tests/unit/libstore}/nar-info-disk-cache.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/nar-info.cc (98%) rename {src/libstore/tests => tests/unit/libstore}/outputs-spec.cc (92%) rename {src/libstore/tests => tests/unit/libstore}/path-info.cc (97%) rename {src/libstore/tests => tests/unit/libstore}/path.cc (59%) rename {src/libstore/tests => tests/unit/libstore}/references.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/serve-protocol.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.bad_format (100%) rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.valid (100%) rename {src/libstore/tests => tests/unit/libstore}/worker-protocol.cc (100%) create mode 100644 tests/unit/libutil-support/local.mk rename {src/libutil => tests/unit/libutil-support}/tests/characterization.hh (95%) create mode 100644 tests/unit/libutil-support/tests/hash.cc rename {src/libutil => tests/unit/libutil-support}/tests/hash.hh (100%) rename {src/libutil/tests => tests/unit/libutil}/args.cc (98%) rename {src/libutil/tests => tests/unit/libutil}/canon-path.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/chunked-vector.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/closure.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/compression.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/config.cc (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/check-data.sh (98%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world-blob.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.txt (100%) rename {src/libutil/tests => tests/unit/libutil}/git.cc (97%) rename {src/libutil/tests => tests/unit/libutil}/hash.cc (92%) rename {src/libutil/tests => tests/unit/libutil}/hilite.cc (100%) create mode 100644 tests/unit/libutil/local.mk rename {src/libutil/tests => tests/unit/libutil}/logging.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/lru-cache.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/pool.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/references.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/suggestions.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/tests.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/url.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/xml-writer.cc (100%) diff --git a/.gitignore b/.gitignore index 04d96ca2c..23ba1077f 100644 --- a/.gitignore +++ b/.gitignore @@ -41,14 +41,14 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libnixexpr-tests +/tests/unit/libexpr/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libnixstore-tests +/tests/unit/libstore/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libnixutil-tests +/tests/unit/libutil/libnixutil-tests /src/nix/nix diff --git a/Makefile b/Makefile index 4f4ac0c6e..8e0719b86 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,13 @@ makefiles = \ endif ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) -UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ - src/libutil/tests/local.mk \ - src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk + tests/unit/libutil/local.mk \ + tests/unit/libutil-support/local.mk \ + tests/unit/libstore/local.mk \ + tests/unit/libstore-support/local.mk \ + tests/unit/libexpr/local.mk \ + tests/unit/libexpr-support/local.mk endif ifeq ($(ENABLE_TESTS), yes) diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 599be2470..ad5af97e6 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -39,17 +39,21 @@ INPUT = \ src/libcmd \ src/libexpr \ src/libexpr/flake \ - src/libexpr/tests \ - src/libexpr/tests/value \ + tests/unit/libexpr \ + tests/unit/libexpr/value \ + tests/unit/libexpr/test \ + tests/unit/libexpr/test/value \ src/libexpr/value \ src/libfetchers \ src/libmain \ src/libstore \ src/libstore/build \ src/libstore/builtins \ - src/libstore/tests \ + tests/unit/libstore \ + tests/unit/libstore/test \ src/libutil \ - src/libutil/tests \ + tests/unit/libutil \ + tests/unit/libutil/test \ src/nix \ src/nix-env \ src/nix-store diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 0b45b88a3..d8d162379 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -20,6 +20,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. [googletest]: https://google.github.io/googletest/ [rapidcheck]: https://github.com/emil-e/rapidcheck +[property testing]: https://en.wikipedia.org/wiki/Property_testing ### Source and header layout @@ -28,34 +29,50 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > ``` > src > ├── libexpr +> │ ├── local.mk > │ ├── value/context.hh > │ ├── value/context.cc +> │ … +> │ +> ├── tests > │ │ > │ … -> └── tests -> │ ├── value/context.hh -> │ ├── value/context.cc +> │ └── unit +> │ ├── libutil +> │ │ ├── local.mk +> │ │ … +> │ │ └── data +> │ │ ├── git/tree.txt +> │ │ … > │ │ -> │ … -> │ -> ├── unit-test-data -> │ ├── libstore -> │ │ ├── worker-protocol/content-address.bin -> │ │ … -> │ … +> │ ├── libexpr-support +> │ │ ├── local.mk +> │ │ └── tests +> │ │ ├── value/context.hh +> │ │ ├── value/context.cc +> │ │ … +> │ │ +> │ ├── libexpr +> │ … ├── local.mk +> │ ├── value/context.cc +> │ … > … > ``` -The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. +Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. -The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. -The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. +The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Note that each executable only gets the data for its tests. -> **Note** -> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. -> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. -> That is why we have the test data nested within a single `unit-test-data` directory. +The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. + +The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. +But organizing the tests this way has one big benefit: +there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library. ### Running tests @@ -69,7 +86,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -80,6 +97,18 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. +### Unit test support libraries + +There are headers and code which are not just used to test the library in question, but also downstream libraries. +For example, we do [property testing] with the [rapidcheck] library. +This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests. +Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it. +Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them. + +It is important that these testing libraries don't contain any actual tests themselves. +On some platforms they would be run as part of every test executable that uses them, which is redundant. +On other platforms they wouldn't be run at all. + ## Functional tests The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. diff --git a/flake.nix b/flake.nix index b76beefc6..5cf2a2419 100644 --- a/flake.nix +++ b/flake.nix @@ -89,7 +89,7 @@ ./misc ./precompiled-headers.h ./src - ./unit-test-data + ./tests/unit ./COPYING ./scripts/local.mk functionalTestFiles diff --git a/mk/common-test.sh b/mk/common-test.sh index 00ccd1584..2783d293b 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,7 +1,7 @@ # Remove overall test dir (at most one of the two should match) and # remove file extension. test_name=$(echo -n "$test" | sed \ - -e "s|^unit-test-data/||" \ + -e "s|^tests/unit/[^/]*/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) diff --git a/mk/programs.mk b/mk/programs.mk index a88d9d949..6235311e9 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) + $(trace-test) $$($(1)_ENV) $$($(1)_PATH) endef diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk deleted file mode 100644 index 7689a03e0..000000000 --- a/src/libexpr/tests/local.mk +++ /dev/null @@ -1,23 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-tests_INSTALL_DIR := $(checkbindir) -else - libexpr-tests_INSTALL_DIR := -endif - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) - -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers - -libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers - -libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk deleted file mode 100644 index e9b8b4f99..000000000 --- a/src/libstore/tests/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libstore-tests-exe_RUN - -programs += libstore-tests-exe - -libstore-tests-exe_NAME = libnixstore-tests - -libstore-tests-exe_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests-exe_INSTALL_DIR := $(checkbindir) -else - libstore-tests-exe_INSTALL_DIR := -endif - -libstore-tests-exe_LIBS = libstore-tests - -libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests_INSTALL_DIR := $(checklibdir) -else - libstore-tests_INSTALL_DIR := -endif - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil - -libstore-tests_LIBS = libutil-tests libstore libutil - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk deleted file mode 100644 index 66886c45f..000000000 --- a/src/libutil/tests/local.mk +++ /dev/null @@ -1,41 +0,0 @@ -check: libutil-tests-exe_RUN - -programs += libutil-tests-exe - -libutil-tests-exe_NAME = libnixutil-tests - -libutil-tests-exe_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests-exe_INSTALL_DIR := $(checkbindir) -else - libutil-tests-exe_INSTALL_DIR := -endif - -libutil-tests-exe_LIBS = libutil-tests - -libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests_INSTALL_DIR := $(checklibdir) -else - libutil-tests_INSTALL_DIR := -endif - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_CXXFLAGS += -I src/libutil - -libutil-tests_LIBS = libutil - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -check: unit-test-data/libutil/git/check-data.sh.test - -$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh)) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk new file mode 100644 index 000000000..28e87b8f2 --- /dev/null +++ b/tests/unit/libexpr-support/local.mk @@ -0,0 +1,23 @@ +libraries += libexpr-test-support + +libexpr-test-support_NAME = libnixexpr-test-support + +libexpr-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-test-support_INSTALL_DIR := $(checklibdir) +else + libexpr-test-support_INSTALL_DIR := +endif + +libexpr-test-support_SOURCES := \ + $(wildcard $(d)/tests/*.cc) \ + $(wildcard $(d)/tests/value/*.cc) + +libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-test-support_LIBS = \ + libstore-test-support libutil-test-support \ + libexpr libstore libutil + +libexpr-test-support_LDFLAGS := -lrapidcheck diff --git a/src/libexpr/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh similarity index 100% rename from src/libexpr/tests/libexpr.hh rename to tests/unit/libexpr-support/tests/libexpr.hh diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc new file mode 100644 index 000000000..8658bdaef --- /dev/null +++ b/tests/unit/libexpr-support/tests/value/context.cc @@ -0,0 +1,30 @@ +#include + +#include "tests/path.hh" +#include "tests/value/context.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + case 2: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libexpr/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh similarity index 95% rename from src/libexpr/tests/value/context.hh rename to tests/unit/libexpr-support/tests/value/context.hh index c0bc97ba3..8c68c78bb 100644 --- a/src/libexpr/tests/value/context.hh +++ b/tests/unit/libexpr-support/tests/value/context.hh @@ -3,7 +3,7 @@ #include -#include +#include "value/context.hh" namespace rc { using namespace nix; diff --git a/src/libexpr/tests/derived-path.cc b/tests/unit/libexpr/derived-path.cc similarity index 100% rename from src/libexpr/tests/derived-path.cc rename to tests/unit/libexpr/derived-path.cc diff --git a/src/libexpr/tests/error_traces.cc b/tests/unit/libexpr/error_traces.cc similarity index 100% rename from src/libexpr/tests/error_traces.cc rename to tests/unit/libexpr/error_traces.cc diff --git a/src/libexpr/tests/flakeref.cc b/tests/unit/libexpr/flakeref.cc similarity index 100% rename from src/libexpr/tests/flakeref.cc rename to tests/unit/libexpr/flakeref.cc diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc similarity index 100% rename from src/libexpr/tests/json.cc rename to tests/unit/libexpr/json.cc diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk new file mode 100644 index 000000000..5743880d7 --- /dev/null +++ b/tests/unit/libexpr/local.mk @@ -0,0 +1,36 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libexpr-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) + +libexpr-tests_EXTRA_INCLUDES = \ + -I tests/unit/libexpr-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libexpr \ + -I src/libfetchers \ + -I src/libstore \ + -I src/libutil + +libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-tests_LIBS = \ + libexpr-test-support libstore-test-support libutils-test-support \ + libexpr libfetchers libstore libutil + +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc similarity index 100% rename from src/libexpr/tests/primops.cc rename to tests/unit/libexpr/primops.cc diff --git a/src/libexpr/tests/search-path.cc b/tests/unit/libexpr/search-path.cc similarity index 100% rename from src/libexpr/tests/search-path.cc rename to tests/unit/libexpr/search-path.cc diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc similarity index 100% rename from src/libexpr/tests/trivial.cc rename to tests/unit/libexpr/trivial.cc diff --git a/src/libexpr/tests/value/context.cc b/tests/unit/libexpr/value/context.cc similarity index 83% rename from src/libexpr/tests/value/context.cc rename to tests/unit/libexpr/value/context.cc index 92d4889ab..761286dbd 100644 --- a/src/libexpr/tests/value/context.cc +++ b/tests/unit/libexpr/value/context.cc @@ -117,36 +117,6 @@ TEST(NixStringContextElemTest, built_built_xp) { NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::DrvDeep { - .drvPath = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - case 2: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libexpr/tests/value/print.cc b/tests/unit/libexpr/value/print.cc similarity index 100% rename from src/libexpr/tests/value/print.cc rename to tests/unit/libexpr/value/print.cc diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk new file mode 100644 index 000000000..d5d657c91 --- /dev/null +++ b/tests/unit/libstore-support/local.mk @@ -0,0 +1,21 @@ +libraries += libstore-test-support + +libstore-test-support_NAME = libnixstore-test-support + +libstore-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-test-support_INSTALL_DIR := $(checklibdir) +else + libstore-test-support_INSTALL_DIR := +endif + +libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-test-support_LIBS = \ + libutil-test-support \ + libstore libutil + +libstore-test-support_LDFLAGS := -lrapidcheck diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc new file mode 100644 index 000000000..091706dba --- /dev/null +++ b/tests/unit/libstore-support/tests/derived-path.cc @@ -0,0 +1,57 @@ +#include + +#include + +#include "tests/derived-path.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .outputs = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh similarity index 100% rename from src/libstore/tests/derived-path.hh rename to tests/unit/libstore-support/tests/derived-path.hh diff --git a/src/libstore/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh similarity index 100% rename from src/libstore/tests/libstore.hh rename to tests/unit/libstore-support/tests/libstore.hh diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc new file mode 100644 index 000000000..e9d602203 --- /dev/null +++ b/tests/unit/libstore-support/tests/outputs-spec.cc @@ -0,0 +1,24 @@ +#include "tests/outputs-spec.hh" + +#include + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + case 1: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh similarity index 89% rename from src/libstore/tests/outputs-spec.hh rename to tests/unit/libstore-support/tests/outputs-spec.hh index ded331b33..f5bf9042d 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/tests/unit/libstore-support/tests/outputs-spec.hh @@ -5,7 +5,7 @@ #include -#include +#include "tests/path.hh" namespace rc { using namespace nix; diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc new file mode 100644 index 000000000..e5f169e94 --- /dev/null +++ b/tests/unit/libstore-support/tests/path.cc @@ -0,0 +1,82 @@ +#include + +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/path.hh" + +namespace nix { + +void showValue(const StorePath & p, std::ostream & os) +{ + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - StorePath::HashLen); + + std::string pre; + pre.reserve(len); + + for (size_t c = 0; c < len; ++c) { + switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { + case 0 ... 9: + pre += '0' + i; + case 10 ... 35: + pre += 'A' + (i - 10); + break; + case 36 ... 61: + pre += 'a' + (i - 36); + break; + case 62: + pre += '+'; + break; + case 63: + pre += '-'; + break; + case 64: + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } + case 65: + pre += '_'; + break; + case 66: + pre += '?'; + break; + case 67: + pre += '='; + break; + default: + assert(false); + } + } + + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); +} + +} // namespace rc diff --git a/src/libstore/tests/path.hh b/tests/unit/libstore-support/tests/path.hh similarity index 82% rename from src/libstore/tests/path.hh rename to tests/unit/libstore-support/tests/path.hh index 21cb62310..4751b3373 100644 --- a/src/libstore/tests/path.hh +++ b/tests/unit/libstore-support/tests/path.hh @@ -11,6 +11,9 @@ struct StorePathName { std::string name; }; +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os); + } namespace rc { diff --git a/src/libstore/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh similarity index 96% rename from src/libstore/tests/protocol.hh rename to tests/unit/libstore-support/tests/protocol.hh index 466032a79..3c9e52c11 100644 --- a/src/libstore/tests/protocol.hh +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -12,7 +12,7 @@ namespace nix { template class ProtoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; + Path unitTestData = getUnitTestData() + "/" + protocolDir; Path goldenMaster(std::string_view testStem) const override { return unitTestData + "/" + testStem + ".bin"; diff --git a/src/libstore/tests/common-protocol.cc b/tests/unit/libstore/common-protocol.cc similarity index 100% rename from src/libstore/tests/common-protocol.cc rename to tests/unit/libstore/common-protocol.cc diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/content-address.bin rename to tests/unit/libstore/data/common-protocol/content-address.bin diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/drv-output.bin rename to tests/unit/libstore/data/common-protocol/drv-output.bin diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-content-address.bin rename to tests/unit/libstore/data/common-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-store-path.bin rename to tests/unit/libstore/data/common-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/realisation.bin rename to tests/unit/libstore/data/common-protocol/realisation.bin diff --git a/unit-test-data/libstore/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/set.bin rename to tests/unit/libstore/data/common-protocol/set.bin diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/store-path.bin rename to tests/unit/libstore/data/common-protocol/store-path.bin diff --git a/unit-test-data/libstore/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/string.bin rename to tests/unit/libstore/data/common-protocol/string.bin diff --git a/unit-test-data/libstore/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/vector.bin rename to tests/unit/libstore/data/common-protocol/vector.bin diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv similarity index 100% rename from unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv rename to tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv similarity index 100% rename from unit-test-data/libstore/derivation/bad-version.drv rename to tests/unit/libstore/data/derivation/bad-version.drv diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv similarity index 100% rename from unit-test-data/libstore/derivation/dynDerivationDeps.drv rename to tests/unit/libstore/data/derivation/dynDerivationDeps.drv diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json similarity index 100% rename from unit-test-data/libstore/derivation/dynDerivationDeps.json rename to tests/unit/libstore/data/derivation/dynDerivationDeps.json diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedFlat.json rename to tests/unit/libstore/data/derivation/output-caFixedFlat.json diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedNAR.json rename to tests/unit/libstore/data/derivation/output-caFixedNAR.json diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedText.json rename to tests/unit/libstore/data/derivation/output-caFixedText.json diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFloating.json rename to tests/unit/libstore/data/derivation/output-caFloating.json diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json similarity index 100% rename from unit-test-data/libstore/derivation/output-deferred.json rename to tests/unit/libstore/data/derivation/output-deferred.json diff --git a/unit-test-data/libstore/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json similarity index 100% rename from unit-test-data/libstore/derivation/output-impure.json rename to tests/unit/libstore/data/derivation/output-impure.json diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json similarity index 100% rename from unit-test-data/libstore/derivation/output-inputAddressed.json rename to tests/unit/libstore/data/derivation/output-inputAddressed.json diff --git a/unit-test-data/libstore/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv similarity index 100% rename from unit-test-data/libstore/derivation/simple.drv rename to tests/unit/libstore/data/derivation/simple.drv diff --git a/unit-test-data/libstore/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json similarity index 100% rename from unit-test-data/libstore/derivation/simple.json rename to tests/unit/libstore/data/derivation/simple.json diff --git a/unit-test-data/libstore/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json similarity index 100% rename from unit-test-data/libstore/nar-info/impure.json rename to tests/unit/libstore/data/nar-info/impure.json diff --git a/unit-test-data/libstore/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json similarity index 100% rename from unit-test-data/libstore/nar-info/pure.json rename to tests/unit/libstore/data/nar-info/pure.json diff --git a/unit-test-data/libstore/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json similarity index 100% rename from unit-test-data/libstore/path-info/impure.json rename to tests/unit/libstore/data/path-info/impure.json diff --git a/unit-test-data/libstore/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json similarity index 100% rename from unit-test-data/libstore/path-info/pure.json rename to tests/unit/libstore/data/path-info/pure.json diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.2.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.2.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.3.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.3.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.6.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.6.bin diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/content-address.bin rename to tests/unit/libstore/data/serve-protocol/content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/drv-output.bin rename to tests/unit/libstore/data/serve-protocol/drv-output.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-content-address.bin rename to tests/unit/libstore/data/serve-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-store-path.bin rename to tests/unit/libstore/data/serve-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/realisation.bin rename to tests/unit/libstore/data/serve-protocol/realisation.bin diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/set.bin rename to tests/unit/libstore/data/serve-protocol/set.bin diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/store-path.bin rename to tests/unit/libstore/data/serve-protocol/store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/string.bin rename to tests/unit/libstore/data/serve-protocol/string.bin diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/vector.bin rename to tests/unit/libstore/data/serve-protocol/vector.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.27.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.27.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.28.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.28.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/content-address.bin rename to tests/unit/libstore/data/worker-protocol/content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.29.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.30.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin diff --git a/unit-test-data/libstore/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/drv-output.bin rename to tests/unit/libstore/data/worker-protocol/drv-output.bin diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-content-address.bin rename to tests/unit/libstore/data/worker-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-store-path.bin rename to tests/unit/libstore/data/worker-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin rename to tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin diff --git a/unit-test-data/libstore/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/realisation.bin rename to tests/unit/libstore/data/worker-protocol/realisation.bin diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/set.bin rename to tests/unit/libstore/data/worker-protocol/set.bin diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/store-path.bin rename to tests/unit/libstore/data/worker-protocol/store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/string.bin rename to tests/unit/libstore/data/worker-protocol/string.bin diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/vector.bin rename to tests/unit/libstore/data/worker-protocol/vector.bin diff --git a/src/libstore/tests/derivation.cc b/tests/unit/libstore/derivation.cc similarity index 99% rename from src/libstore/tests/derivation.cc rename to tests/unit/libstore/derivation.cc index 7becfa5ab..a7f4488fa 100644 --- a/src/libstore/tests/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -13,7 +13,7 @@ using nlohmann::json; class DerivationTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/derivation"; + Path unitTestData = getUnitTestData() + "/derivation"; public: Path goldenMaster(std::string_view testStem) const override { diff --git a/src/libstore/tests/derived-path.cc b/tests/unit/libstore/derived-path.cc similarity index 66% rename from src/libstore/tests/derived-path.cc rename to tests/unit/libstore/derived-path.cc index 3fa3c0801..c62d79a78 100644 --- a/src/libstore/tests/derived-path.cc +++ b/tests/unit/libstore/derived-path.cc @@ -1,64 +1,11 @@ #include -#include #include #include #include "tests/derived-path.hh" #include "tests/libstore.hh" -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Opaque { - .path = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(SingleDerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .output = (*gen::arbitrary()).name, - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .outputs = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - namespace nix { class DerivedPathTest : public LibStoreTest diff --git a/src/libstore/tests/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc similarity index 100% rename from src/libstore/tests/downstream-placeholder.cc rename to tests/unit/libstore/downstream-placeholder.cc diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk new file mode 100644 index 000000000..63f6d011f --- /dev/null +++ b/tests/unit/libstore/local.mk @@ -0,0 +1,31 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_NAME = libnixstore-tests + +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libstore-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checkbindir) +else + libstore-tests_INSTALL_DIR := +endif + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libstore \ + -I src/libutil + +libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libstore libutil + +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc similarity index 97% rename from src/libstore/tests/machines.cc rename to tests/unit/libstore/machines.cc index fede328ea..5b66e5a5b 100644 --- a/src/libstore/tests/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -139,7 +139,7 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath("src/libstore/tests/test-data/machines.valid"); + auto path = absPath("tests/unit/libstore/test-data/machines.valid"); ASSERT_TRUE(pathExists(path)); settings.builders = std::string("@") + path; @@ -166,6 +166,6 @@ TEST(machines, getMachinesWithIncorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { - settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format"); EXPECT_THROW(getMachines(), FormatError); } diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc similarity index 100% rename from src/libstore/tests/nar-info-disk-cache.cc rename to tests/unit/libstore/nar-info-disk-cache.cc diff --git a/src/libstore/tests/nar-info.cc b/tests/unit/libstore/nar-info.cc similarity index 98% rename from src/libstore/tests/nar-info.cc rename to tests/unit/libstore/nar-info.cc index c5b21d56b..31e20ca14 100644 --- a/src/libstore/tests/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -12,7 +12,7 @@ using nlohmann::json; class NarInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/nar-info"; + Path unitTestData = getUnitTestData() + "/nar-info"; Path goldenMaster(PathView testStem) const override { return unitTestData + "/" + testStem + ".json"; diff --git a/src/libstore/tests/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc similarity index 92% rename from src/libstore/tests/outputs-spec.cc rename to tests/unit/libstore/outputs-spec.cc index 952945185..456196be1 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/tests/unit/libstore/outputs-spec.cc @@ -1,4 +1,4 @@ -#include "outputs-spec.hh" +#include "tests/outputs-spec.hh" #include #include @@ -199,31 +199,6 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just((OutputsSpec) OutputsSpec::All { }); - case 1: - return gen::just((OutputsSpec) OutputsSpec::Names { - *gen::nonEmpty(gen::container(gen::map( - gen::arbitrary(), - [](StorePathName n) { return n.name; }))), - }); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libstore/tests/path-info.cc b/tests/unit/libstore/path-info.cc similarity index 97% rename from src/libstore/tests/path-info.cc rename to tests/unit/libstore/path-info.cc index 49bf623bd..18f00ca19 100644 --- a/src/libstore/tests/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -12,7 +12,7 @@ using nlohmann::json; class PathInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/path-info"; + Path unitTestData = getUnitTestData() + "/path-info"; Path goldenMaster(PathView testStem) const override { return unitTestData + "/" + testStem + ".json"; diff --git a/src/libstore/tests/path.cc b/tests/unit/libstore/path.cc similarity index 59% rename from src/libstore/tests/path.cc rename to tests/unit/libstore/path.cc index 5a84d646c..30631b5fd 100644 --- a/src/libstore/tests/path.cc +++ b/tests/unit/libstore/path.cc @@ -66,79 +66,6 @@ TEST_DO_PARSE(equals_sign, "foo=foo") #undef TEST_DO_PARSE -// For rapidcheck -void showValue(const StorePath & p, std::ostream & os) { - os << p.to_string(); -} - -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - auto len = *gen::inRange( - 1, - StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - - std::string pre; - pre.reserve(len); - - for (size_t c = 0; c < len; ++c) { - switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { - case 0 ... 9: - pre += '0' + i; - case 10 ... 35: - pre += 'A' + (i - 10); - break; - case 36 ... 61: - pre += 'a' + (i - 36); - break; - case 62: - pre += '+'; - break; - case 63: - pre += '-'; - break; - case 64: - // names aren't permitted to start with a period, - // so just fall through to the next case here - if (c != 0) { - pre += '.'; - break; - } - case 65: - pre += '_'; - break; - case 66: - pre += '?'; - break; - case 67: - pre += '='; - break; - default: - assert(false); - } - } - - return gen::just(StorePathName { - .name = std::move(pre), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(StorePath { - *gen::arbitrary(), - (*gen::arbitrary()).name, - }); -} - -} // namespace rc - -namespace nix { - #ifndef COVERAGE RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc similarity index 100% rename from src/libstore/tests/references.cc rename to tests/unit/libstore/references.cc diff --git a/src/libstore/tests/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc similarity index 100% rename from src/libstore/tests/serve-protocol.cc rename to tests/unit/libstore/serve-protocol.cc diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/test-data/machines.bad_format similarity index 100% rename from src/libstore/tests/test-data/machines.bad_format rename to tests/unit/libstore/test-data/machines.bad_format diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/test-data/machines.valid similarity index 100% rename from src/libstore/tests/test-data/machines.valid rename to tests/unit/libstore/test-data/machines.valid diff --git a/src/libstore/tests/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc similarity index 100% rename from src/libstore/tests/worker-protocol.cc rename to tests/unit/libstore/worker-protocol.cc diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk new file mode 100644 index 000000000..43a1551e5 --- /dev/null +++ b/tests/unit/libutil-support/local.mk @@ -0,0 +1,19 @@ +libraries += libutil-test-support + +libutil-test-support_NAME = libnixutil-test-support + +libutil-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-test-support_INSTALL_DIR := $(checklibdir) +else + libutil-test-support_INSTALL_DIR := +endif + +libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-test-support_LIBS = libutil + +libutil-test-support_LDFLAGS := -lrapidcheck diff --git a/src/libutil/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh similarity index 95% rename from src/libutil/tests/characterization.hh rename to tests/unit/libutil-support/tests/characterization.hh index 6eb513d68..9d6c850f0 100644 --- a/src/libutil/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -9,8 +9,8 @@ namespace nix { /** - * The path to the `unit-test-data` directory. See the contributing - * guide in the manual for further details. + * The path to the unit test data directory. See the contributing guide + * in the manual for further details. */ static Path getUnitTestData() { return getEnv("_NIX_TEST_UNIT_DATA").value(); diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc new file mode 100644 index 000000000..577e9890e --- /dev/null +++ b/tests/unit/libutil-support/tests/hash.cc @@ -0,0 +1,20 @@ +#include + +#include + +#include "hash.hh" + +#include "tests/hash.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh similarity index 100% rename from src/libutil/tests/hash.hh rename to tests/unit/libutil-support/tests/hash.hh diff --git a/src/libutil/tests/args.cc b/tests/unit/libutil/args.cc similarity index 98% rename from src/libutil/tests/args.cc rename to tests/unit/libutil/args.cc index bea74a8c8..950224430 100644 --- a/src/libutil/tests/args.cc +++ b/tests/unit/libutil/args.cc @@ -1,5 +1,5 @@ -#include "../args.hh" -#include "libutil/fs-sink.hh" +#include "args.hh" +#include "fs-sink.hh" #include #include @@ -165,4 +165,4 @@ RC_GTEST_PROP( #endif -} \ No newline at end of file +} diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc similarity index 100% rename from src/libutil/tests/canon-path.cc rename to tests/unit/libutil/canon-path.cc diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc similarity index 100% rename from src/libutil/tests/chunked-vector.cc rename to tests/unit/libutil/chunked-vector.cc diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc similarity index 100% rename from src/libutil/tests/closure.cc rename to tests/unit/libutil/closure.cc diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc similarity index 100% rename from src/libutil/tests/compression.cc rename to tests/unit/libutil/compression.cc diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc similarity index 100% rename from src/libutil/tests/config.cc rename to tests/unit/libutil/config.cc diff --git a/unit-test-data/libutil/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh similarity index 98% rename from unit-test-data/libutil/git/check-data.sh rename to tests/unit/libutil/data/git/check-data.sh index 68b705c95..b3f59c4f1 100644 --- a/unit-test-data/libutil/git/check-data.sh +++ b/tests/unit/libutil/data/git/check-data.sh @@ -2,7 +2,7 @@ set -eu -o pipefail -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data mkdir -p $TEST_ROOT repo="$TEST_ROOT/scratch" diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin similarity index 100% rename from unit-test-data/libutil/git/hello-world-blob.bin rename to tests/unit/libutil/data/git/hello-world-blob.bin diff --git a/unit-test-data/libutil/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin similarity index 100% rename from unit-test-data/libutil/git/hello-world.bin rename to tests/unit/libutil/data/git/hello-world.bin diff --git a/unit-test-data/libutil/git/tree.bin b/tests/unit/libutil/data/git/tree.bin similarity index 100% rename from unit-test-data/libutil/git/tree.bin rename to tests/unit/libutil/data/git/tree.bin diff --git a/unit-test-data/libutil/git/tree.txt b/tests/unit/libutil/data/git/tree.txt similarity index 100% rename from unit-test-data/libutil/git/tree.txt rename to tests/unit/libutil/data/git/tree.txt diff --git a/src/libutil/tests/git.cc b/tests/unit/libutil/git.cc similarity index 97% rename from src/libutil/tests/git.cc rename to tests/unit/libutil/git.cc index 2842ea4d0..551a2d105 100644 --- a/src/libutil/tests/git.cc +++ b/tests/unit/libutil/git.cc @@ -11,7 +11,7 @@ using namespace git; class GitTest : public CharacterizationTest { - Path unitTestData = getUnitTestData() + "/libutil/git"; + Path unitTestData = getUnitTestData() + "/git"; public: @@ -86,8 +86,8 @@ TEST_F(GitTest, blob_write) { /** * This data is for "shallow" tree tests. However, we use "real" hashes - * so that we can check our test data in the corresponding functional - * test (`git-hashing/unit-test-data`). + * so that we can check our test data in a small shell script test test + * (`tests/unit/libutil/data/git/check-data.sh`). */ const static Tree tree = { { diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc similarity index 92% rename from src/libutil/tests/hash.cc rename to tests/unit/libutil/hash.cc index 9a5ebbb30..92291afce 100644 --- a/src/libutil/tests/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -1,12 +1,8 @@ #include -#include #include -#include -#include - -#include "tests/hash.hh" +#include "hash.hh" namespace nix { @@ -68,7 +64,6 @@ namespace nix { "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); } - TEST(hashString, testKnownSHA512Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; @@ -95,16 +90,3 @@ namespace nix { ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); } } - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - Hash hash(htSHA1); - for (size_t i = 0; i < hash.hashSize; ++i) - hash.hash[i] = *gen::arbitrary(); - return gen::just(hash); -} - -} diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc similarity index 100% rename from src/libutil/tests/hilite.cc rename to tests/unit/libutil/hilite.cc diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk new file mode 100644 index 000000000..930efb90b --- /dev/null +++ b/tests/unit/libutil/local.mk @@ -0,0 +1,31 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_NAME = libnixutil-tests + +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libutil-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checkbindir) +else + libutil-tests_INSTALL_DIR := +endif + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_EXTRA_INCLUDES = \ + -I tests/unit/libutil-support \ + -I src/libutil + +libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-tests_LIBS = libutil-test-support libutil + +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: $(d)/data/git/check-data.sh.test + +$(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc similarity index 100% rename from src/libutil/tests/logging.cc rename to tests/unit/libutil/logging.cc diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc similarity index 100% rename from src/libutil/tests/lru-cache.cc rename to tests/unit/libutil/lru-cache.cc diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc similarity index 100% rename from src/libutil/tests/pool.cc rename to tests/unit/libutil/pool.cc diff --git a/src/libutil/tests/references.cc b/tests/unit/libutil/references.cc similarity index 100% rename from src/libutil/tests/references.cc rename to tests/unit/libutil/references.cc diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc similarity index 100% rename from src/libutil/tests/suggestions.cc rename to tests/unit/libutil/suggestions.cc diff --git a/src/libutil/tests/tests.cc b/tests/unit/libutil/tests.cc similarity index 100% rename from src/libutil/tests/tests.cc rename to tests/unit/libutil/tests.cc diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc similarity index 100% rename from src/libutil/tests/url.cc rename to tests/unit/libutil/url.cc diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc similarity index 100% rename from src/libutil/tests/xml-writer.cc rename to tests/unit/libutil/xml-writer.cc From 9c42b2c954a8707d2ce084097228f69d467f0d89 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 14 Nov 2023 11:40:56 +0100 Subject: [PATCH 384/402] reword description for the `fetch-tree` experimental feature without knowing a lot of context, it's not clear who "we" are in that text. I'm also strongly opposed to adding procedural notes into a reference manual; it just won't age well. this change leaves a factual description of the experimental feature and its purpose. (cherry picked from commit 3c6244b55e36980d3ecc6cc978c18a8c20fb13e1) --- src/libutil/experimental-features.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ac4d189e1..86665cd9d 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -80,12 +80,11 @@ constexpr std::array xpFeatureDetails .description = R"( Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. - `fetchTree` exposes a large suite of fetching functionality in a more systematic way. + `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. + This built-in was previously guarded by the `flakes` experimental feature because of that overlap. - This built-in was previously guarded by the `flakes` experimental feature because of that overlap, - but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. - Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation. + Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation. )", }, { From aaeab0040126d7e0bf59886abd4a1e2855482ee9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 14 Nov 2023 11:44:34 +0100 Subject: [PATCH 385/402] fix up release note (cherry picked from commit 2ece9d5b925ab455c31b78d209e2412fd2d133d3) --- doc/manual/src/release-notes/rl-2.19.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index 4eecaf929..ba6eb9c64 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -18,7 +18,7 @@ - `nix-shell` shebang lines now support single-quoted arguments. - `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes. + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: From 45cde5a343cbca2ccb63c770bc37f35fcfd792a5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 5 Dec 2023 18:10:37 -0500 Subject: [PATCH 386/402] Add missing `-pthread` for test support libraries This is good in general (see how the other libraries also have long had it, since 49fe9592a47e7819179c2de4fd6068e897e944c7) but in particular needed to fix the NetBSD build. (cherry picked from commit b23273f6a29c725646b3523b1c35a0ae4a84ef61) --- tests/unit/libexpr-support/local.mk | 2 +- tests/unit/libstore-support/local.mk | 2 +- tests/unit/libutil-support/local.mk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk index 28e87b8f2..12a76206a 100644 --- a/tests/unit/libexpr-support/local.mk +++ b/tests/unit/libexpr-support/local.mk @@ -20,4 +20,4 @@ libexpr-test-support_LIBS = \ libstore-test-support libutil-test-support \ libexpr libstore libutil -libexpr-test-support_LDFLAGS := -lrapidcheck +libexpr-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk index d5d657c91..ff075c96a 100644 --- a/tests/unit/libstore-support/local.mk +++ b/tests/unit/libstore-support/local.mk @@ -18,4 +18,4 @@ libstore-test-support_LIBS = \ libutil-test-support \ libstore libutil -libstore-test-support_LDFLAGS := -lrapidcheck +libstore-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk index 43a1551e5..2ee2cdb6c 100644 --- a/tests/unit/libutil-support/local.mk +++ b/tests/unit/libutil-support/local.mk @@ -16,4 +16,4 @@ libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) libutil-test-support_LIBS = libutil -libutil-test-support_LDFLAGS := -lrapidcheck +libutil-test-support_LDFLAGS := -pthread -lrapidcheck From e6a03920ad59428ae5e7219760df91733562d790 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 00:44:55 -0500 Subject: [PATCH 387/402] Give `Derivation::tryResolve` an `evalStore` argument *N.B. Backport is modified not to change any call sites / behavior.* This is needed for building CA deriations with a src store / dest store split. In particular it is needed for Hydra. https://github.com/NixOS/hydra/issues/838 currently puts realizations, and thus build outputs, in the local store, but it should not. (cherry picked with modifications from commit 96dd757b0c0f3d6702f8e38467a8bf467b43154e) --- src/libstore/derivations.cc | 4 ++-- src/libstore/derivations.hh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6d9c8b9d6..10962d142 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1002,13 +1002,13 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) const +std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { std::map, StorePath> inputDrvOutputs; std::function::ChildNode &)> accum; accum = [&](auto & inputDrv, auto & node) { - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) { if (outputPath) { inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); if (auto p = get(node.childMap, outputName)) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fa14e7536..7309918ce 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -340,7 +340,7 @@ struct Derivation : BasicDerivation * 2. Input placeholders are replaced with realized input store * paths. */ - std::optional tryResolve(Store & store) const; + std::optional tryResolve(Store & store, Store * evalStore = nullptr) const; /** * Like the above, but instead of querying the Nix database for From 5fc116a620f3f11ee0764741249e26b626e87638 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 18:51:23 -0500 Subject: [PATCH 388/402] Give `Store::queryDerivationOutputMap` and `evalStore` argument Picking up where https://github.com/NixOS/nix/pull/9563 left off. (cherry picked from commit 5f30c8acc7e0cad08924cc53e350e811d097fae7) --- src/libstore/store-api.cc | 4 ++-- src/libstore/store-api.hh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0f88d9b92..fe1903ec6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -545,8 +545,8 @@ std::map> Store::queryPartialDerivationOut return outputs; } -OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { - auto resp = queryPartialDerivationOutputMap(path); +OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) { + auto resp = queryPartialDerivationOutputMap(path, evalStore); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 32ad2aa44..a58dd1535 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -460,7 +460,7 @@ public: * Query the mapping outputName=>outputPath for the given derivation. * Assume every output has a mapping and throw an exception otherwise. */ - OutputPathMap queryDerivationOutputMap(const StorePath & path); + OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr); /** * Query the full store path given the hash part of a valid store From f72b0b5b000dba1edf499e53dcd6934144306a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Mon, 11 Dec 2023 16:02:09 +0100 Subject: [PATCH 389/402] Fix query parsing for path-like flakes (cherry picked from commit f45d2ee2b7090560fc30a227d638684268af700d) --- src/libexpr/flake/flakeref.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 16f45ace7..8b0eb7460 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -90,7 +90,7 @@ std::pair parsePathFlakeRefWithFragment( fragment = percentDecode(url.substr(fragmentStart+1)); } if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); } if (baseDir) { From 1e92097ce3f0470442316e3ae8daa09348aa07a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Mon, 11 Dec 2023 16:05:34 +0100 Subject: [PATCH 390/402] Add test cases for flake urls with fragments (cherry picked from commit 994f1b5c0de44319992ef6b1b106cee3fa400dc4) --- tests/functional/flakes/flakes.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index ccf1699f9..7506b6b3b 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -193,6 +193,14 @@ nix build -o "$TEST_ROOT/result" flake1 nix build -o "$TEST_ROOT/result" "$flake1Dir" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" +# Test explicit packages.default. +nix build -o "$TEST_ROOT/result" "$flake1Dir#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" + +# Test explicit packages.default with query. +nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" + # Check that store symlinks inside a flake are not interpreted as flakes. nix build -o "$flake1Dir/result" "git+file://$flake1Dir" nix path-info "$flake1Dir/result" From 90c7904abfcbe4ce82c110833c4be710c8608054 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 12:41:47 +0100 Subject: [PATCH 391/402] isAllowedURI: Extract function and test (cherry picked from commit 91ba7b230777e3fb023bda48c269d533702e50e8) --- src/libexpr/eval.cc | 19 +++++-- src/libexpr/eval.hh | 5 ++ tests/unit/libexpr/eval.cc | 106 +++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 tests/unit/libexpr/eval.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 46a49c891..2d9dd14b1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -602,6 +602,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } + SourcePath EvalState::checkSourcePath(const SourcePath & path_) { // Don't check non-rootFS accessors, they're in a different namespace. @@ -650,21 +651,29 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) } -void EvalState::checkURI(const std::string & uri) +bool isAllowedURI(std::string_view uri, const Strings & allowedUris) { - if (!evalSettings.restrictEval) return; - /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit access to https://github.com. Note: this allows 'http://' and 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : evalSettings.allowedUris.get()) + for (auto & prefix : allowedUris) { if (uri == prefix || (uri.size() > prefix.size() && prefix.size() > 0 && hasPrefix(uri, prefix) && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) - return; + return true; + } + + return false; +} + +void EvalState::checkURI(const std::string & uri) +{ + if (!evalSettings.restrictEval) return; + + if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9a92992c1..23fbbf0af 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -841,6 +841,11 @@ std::string showType(const Value & v); */ SourcePath resolveExprPath(SourcePath path); +/** + * Whether a URI is allowed, assuming restrictEval is enabled + */ +bool isAllowedURI(std::string_view uri, const Strings & allowedPaths); + struct InvalidPathError : EvalError { Path path; diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc new file mode 100644 index 000000000..cc5d6bbfa --- /dev/null +++ b/tests/unit/libexpr/eval.cc @@ -0,0 +1,106 @@ +#include +#include + +#include "eval.hh" +#include "tests/libexpr.hh" + +namespace nix { + +TEST(nix_isAllowedURI, http_example_com) { + Strings allowed; + allowed.push_back("http://example.com"); + + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.co", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); +} + +TEST(nix_isAllowedURI, http_example_com_foo) { + Strings allowed; + allowed.push_back("http://example.com/foo"); + + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); + // Broken? + // ASSERT_TRUE(isAllowedURI("http://example.com/foo?ok=1", allowed)); +} + +TEST(nix_isAllowedURI, http) { + Strings allowed; + allowed.push_back("http://"); + + ASSERT_TRUE(isAllowedURI("http://", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("https://", allowed)); + ASSERT_FALSE(isAllowedURI("http:foo", allowed)); +} + +TEST(nix_isAllowedURI, https) { + Strings allowed; + allowed.push_back("https://"); + + ASSERT_TRUE(isAllowedURI("https://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("https://example.com/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/https:", allowed)); +} + +TEST(nix_isAllowedURI, absolute_path) { + Strings allowed; + allowed.push_back("/var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("/var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); +} + +TEST(nix_isAllowedURI, file_url) { + Strings allowed; + allowed.push_back("file:///var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("file:///var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http:///var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///", allowed)); + ASSERT_FALSE(isAllowedURI("file://", allowed)); +} + +} // namespace nix \ No newline at end of file From ec5e4041bab00bb75d43081c3671162aaceaa384 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 12:43:20 +0100 Subject: [PATCH 392/402] isAllowedURI: Remove incorrect note (cherry picked from commit 6cbba914a70eb5da6447fee5528a63723ed13245) --- src/libexpr/eval.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2d9dd14b1..624630c99 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -655,8 +655,7 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) { /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit - access to https://github.com. Note: this allows 'http://' and - 'https://' as prefixes for any http/https URI. */ + access to https://github.com. */ for (auto & prefix : allowedUris) { if (uri == prefix || (uri.size() > prefix.size() From 4795569bf7a8bd9209f3abaa0e1be16790c3d7ba Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 14:08:22 +0100 Subject: [PATCH 393/402] isAllowedURI: Format (cherry picked from commit 1fa958dda1ef0cb37441ef8d1a84faf6d501ac12) --- src/libexpr/eval.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 624630c99..9ee39625b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -657,11 +657,14 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) prefix. Thus, the prefix https://github.co does not permit access to https://github.com. */ for (auto & prefix : allowedUris) { - if (uri == prefix || - (uri.size() > prefix.size() - && prefix.size() > 0 - && hasPrefix(uri, prefix) - && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) + if (uri == prefix + // Allow access to subdirectories of the prefix. + || (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && ( + prefix[prefix.size() - 1] == '/' + || uri[prefix.size()] == '/'))) return true; } From 772a8efff4f14048dd48ba68f2435b813687076f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 14:39:49 +0100 Subject: [PATCH 394/402] Add nix::isASCII*, locale-independent (cherry picked from commit 79eb2920bb51c7ec9528a403986e79f04738e2be) --- src/libutil/string.hh | 17 +++++++++++ tests/unit/libutil/string.cc | 59 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/libutil/string.hh create mode 100644 tests/unit/libutil/string.cc diff --git a/src/libutil/string.hh b/src/libutil/string.hh new file mode 100644 index 000000000..16ef75643 --- /dev/null +++ b/src/libutil/string.hh @@ -0,0 +1,17 @@ +#pragma once + +namespace nix { + + /** Locale-independent version of std::islower(). */ + inline bool isASCIILower(char c) { return c >= 'a' && c <= 'z'; }; + + /** Locale-independent version of std::isupper(). */ + inline bool isASCIIUpper(char c) { return c >= 'A' && c <= 'Z'; }; + + /** Locale-independent version of std::isalpha(). */ + inline bool isASCIIAlpha(char c) { return isASCIILower(c) || isASCIIUpper(c); }; + + /** Locale-independent version of std::isdigit(). */ + inline bool isASCIIDigit(char c) { return c >= '0' && c <= '9'; }; + +} diff --git a/tests/unit/libutil/string.cc b/tests/unit/libutil/string.cc new file mode 100644 index 000000000..381f2cc15 --- /dev/null +++ b/tests/unit/libutil/string.cc @@ -0,0 +1,59 @@ +#include +#include "string.hh" + +namespace nix { + +TEST(string, isASCIILower) { + ASSERT_TRUE(isASCIILower('a')); + ASSERT_TRUE(isASCIILower('z')); + ASSERT_FALSE(isASCIILower('A')); + ASSERT_FALSE(isASCIILower('Z')); + ASSERT_FALSE(isASCIILower('0')); + ASSERT_FALSE(isASCIILower('9')); + ASSERT_FALSE(isASCIILower(' ')); + ASSERT_FALSE(isASCIILower('\n')); + ASSERT_FALSE(isASCIILower('\t')); + ASSERT_FALSE(isASCIILower(':')); +} + +TEST(string, isASCIIUpper) { + ASSERT_FALSE(isASCIIUpper('a')); + ASSERT_FALSE(isASCIIUpper('z')); + ASSERT_TRUE(isASCIIUpper('A')); + ASSERT_TRUE(isASCIIUpper('Z')); + ASSERT_FALSE(isASCIIUpper('0')); + ASSERT_FALSE(isASCIIUpper('9')); + ASSERT_FALSE(isASCIIUpper(' ')); + ASSERT_FALSE(isASCIIUpper('\n')); + ASSERT_FALSE(isASCIIUpper('\t')); + ASSERT_FALSE(isASCIIUpper(':')); +} + +TEST(string, isASCIIAlpha) { + ASSERT_TRUE(isASCIIAlpha('a')); + ASSERT_TRUE(isASCIIAlpha('z')); + ASSERT_TRUE(isASCIIAlpha('A')); + ASSERT_TRUE(isASCIIAlpha('Z')); + ASSERT_FALSE(isASCIIAlpha('0')); + ASSERT_FALSE(isASCIIAlpha('9')); + ASSERT_FALSE(isASCIIAlpha(' ')); + ASSERT_FALSE(isASCIIAlpha('\n')); + ASSERT_FALSE(isASCIIAlpha('\t')); + ASSERT_FALSE(isASCIIAlpha(':')); +} + +TEST(string, isASCIIDigit) { + ASSERT_FALSE(isASCIIDigit('a')); + ASSERT_FALSE(isASCIIDigit('z')); + ASSERT_FALSE(isASCIIDigit('A')); + ASSERT_FALSE(isASCIIDigit('Z')); + ASSERT_TRUE(isASCIIDigit('0')); + ASSERT_TRUE(isASCIIDigit('1')); + ASSERT_TRUE(isASCIIDigit('9')); + ASSERT_FALSE(isASCIIDigit(' ')); + ASSERT_FALSE(isASCIIDigit('\n')); + ASSERT_FALSE(isASCIIDigit('\t')); + ASSERT_FALSE(isASCIIDigit(':')); +} + +} \ No newline at end of file From 2116ee245495b6efa58d61e2ac801c52016dfe31 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 15:14:41 +0100 Subject: [PATCH 395/402] isValidSchemeName: Add function (cherry picked from commit d3a85b68347071d8d93ec796a38c707483d7b272) --- src/libutil/url.cc | 17 +++++++++++++++++ src/libutil/url.hh | 9 +++++++++ tests/unit/libutil/url.cc | 18 ++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9b438e6cd..f1d26a061 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -2,6 +2,7 @@ #include "url-parts.hh" #include "util.hh" #include "split.hh" +#include "string.hh" namespace nix { @@ -175,4 +176,20 @@ std::string fixGitURL(const std::string & url) } } +// https://www.rfc-editor.org/rfc/rfc3986#section-3.1 +bool isValidSchemeName(std::string_view s) +{ + if (s.empty()) return false; + if (!isASCIIAlpha(s[0])) return false; + for (auto c : s.substr(1)) { + if (isASCIIAlpha(c)) continue; + if (isASCIIDigit(c)) continue; + if (c == '+') continue; + if (c == '-') continue; + if (c == '.') continue; + return false; + } + return true; +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 26c2dcc28..603ab35b6 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -50,4 +50,13 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme); changes absolute paths into file:// URLs. */ std::string fixGitURL(const std::string & url); +/** + * Whether a string is valid as RFC 3986 scheme name. + * Colon `:` is part of the URI; not the scheme name, and therefore rejected. + * See https://www.rfc-editor.org/rfc/rfc3986#section-3.1 + * + * Does not check whether the scheme is understood, as that's context-dependent. + */ +bool isValidSchemeName(std::string_view scheme); + } diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc index a678dad20..09fa4e218 100644 --- a/tests/unit/libutil/url.cc +++ b/tests/unit/libutil/url.cc @@ -344,4 +344,22 @@ namespace nix { ASSERT_EQ(percentDecode(e), s); } +TEST(nix, isValidSchemeName) { + ASSERT_TRUE(isValidSchemeName("http")); + ASSERT_TRUE(isValidSchemeName("https")); + ASSERT_TRUE(isValidSchemeName("file")); + ASSERT_TRUE(isValidSchemeName("file+https")); + ASSERT_TRUE(isValidSchemeName("fi.le")); + ASSERT_TRUE(isValidSchemeName("file-ssh")); + ASSERT_TRUE(isValidSchemeName("file+")); + ASSERT_TRUE(isValidSchemeName("file.")); + ASSERT_TRUE(isValidSchemeName("file1")); + ASSERT_FALSE(isValidSchemeName("file:")); + ASSERT_FALSE(isValidSchemeName("file/")); + ASSERT_FALSE(isValidSchemeName("+file")); + ASSERT_FALSE(isValidSchemeName(".file")); + ASSERT_FALSE(isValidSchemeName("-file")); + ASSERT_FALSE(isValidSchemeName("1file")); +} + } From ffb624665084b29355994f752330ed4552d74e4b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 15:27:29 +0100 Subject: [PATCH 396/402] allowed-uris: Match whole schemes also when scheme is not followed by slashes (cherry picked from commit a05bc9eb92371af631fc9fb83c3595957fb56943) --- ...llowed-uris-can-now-match-whole-schemes.md | 7 ++++ src/libexpr/eval-settings.hh | 5 +++ src/libexpr/eval.cc | 17 ++++++++- tests/unit/libexpr/eval.cc | 35 +++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md diff --git a/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md new file mode 100644 index 000000000..3cf75a612 --- /dev/null +++ b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md @@ -0,0 +1,7 @@ +--- +synopsis: Option `allowed-uris` can now match whole schemes in URIs without slashes +prs: 9547 +--- + +If a scheme, such as `github:` is specified in the `allowed-uris` option, all URIs starting with `github:` are allowed. +Previously this only worked for schemes whose URIs used the `://` syntax. diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index db2971acb..3009a462c 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -68,6 +68,11 @@ struct EvalSettings : Config evaluation mode. For example, when set to `https://github.com/NixOS`, builtin functions such as `fetchGit` are allowed to access `https://github.com/NixOS/patchelf.git`. + + Access is granted when + - the URI is equal to the prefix, + - or the URI is a subpath of the prefix, + - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme. )"}; Setting traceFunctionCalls{this, false, "trace-function-calls", diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9ee39625b..e1fbddd45 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -16,6 +16,7 @@ #include "fs-input-accessor.hh" #include "memory-input-accessor.hh" #include "signals.hh" +#include "url.hh" #include #include @@ -602,6 +603,14 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } +inline static bool isJustSchemePrefix(std::string_view prefix) +{ + return + !prefix.empty() + && prefix[prefix.size() - 1] == ':' + && isValidSchemeName(prefix.substr(0, prefix.size() - 1)); +} + SourcePath EvalState::checkSourcePath(const SourcePath & path_) { @@ -663,8 +672,14 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) && prefix.size() > 0 && hasPrefix(uri, prefix) && ( + // Allow access to subdirectories of the prefix. prefix[prefix.size() - 1] == '/' - || uri[prefix.size()] == '/'))) + || uri[prefix.size()] == '/' + + // Allow access to whole schemes + || isJustSchemePrefix(prefix) + ) + )) return true; } diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc index cc5d6bbfa..93d3f658f 100644 --- a/tests/unit/libexpr/eval.cc +++ b/tests/unit/libexpr/eval.cc @@ -103,4 +103,39 @@ TEST(nix_isAllowedURI, file_url) { ASSERT_FALSE(isAllowedURI("file://", allowed)); } +TEST(nix_isAllowedURI, github_all) { + Strings allowed; + allowed.push_back("github:"); + ASSERT_TRUE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("github", allowed)); +} + +TEST(nix_isAllowedURI, github_org) { + Strings allowed; + allowed.push_back("github:foo"); + ASSERT_FALSE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_FALSE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); +} + +TEST(nix_isAllowedURI, non_scheme_colon) { + Strings allowed; + allowed.push_back("https://foo/bar:"); + ASSERT_TRUE(isAllowedURI("https://foo/bar:", allowed)); + ASSERT_TRUE(isAllowedURI("https://foo/bar:/baz", allowed)); + ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed)); +} + } // namespace nix \ No newline at end of file From 598b0e23172dfb2100e8dce6a327b6a4350850c1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:22:54 +0100 Subject: [PATCH 397/402] schemeRegex -> schemeNameRegex Scheme could be understood to include the typical `:` separator. (cherry picked from commit 2e451a663eff96b89360cfd3c0d5eaa60ca46181) --- src/libutil/url-parts.hh | 2 +- src/libutil/url.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 5c5a30dc2..07bc8d0cd 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; +const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)"; const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f1d26a061..40e3c814f 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -13,7 +13,7 @@ std::regex revRegex(revRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { static std::regex uriRegex( - "((" + schemeRegex + "):" + "((" + schemeNameRegex + "):" + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" + "(?:#(" + queryRegex + "))?", From ebdb6926fd9115d50f4be77595191c2a5194bb68 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:43:54 +0100 Subject: [PATCH 398/402] isValidSchemeName: Use regex As requested by Eelco Dolstra. I think it used to be simpler. (cherry picked from commit 4eaeda6604e2f8977728f14415fe92350d047970) --- src/libutil/url.cc | 15 +++------------ tests/unit/libutil/url.cc | 5 +++++ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 40e3c814f..fad438ce5 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -2,7 +2,6 @@ #include "url-parts.hh" #include "util.hh" #include "split.hh" -#include "string.hh" namespace nix { @@ -179,17 +178,9 @@ std::string fixGitURL(const std::string & url) // https://www.rfc-editor.org/rfc/rfc3986#section-3.1 bool isValidSchemeName(std::string_view s) { - if (s.empty()) return false; - if (!isASCIIAlpha(s[0])) return false; - for (auto c : s.substr(1)) { - if (isASCIIAlpha(c)) continue; - if (isASCIIDigit(c)) continue; - if (c == '+') continue; - if (c == '-') continue; - if (c == '.') continue; - return false; - } - return true; + static std::regex regex(schemeNameRegex, std::regex::ECMAScript); + + return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default); } } diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc index 09fa4e218..7d08f467e 100644 --- a/tests/unit/libutil/url.cc +++ b/tests/unit/libutil/url.cc @@ -360,6 +360,11 @@ TEST(nix, isValidSchemeName) { ASSERT_FALSE(isValidSchemeName(".file")); ASSERT_FALSE(isValidSchemeName("-file")); ASSERT_FALSE(isValidSchemeName("1file")); + // regex ok? + ASSERT_FALSE(isValidSchemeName("\nhttp")); + ASSERT_FALSE(isValidSchemeName("\nhttp\n")); + ASSERT_FALSE(isValidSchemeName("http\n")); + ASSERT_FALSE(isValidSchemeName("http ")); } } From 01cf57703a3adfebbff101b1370890fbda5fdf9c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:44:53 +0100 Subject: [PATCH 399/402] Revert "Add nix::isASCII*, locale-independent" This reverts commit 79eb2920bb51c7ec9528a403986e79f04738e2be. Not used at this time. (cherry picked from commit 0b87ba50c08d83384e11a8e6db1e2f97fba4b61c) --- src/libutil/string.hh | 17 ----------- tests/unit/libutil/string.cc | 59 ------------------------------------ 2 files changed, 76 deletions(-) delete mode 100644 src/libutil/string.hh delete mode 100644 tests/unit/libutil/string.cc diff --git a/src/libutil/string.hh b/src/libutil/string.hh deleted file mode 100644 index 16ef75643..000000000 --- a/src/libutil/string.hh +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -namespace nix { - - /** Locale-independent version of std::islower(). */ - inline bool isASCIILower(char c) { return c >= 'a' && c <= 'z'; }; - - /** Locale-independent version of std::isupper(). */ - inline bool isASCIIUpper(char c) { return c >= 'A' && c <= 'Z'; }; - - /** Locale-independent version of std::isalpha(). */ - inline bool isASCIIAlpha(char c) { return isASCIILower(c) || isASCIIUpper(c); }; - - /** Locale-independent version of std::isdigit(). */ - inline bool isASCIIDigit(char c) { return c >= '0' && c <= '9'; }; - -} diff --git a/tests/unit/libutil/string.cc b/tests/unit/libutil/string.cc deleted file mode 100644 index 381f2cc15..000000000 --- a/tests/unit/libutil/string.cc +++ /dev/null @@ -1,59 +0,0 @@ -#include -#include "string.hh" - -namespace nix { - -TEST(string, isASCIILower) { - ASSERT_TRUE(isASCIILower('a')); - ASSERT_TRUE(isASCIILower('z')); - ASSERT_FALSE(isASCIILower('A')); - ASSERT_FALSE(isASCIILower('Z')); - ASSERT_FALSE(isASCIILower('0')); - ASSERT_FALSE(isASCIILower('9')); - ASSERT_FALSE(isASCIILower(' ')); - ASSERT_FALSE(isASCIILower('\n')); - ASSERT_FALSE(isASCIILower('\t')); - ASSERT_FALSE(isASCIILower(':')); -} - -TEST(string, isASCIIUpper) { - ASSERT_FALSE(isASCIIUpper('a')); - ASSERT_FALSE(isASCIIUpper('z')); - ASSERT_TRUE(isASCIIUpper('A')); - ASSERT_TRUE(isASCIIUpper('Z')); - ASSERT_FALSE(isASCIIUpper('0')); - ASSERT_FALSE(isASCIIUpper('9')); - ASSERT_FALSE(isASCIIUpper(' ')); - ASSERT_FALSE(isASCIIUpper('\n')); - ASSERT_FALSE(isASCIIUpper('\t')); - ASSERT_FALSE(isASCIIUpper(':')); -} - -TEST(string, isASCIIAlpha) { - ASSERT_TRUE(isASCIIAlpha('a')); - ASSERT_TRUE(isASCIIAlpha('z')); - ASSERT_TRUE(isASCIIAlpha('A')); - ASSERT_TRUE(isASCIIAlpha('Z')); - ASSERT_FALSE(isASCIIAlpha('0')); - ASSERT_FALSE(isASCIIAlpha('9')); - ASSERT_FALSE(isASCIIAlpha(' ')); - ASSERT_FALSE(isASCIIAlpha('\n')); - ASSERT_FALSE(isASCIIAlpha('\t')); - ASSERT_FALSE(isASCIIAlpha(':')); -} - -TEST(string, isASCIIDigit) { - ASSERT_FALSE(isASCIIDigit('a')); - ASSERT_FALSE(isASCIIDigit('z')); - ASSERT_FALSE(isASCIIDigit('A')); - ASSERT_FALSE(isASCIIDigit('Z')); - ASSERT_TRUE(isASCIIDigit('0')); - ASSERT_TRUE(isASCIIDigit('1')); - ASSERT_TRUE(isASCIIDigit('9')); - ASSERT_FALSE(isASCIIDigit(' ')); - ASSERT_FALSE(isASCIIDigit('\n')); - ASSERT_FALSE(isASCIIDigit('\t')); - ASSERT_FALSE(isASCIIDigit(':')); -} - -} \ No newline at end of file From 4b38ebb009313076f43a53b4fd203e36633ea45e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Jan 2024 19:14:50 +0100 Subject: [PATCH 400/402] withFramedSink(): Receive interrupts on the stderr thread Otherwise Nix deadlocks when Ctrl-C is received in withFramedSink(): the parent thread will wait forever for the stderr thread to shut down. Fixes the hang reported in https://github.com/NixOS/nix/issues/7245#issuecomment-1770560923. (cherry picked from commit 24e70489e59f9ab75310382dc59df09796ea8df4) --- src/libstore/remote-store.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f16949f42..77dc97e39 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -16,6 +16,8 @@ #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" +#include "signals.hh" + #include namespace nix { @@ -1071,6 +1073,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function Date: Wed, 3 Jan 2024 19:30:02 +0100 Subject: [PATCH 401/402] Make some more threads receive interrupts Shouldn't hurt to do this. In particular, this should speed up shutting down the PathSubstitutionGoal thread if it's copying from a remote store. (cherry picked from commit 295a2ff8bdff0f681e0c5c54e9378460eb63b663) --- src/libstore/build/substitution-goal.cc | 3 +++ src/libutil/thread-pool.cc | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 93867007d..c7e8e2825 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -2,6 +2,7 @@ #include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" +#include "signals.hh" namespace nix { @@ -217,6 +218,8 @@ void PathSubstitutionGoal::tryToRun() thr = std::thread([this]() { try { + ReceiveInterrupts receiveInterrupts; + /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide.close(); }); diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index c5e735617..9a7dfee56 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -79,6 +79,8 @@ void ThreadPool::process() void ThreadPool::doWork(bool mainThread) { + ReceiveInterrupts receiveInterrupts; + if (!mainThread) interruptCheck = [&]() { return (bool) quit; }; From 958ecd81a8ef8ba767dccefc115e9b4124eb0044 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 20:57:50 +0100 Subject: [PATCH 402/402] fix an old lost direct (#9718) this part must have been moved quite a while ago, but apparently so far no one noticed (cherry picked from commit 6db805b3d1e4eccd0103d9856b8ab3d01efba51f) Co-authored-by: Valentin Gagarin --- doc/manual/redirects.js | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 3b507adf3..d04f32b49 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -21,6 +21,7 @@ const redirects = { "chap-distributed-builds": "advanced-topics/distributed-builds.html", "chap-post-build-hook": "advanced-topics/post-build-hook.html", "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", + "chap-writing-nix-expressions": "language/index.html", "part-command-ref": "command-ref/command-ref.html", "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation", "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",