From b9c2f834ee37c76fcb21bc5dbcd60bb58a229194 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Mon, 22 May 2023 23:39:31 +0200 Subject: [PATCH 01/44] Check exact error codes in linux-sandbox.sh --- tests/linux-sandbox-cert-test.nix | 45 ++++++++++++++++--------------- tests/linux-sandbox.sh | 18 ++++++++----- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/tests/linux-sandbox-cert-test.nix b/tests/linux-sandbox-cert-test.nix index 2b86dad2e..2fc083ea9 100644 --- a/tests/linux-sandbox-cert-test.nix +++ b/tests/linux-sandbox-cert-test.nix @@ -1,29 +1,30 @@ -{ fixed-output }: +{ mode }: with import ./config.nix; -mkDerivation ({ - name = "ssl-export"; - buildCommand = '' - # Add some indirection, otherwise grepping into the debug output finds the string. - report () { echo CERT_$1_IN_SANDBOX; } +mkDerivation ( + { + name = "ssl-export"; + buildCommand = '' + # Add some indirection, otherwise grepping into the debug output finds the string. + report () { echo CERT_$1_IN_SANDBOX; } - if [ -f /etc/ssl/certs/ca-certificates.crt ]; then - content=$( $TEST_ROOT/log) -if grepQuiet 'error: renaming' $TEST_ROOT/log; then false; fi -grepQuiet 'may not be deterministic' $TEST_ROOT/log +expectStderr 104 nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K \ + | tee >( grepQuietInverse 'error: renaming' ) \ + | grepQuiet 'may not be deterministic' # Test that sandboxed builds cannot write to /etc easily -(! nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store) +expect 100 nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store ## Test mounting of SSL certificates into the sandbox testCert () { - (! nix-build linux-sandbox-cert-test.nix --argstr fixed-output "$2" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$3" 2> $TEST_ROOT/log) - cat $TEST_ROOT/log - grepQuiet "CERT_${1}_IN_SANDBOX" $TEST_ROOT/log + expectation=$1 # "missing" | "present" + mode=$2 # "normal" | "fixed-output" + certFile=$3 # a string that can be the path to a cert file + [ "$mode" == fixed-output ] && ret=1 || ret=100 + expectStderr $ret nix-build linux-sandbox-cert-test.nix --argstr mode "$mode" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$certFile" | \ + # tee /dev/stderr | \ + grepQuiet "CERT_${expectation}_IN_SANDBOX" } nocert=$TEST_ROOT/no-cert-file.pem From f0233f3a3fd92d443d2502ed29aea076b6572ab7 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 23 May 2023 09:59:49 +0200 Subject: [PATCH 02/44] Further refactor linux-sandbox.sh and fix tee usage --- tests/linux-sandbox.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 5c4e0cae9..19a7527bf 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -11,6 +11,8 @@ requireSandboxSupport # otherwise things get complicated (e.g. if it's in /bin, do we need # /lib as well?). if [[ ! $SHELL =~ /nix/store ]]; then skipTest "Shell is not from Nix store"; fi +# An alias to automatically bind-mount the $SHELL on nix-build invocations +nix-sandbox-build () { nix-build --no-out-link --sandbox-paths /nix/store "$@"; } chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 @@ -18,7 +20,7 @@ rm -rf $TEST_ROOT/store0 export NIX_STORE_DIR=/my/store export NIX_REMOTE=$TEST_ROOT/store0 -outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store) +outPath=$(nix-sandbox-build dependencies.nix) [[ $outPath =~ /my/store/.*-dependencies ]] @@ -29,17 +31,18 @@ nix store ls -R -l $outPath | grep foobar nix store cat $outPath/foobar | grep FOOBAR # Test --check without hash rewriting. -nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store +nix-sandbox-build dependencies.nix --check # Test that sandboxed builds with --check and -K can move .check directory to store -nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link +nix-sandbox-build check.nix -A nondeterministic -expectStderr 104 nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K \ - | tee >( grepQuietInverse 'error: renaming' ) \ - | grepQuiet 'may not be deterministic' +expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log +grepQuietInverse 'error: renaming' $TEST_ROOT/log +grepQuiet 'may not be deterministic' $TEST_ROOT/log # Test that sandboxed builds cannot write to /etc easily -expect 100 nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store +expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' | + grepQuiet "/etc/test: Permission denied" ## Test mounting of SSL certificates into the sandbox @@ -48,9 +51,9 @@ testCert () { mode=$2 # "normal" | "fixed-output" certFile=$3 # a string that can be the path to a cert file [ "$mode" == fixed-output ] && ret=1 || ret=100 - expectStderr $ret nix-build linux-sandbox-cert-test.nix --argstr mode "$mode" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$certFile" | \ - # tee /dev/stderr | \ - grepQuiet "CERT_${expectation}_IN_SANDBOX" + expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | + # tee /dev/stderr | + grepQuiet "CERT_${expectation}_IN_SANDBOX" } nocert=$TEST_ROOT/no-cert-file.pem From e09b40e0d0b68ca7c3646ddffb50e1356daec997 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 5 Mar 2023 02:36:26 +0100 Subject: [PATCH 03/44] reword documentation on trusted users and substituters this is to make it slightly easier to scan over --- src/libstore/globals.hh | 14 ++++++-------- src/nix/daemon.cc | 30 +++++++++++++++--------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 820898350..b0c025c8e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -700,8 +700,8 @@ public: At least one of the following conditions must be met for Nix to use a substituter: - - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + - The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list + - The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) )", @@ -710,12 +710,10 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), - separated by whitespace. These are - not used by default, but can be enabled by users of the Nix daemon - by specifying `--option substituters urls` on the command - line. Unprivileged users are only allowed to pass a subset of the - URLs listed in `substituters` and `trusted-substituters`. + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). + + Unprivileged users are only allowed to pass as `substituters` only those URLs listed in `trusted-substituters`. )", {"trusted-binary-caches"}}; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 9fe9b3b1e..09adab5d3 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -55,19 +55,16 @@ struct AuthorizationSettings : Config { Setting trustedUsers{ this, {"root"}, "trusted-users", R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. + A list of user names, separated by whitespace. + These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NAR](@docroot@/glossary.md#gloss-nar)s. + + You can also specify groups by prefixing names with `@`. + For instance, `@wheel` means all users in the `wheel` group. > **Warning** > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. + > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system. + > For example, the user can set [`sandbox-paths`](#conf-sandbox-paths) and thereby obtain read access to directories that are otherwise inacessible to them. )"}; /** @@ -76,12 +73,15 @@ struct AuthorizationSettings : Config { Setting allowedUsers{ this, {"*"}, "allowed-users", R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. + A list user names, separated by whitespace. + These users are allowed to connect to the Nix daemon. - Note that trusted users are always allowed to connect. + As with the [`trusted-users`](#conf-trusted-users) option, you can specify groups by prefixing names with `@`. + Also, you can allow all users by specifying `*`. + + > **Note** + > + > Trusted users are always allowed to connect to the Nix daemon. )"}; }; From b7d47e1d22e7ce2785487d325cc3dd35a43f16b5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 04:58:07 +0200 Subject: [PATCH 04/44] fix wording --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b0c025c8e..f63ec8b50 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -713,7 +713,7 @@ public: A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). - Unprivileged users are only allowed to pass as `substituters` only those URLs listed in `trusted-substituters`. + Unprivileged users are allowed to pass as `substituters` only those URLs listed in `trusted-substituters`. )", {"trusted-binary-caches"}}; From e1fa48f17c0bb0f73e97e684077cec8dfa1d7a3d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Jun 2023 07:41:37 -0400 Subject: [PATCH 05/44] Update src/nix/daemon.cc Co-authored-by: Valentin Gagarin --- src/nix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 09adab5d3..a57070c56 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -81,7 +81,7 @@ struct AuthorizationSettings : Config { > **Note** > - > Trusted users are always allowed to connect to the Nix daemon. + > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always to connect to the Nix daemon. )"}; }; From 2ceacce484e21ac116a79c74877327355fd153d0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 15:57:54 +0200 Subject: [PATCH 06/44] Update src/libstore/globals.hh --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f63ec8b50..46147a5e1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -713,7 +713,7 @@ public: A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). - Unprivileged users are allowed to pass as `substituters` only those URLs listed in `trusted-substituters`. + Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. )", {"trusted-binary-caches"}}; From 4a33d5fe3549137bacf9373e5ba7bfe11a421099 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 14:33:31 +0200 Subject: [PATCH 07/44] fix link text Co-authored-by: Robert Hensing --- src/nix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index a57070c56..d0e70a7b1 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -56,7 +56,7 @@ struct AuthorizationSettings : Config { this, {"root"}, "trusted-users", R"( A list of user names, separated by whitespace. - These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NAR](@docroot@/glossary.md#gloss-nar)s. + These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NARs](@docroot@/glossary.md#gloss-nar). You can also specify groups by prefixing names with `@`. For instance, `@wheel` means all users in the `wheel` group. From 1a8ca85d488ddacf26f2aeddddab926c0e081d98 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 14:34:11 +0200 Subject: [PATCH 08/44] use "store URLs" consistently --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 46147a5e1..d2efd1505 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -710,7 +710,7 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. From 126eea48e300ab365c46ce062776e74a3907a7c8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 14:36:53 +0200 Subject: [PATCH 09/44] do not refer to `trusted-users` another time --- src/nix/daemon.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index d0e70a7b1..43d2f8f86 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -76,7 +76,8 @@ struct AuthorizationSettings : Config { A list user names, separated by whitespace. These users are allowed to connect to the Nix daemon. - As with the [`trusted-users`](#conf-trusted-users) option, you can specify groups by prefixing names with `@`. + You can specify groups by prefixing names with `@`. + For instance, `@wheel` means all users in the `wheel` group. Also, you can allow all users by specifying `*`. > **Note** From baef05e6fefb46b3bdddb2785861bd7190920506 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 14:37:08 +0200 Subject: [PATCH 10/44] fix typo Co-authored-by: Robert Hensing --- src/nix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 43d2f8f86..6dcd7f94d 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -82,7 +82,7 @@ struct AuthorizationSettings : Config { > **Note** > - > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always to connect to the Nix daemon. + > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always connect to the Nix daemon. )"}; }; From f695a74751c314cc426ff7bbc67ce5de8b58bbfd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 17:58:01 +0200 Subject: [PATCH 11/44] Update src/libstore/globals.hh --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d2efd1505..05aa8288a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -711,7 +711,7 @@ public: this, {}, "trusted-substituters", R"( A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. - These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). + These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. )", From c8825e9d8c3fa802811f3829d055e3ef9aae90e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 16 Jun 2023 15:19:14 -0400 Subject: [PATCH 12/44] Create nlohmann serializers for `std::optional` and use This is somewhat tricky. --- src/libutil/args.cc | 15 ++---- src/libutil/experimental-features.hh | 8 ++- src/libutil/json-utils.cc | 19 +++++++ src/libutil/json-utils.hh | 80 +++++++++++++++++++++++----- 4 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 src/libutil/json-utils.cc diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 081dbeb28..3cf3ed9ca 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,10 +1,9 @@ #include "args.hh" #include "hash.hh" +#include "json-utils.hh" #include -#include - namespace nix { void Args::addFlag(Flag && flag_) @@ -247,11 +246,7 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; - // TODO With C++23 use `std::optional::tranform` - if (auto & xp = flag->experimentalFeature) - j["experimental-feature"] = showExperimentalFeature(*xp); - else - j["experimental-feature"] = nullptr; + j["experimental-feature"] = flag->experimentalFeature; flags[name] = std::move(j); } @@ -416,11 +411,7 @@ nlohmann::json MultiCommand::toJSON() cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); - // TODO With C++23 use `std::optional::tranform` - if (auto xp = command->experimentalFeature()) - cat["experimental-feature"] = showExperimentalFeature(*xp); - else - cat["experimental-feature"] = nullptr; + cat["experimental-feature"] = command->experimentalFeature(); cmds[name] = std::move(j); } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 892c6c371..15ff5e0cd 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -3,7 +3,7 @@ #include "comparator.hh" #include "error.hh" -#include "nlohmann/json_fwd.hpp" +#include "json-utils.hh" #include "types.hh" namespace nix { @@ -93,4 +93,10 @@ public: void to_json(nlohmann::json &, const ExperimentalFeature &); void from_json(const nlohmann::json &, ExperimentalFeature &); +/** + * It is always rendered as a string + */ +template<> +struct json_avoids_null : std::true_type {}; + } diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc new file mode 100644 index 000000000..d7220e71d --- /dev/null +++ b/src/libutil/json-utils.cc @@ -0,0 +1,19 @@ +#include "json-utils.hh" + +namespace nix { + +const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +nlohmann::json * get(nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +} diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index eb00e954f..5e63c1af4 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -2,21 +2,77 @@ ///@file #include +#include namespace nix { -const nlohmann::json * get(const nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} +const nlohmann::json * get(const nlohmann::json & map, const std::string & key); -nlohmann::json * get(nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} +nlohmann::json * get(nlohmann::json & map, const std::string & key); + +/** + * For `adl_serializer>` below, we need to track what + * types are not already using `null`. Only for them can we use `null` + * to represent `std::nullopt`. + */ +template +struct json_avoids_null; + +/** + * Handle numbers in default impl + */ +template +struct json_avoids_null : std::bool_constant::value> {}; + +template<> +struct json_avoids_null : std::false_type {}; + +template<> +struct json_avoids_null : std::true_type {}; + +template<> +struct json_avoids_null : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +} + +namespace nlohmann { + +/** + * This "instance" is widely requested, see + * https://github.com/nlohmann/json/issues/1749, but momentum has stalled + * out. Writing there here in Nix as a stop-gap. + * + * We need to make sure the underlying type does not use `null` for this to + * round trip. We do that with a static assert. + */ +template +struct adl_serializer> { + static std::optional from_json(const json & json) { + static_assert( + nix::json_avoids_null::value, + "null is already in use for underlying type's JSON"); + return json.is_null() + ? std::nullopt + : std::optional { adl_serializer::from_json(json) }; + } + static void to_json(json & json, std::optional t) { + static_assert( + nix::json_avoids_null::value, + "null is already in use for underlying type's JSON"); + if (t) + adl_serializer::to_json(json, *t); + else + json = nullptr; + } +}; } From d2ce2e89b178e7e7467cf4c1e572704a4c2ca75e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 May 2023 10:56:59 -0400 Subject: [PATCH 13/44] Split `OptionalPathSetting` from `PathSetting` Rather than doing `allowEmpty` as boolean, have separate types and use `std::optional`. This makes it harder to forget the possibility of an empty path. The `build-hook` setting was categorized as a `PathSetting`, but actually it was split into arguments. No good! Now, it is `Setting` which actually reflects what it means and how it is used. Because of the subtyping, we now also have support for `Setting>` in general. I imagine this can be used to clean up many more settings also. --- src/libstore/build/hook-instance.cc | 6 +- src/libstore/build/local-derivation-goal.cc | 8 ++- src/libstore/globals.cc | 5 +- src/libstore/globals.hh | 6 +- src/libstore/local-fs-store.hh | 14 ++--- src/libstore/store-api.hh | 2 +- src/libutil/abstract-setting-to-json.hh | 1 + src/libutil/config-impl.hh | 61 +++++++++++++++++++- src/libutil/config.cc | 63 ++++++++------------- src/libutil/config.hh | 31 ++++++++-- 10 files changed, 134 insertions(+), 63 deletions(-) diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 075ad554f..337c60bd4 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -5,14 +5,14 @@ namespace nix { HookInstance::HookInstance() { - debug("starting build hook '%s'", settings.buildHook); + debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get())); - auto buildHookArgs = tokenizeString>(settings.buildHook.get()); + auto buildHookArgs = settings.buildHook.get(); if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); - auto buildHook = buildHookArgs.front(); + auto buildHook = canonPath(buildHookArgs.front()); buildHookArgs.pop_front(); Strings args; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7f87cdf55..cf44779b7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -65,8 +65,9 @@ void handleDiffHook( const Path & tryA, const Path & tryB, const Path & drvPath, const Path & tmpDir) { - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { + auto & diffHookOpt = settings.diffHook.get(); + if (diffHookOpt && settings.runDiffHook) { + auto & diffHook = *diffHookOpt; try { auto diffRes = runProgram(RunOptions { .program = diffHook, @@ -1428,7 +1429,8 @@ void LocalDerivationGoal::startDaemon() Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - params["root"] = getLocalStore().rootDir; + if (auto & optRoot = getLocalStore().rootDir.get()) + params["root"] = *optRoot; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref(params, diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d53377239..5a4cb1824 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -100,7 +100,10 @@ Settings::Settings() if (!pathExists(nixExePath)) { nixExePath = getSelfExe().value_or("nix"); } - buildHook = nixExePath + " __build-remote"; + buildHook = { + nixExePath, + "__build-remote", + }; } void loadConfFile() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 820898350..b8dcf1f76 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -236,7 +236,7 @@ public: )", {"build-timeout"}}; - PathSetting buildHook{this, true, "", "build-hook", + Setting buildHook{this, {}, "build-hook", R"( The path to the helper program that executes remote builds. @@ -556,8 +556,8 @@ public: line. )"}; - PathSetting diffHook{ - this, true, "", "diff-hook", + OptionalPathSetting diffHook{ + this, std::nullopt, "diff-hook", R"( Absolute path to an executable capable of diffing build results. The hook is executed if `run-diff-hook` is true, and the diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index a03bb88f5..2ee2ef0c8 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -15,22 +15,22 @@ struct LocalFSStoreConfig : virtual StoreConfig // it to omit the call to the Setting constructor. Clang works fine // either way. - const PathSetting rootDir{(StoreConfig*) this, true, "", + const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + const PathSetting stateDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + const PathSetting logDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + const PathSetting realStoreDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2ecbe2708..14a862eef 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -114,7 +114,7 @@ struct StoreConfig : public Config return ""; } - const PathSetting storeDir_{this, false, settings.nixStore, + const PathSetting storeDir_{this, settings.nixStore, "store", R"( Logical location of the Nix store, usually diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index 7b6c3fcb5..d506dfb74 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -3,6 +3,7 @@ #include #include "config.hh" +#include "json-utils.hh" namespace nix { template diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b6cae5ec3..14822050e 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -50,8 +50,11 @@ template<> void BaseSetting>::appendOrSet(std::set template void BaseSetting::appendOrSet(T && newValue, bool append) { - static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type"); + static_assert( + !trait::appendable, + "using default `appendOrSet` implementation with an appendable type"); assert(!append); + value = std::move(newValue); } @@ -68,4 +71,60 @@ void BaseSetting::set(const std::string & str, bool append) } } +template<> void BaseSetting::convertToArg(Args & args, const std::string & category); + +template +void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.addFlag({ + .longName = name, + .description = fmt("Set the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, + .experimentalFeature = experimentalFeature, + }); + + if (isAppendable()) + args.addFlag({ + .longName = "extra-" + name, + .description = fmt("Append to the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, + .experimentalFeature = experimentalFeature, + }); +} + +#define DECLARE_CONFIG_SERIALISER(TY) \ + template<> TY BaseSetting< TY >::parse(const std::string & str) const; \ + template<> std::string BaseSetting< TY >::to_string() const; + +DECLARE_CONFIG_SERIALISER(std::string) +DECLARE_CONFIG_SERIALISER(std::optional) +DECLARE_CONFIG_SERIALISER(bool) +DECLARE_CONFIG_SERIALISER(Strings) +DECLARE_CONFIG_SERIALISER(StringSet) +DECLARE_CONFIG_SERIALISER(StringMap) +DECLARE_CONFIG_SERIALISER(std::set) + +template +T BaseSetting::parse(const std::string & str) const +{ + static_assert(std::is_integral::value, "Integer required."); + + if (auto n = string2Int(str)) + return *n; + else + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template +std::string BaseSetting::to_string() const +{ + static_assert(std::is_integral::value, "Integer required."); + + return std::to_string(value); +} + } diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 085a884dc..38d406e8a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -219,29 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } -template -void BaseSetting::convertToArg(Args & args, const std::string & category) -{ - args.addFlag({ - .longName = name, - .description = fmt("Set the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[this](std::string s) { overridden = true; set(s); }}, - .experimentalFeature = experimentalFeature, - }); - - if (isAppendable()) - args.addFlag({ - .longName = "extra-" + name, - .description = fmt("Append to the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[this](std::string s) { overridden = true; set(s, true); }}, - .experimentalFeature = experimentalFeature, - }); -} - template<> std::string BaseSetting::parse(const std::string & str) const { return str; @@ -252,21 +229,17 @@ template<> std::string BaseSetting::to_string() const return value; } -template -T BaseSetting::parse(const std::string & str) const +template<> std::optional BaseSetting>::parse(const std::string & str) const { - static_assert(std::is_integral::value, "Integer required."); - if (auto n = string2Int(str)) - return *n; + if (str == "") + return std::nullopt; else - throw UsageError("setting '%s' has invalid value '%s'", name, str); + return { str }; } -template -std::string BaseSetting::to_string() const +template<> std::string BaseSetting>::to_string() const { - static_assert(std::is_integral::value, "Integer required."); - return std::to_string(value); + return value ? *value : ""; } template<> bool BaseSetting::parse(const std::string & str) const @@ -403,15 +376,25 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting>; +static Path parsePath(const AbstractSetting & s, const std::string & str) +{ + if (str == "") + throw UsageError("setting '%s' is a path and paths cannot be empty", s.name); + else + return canonPath(str); +} + Path PathSetting::parse(const std::string & str) const { - if (str == "") { - if (allowEmpty) - return ""; - else - throw UsageError("setting '%s' cannot be empty", name); - } else - return canonPath(str); + return parsePath(*this, str); +} + +std::optional OptionalPathSetting::parse(const std::string & str) const +{ + if (str == "") + return std::nullopt; + else + return parsePath(*this, str); } bool GlobalConfig::set(const std::string & name, const std::string & value) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 2675baed7..cc8532587 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -353,21 +353,20 @@ public: /** * A special setting for Paths. These are automatically canonicalised * (e.g. "/foo//bar/" becomes "/foo/bar"). + * + * It is mandatory to specify a path; i.e. the empty string is not + * permitted. */ class PathSetting : public BaseSetting { - bool allowEmpty; - public: PathSetting(Config * options, - bool allowEmpty, const Path & def, const std::string & name, const std::string & description, const std::set & aliases = {}) : BaseSetting(def, true, name, description, aliases) - , allowEmpty(allowEmpty) { options->addSetting(this); } @@ -379,6 +378,30 @@ public: void operator =(const Path & v) { this->assign(v); } }; +/** + * Like `PathSetting`, but the absence of a path is also allowed. + * + * `std::optional` is used instead of the empty string for clarity. + */ +class OptionalPathSetting : public BaseSetting> +{ +public: + + OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting>(def, true, name, description, aliases) + { + options->addSetting(this); + } + + std::optional parse(const std::string & str) const override; + + void operator =(const std::optional & v) { this->assign(v); } +}; + struct GlobalConfig : public AbstractConfig { typedef std::vector ConfigRegistrations; From e91d19db5f827cec56e32fe9b7c07c8d2c546ce6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 12:23:53 +0200 Subject: [PATCH 14/44] be more serious about security risks with trusted users --- src/nix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 6dcd7f94d..41f826228 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -64,7 +64,7 @@ struct AuthorizationSettings : Config { > **Warning** > > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system. - > For example, the user can set [`sandbox-paths`](#conf-sandbox-paths) and thereby obtain read access to directories that are otherwise inacessible to them. + > For example, the user can access or replace store path contents that are critical for system security. )"}; /** From 71317162c50df91d421eacdf87d50b4ba6d37635 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 13:44:04 +0200 Subject: [PATCH 15/44] use more self-descriptive section headings --- doc/manual/src/contributing/hacking.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index c57d45138..12b91c79e 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -12,14 +12,15 @@ The following instructions assume you already have some version of Nix installed [installation instructions]: ../installation/installation.md -## Nix with flakes +## Building Nix with flakes -This section assumes you are using Nix with [flakes] enabled. See the [next section](#classic-nix) for equivalent instructions which don't require flakes. +This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. +See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[flakes]: ../command-ref/new-cli/nix3-flake.md#description +[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console $ nix develop @@ -55,7 +56,7 @@ To install it in `$(pwd)/outputs` and test it: nix (Nix) 2.12 ``` -To build a release version of Nix: +To build a release version of Nix for the current operating system and CPU architecture: ```console $ nix build @@ -63,12 +64,9 @@ $ nix build You can also build Nix for one of the [supported target platforms](#target-platforms). -## Classic Nix +## Building Nix -This section is for Nix without [flakes]. - -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console $ nix-shell @@ -102,7 +100,7 @@ To install it in `$(pwd)/outputs` and test it: nix (Nix) 2.12 ``` -To build Nix for the current operating system and CPU architecture use +To build a release version of Nix for the current operating system and CPU architecture: ```console $ nix-build From 3a20c7c46c64d2feb8caba767724a78d7efa3279 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 20 Jun 2023 22:51:29 +0200 Subject: [PATCH 16/44] Update tests/linux-sandbox.sh Co-authored-by: John Ericson --- tests/linux-sandbox.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 19a7527bf..f2a77174d 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -52,7 +52,6 @@ testCert () { certFile=$3 # a string that can be the path to a cert file [ "$mode" == fixed-output ] && ret=1 || ret=100 expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | - # tee /dev/stderr | grepQuiet "CERT_${expectation}_IN_SANDBOX" } From a78f929065fd209e3d89853c6beb48942e3820e9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 12:39:07 +0200 Subject: [PATCH 17/44] fix anchor link --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 12b91c79e..310d69310 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -62,7 +62,7 @@ To build a release version of Nix for the current operating system and CPU archi $ nix build ``` -You can also build Nix for one of the [supported target platforms](#target-platforms). +You can also build Nix for one of the [supported target platforms](#platforms). ## Building Nix @@ -106,7 +106,7 @@ To build a release version of Nix for the current operating system and CPU archi $ nix-build ``` -You can also build Nix for one of the [supported target platforms](#target-platforms). +You can also build Nix for one of the [supported target platforms](#platforms). ## Platforms From 085104944b2957f4269fcbfaf0d1874ab0c9ce84 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 21 Jun 2023 09:46:23 +0200 Subject: [PATCH 18/44] add redirects to changed anchors --- doc/manual/redirects.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 5cd6fdea2..6813dc6d6 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -342,6 +342,10 @@ const redirects = { "installation/installing-binary.html": { "uninstalling": "uninstall.html" } + "contributing/hacking.html": { + "nix-with-flakes": "#building-nix-with-flakes" + "classic-nix": "#building-nix" + } }; // the following code matches the current page's URL against the set of redirects. From 5f9a921bc1e6d3e6bf65601be565585cf0190e8f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 21 Jun 2023 14:31:09 +0200 Subject: [PATCH 19/44] do not use "target", as it's a loaded term in the domain of compilers Co-authored-by: Robert Hensing --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 310d69310..6c6f2eb52 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -62,7 +62,7 @@ To build a release version of Nix for the current operating system and CPU archi $ nix build ``` -You can also build Nix for one of the [supported target platforms](#platforms). +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix @@ -106,7 +106,7 @@ To build a release version of Nix for the current operating system and CPU archi $ nix-build ``` -You can also build Nix for one of the [supported target platforms](#platforms). +You can also build Nix for one of the [supported platforms](#platforms). ## Platforms From 5cc22e337023cee9cbffd3f9d2ff0eb3f18a4d12 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Jun 2023 22:29:03 -0400 Subject: [PATCH 20/44] Clarify docs on deleting generations, including fixing a mistake Deleting store info corrected (there is a foot-gun in Nix with `--delete-generations old`!) Also a few things are cleaned up based on feedback. Co-authored-by: Valentin Gagarin Co-authored-by: Eelco Dolstra --- .../src/command-ref/nix-collect-garbage.md | 2 +- .../command-ref/nix-env/delete-generations.md | 28 ++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index a679ceaf7..3cab79f0e 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -57,7 +57,7 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). *period* is a value such as `30d`, which would mean 30 days. - This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-days) on each found profile. + This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index d828a5b9e..adc6fc219 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -20,22 +20,30 @@ This operation deletes the specified generations of the current profile. - The special value `old` - Delete all generations older than the current one. + Delete all generations except the current one. -- `d`:\ - The last *days* days + > **WARNING** + > + > Older *and newer* generations will be deleted by this operation. + > + > One might expect this to just delete older generations than the curent one, but that is only true if the current generation is also the latest. + > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. + > They will also be deleted. + +- `d`:\ + The last *number* days *Example*: `30d` - Delete all generations older than *days* days. - The generation that was active at that point in time is excluded, and will not be deleted. + Delete all generations created more than *number* days ago, except the most recent one of them. + This allows rolling back to generations that were available within the specified period. -- `+`:\ - The last *count* generations up to the present +- `+`:\ + The last *number* generations up to the present *Example*: `+5` - Keep the last *count* generations, along with any newer than current. + Keep the last *number* generations, along with any newer than current. Periodically deleting old generations is important to make garbage collection effective. @@ -61,7 +69,7 @@ $ nix-env --delete-generations 3 4 8 Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those. -## Keep most-recent by count count +## Keep most-recent by count (number of generations) ```console $ nix-env --delete-generations +5 @@ -72,7 +80,7 @@ Suppose `30` is the current generation, and we currently have generations number Then this command will delete generations `20` through `25` (`<= 30 - 5`), and keep generations `26` through `31` (`> 30 - 5`). -## Keep most-recent in days +## Keep most-recent by time (number of days) ```console $ nix-env --delete-generations 30d From c48277c1c15f0b6be6a47271a6f8399fad96f7fd Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Thu, 22 Jun 2023 18:19:42 +0800 Subject: [PATCH 21/44] libexpr: add tests for `nix::Value::print` --- src/libexpr/tests/value/print.cc | 170 +++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 src/libexpr/tests/value/print.cc diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc new file mode 100644 index 000000000..2268658b2 --- /dev/null +++ b/src/libexpr/tests/value/print.cc @@ -0,0 +1,170 @@ +#include "tests/libexpr.hh" + +#include "value.hh" + +namespace nix { + +using namespace testing; + +struct ValuePrintingTests : LibExprTest +{ + template + void test(Value v, std::string_view expected, A... args) + { + std::stringstream out; + v.print(state.symbols, out, args...); + ASSERT_EQ(out.str(), expected); + } +}; + +TEST_F(ValuePrintingTests, tInt) +{ + Value vInt; + vInt.mkInt(10); + test(vInt, "10"); +} + +TEST_F(ValuePrintingTests, tBool) +{ + Value vBool; + vBool.mkBool(true); + test(vBool, "true"); +} + +TEST_F(ValuePrintingTests, tString) +{ + Value vString; + vString.mkString("some-string"); + test(vString, "\"some-string\""); +} + +TEST_F(ValuePrintingTests, tPath) +{ + Value vPath; + vPath.mkString("/foo"); + test(vPath, "\"/foo\""); +} + +TEST_F(ValuePrintingTests, tNull) +{ + Value vNull; + vNull.mkNull(); + test(vNull, "null"); +} + +TEST_F(ValuePrintingTests, tAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, "{ one = 1; two = 2; }"); +} + +TEST_F(ValuePrintingTests, tList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + Value vList; + state.mkList(vList, 5); + vList.bigList.elems[0] = &vOne; + vList.bigList.elems[1] = &vTwo; + vList.bigList.size = 3; + + test(vList, "[ 1 2 (nullptr) ]"); +} + +TEST_F(ValuePrintingTests, vThunk) +{ + Value vThunk; + vThunk.mkThunk(nullptr, nullptr); + + test(vThunk, ""); +} + +TEST_F(ValuePrintingTests, vApp) +{ + Value vApp; + vApp.mkApp(nullptr, nullptr); + + test(vApp, ""); +} + +TEST_F(ValuePrintingTests, vLambda) +{ + Value vLambda; + vLambda.mkLambda(nullptr, nullptr); + + test(vLambda, ""); +} + +TEST_F(ValuePrintingTests, vPrimOp) +{ + Value vPrimOp; + vPrimOp.mkPrimOp(nullptr); + + test(vPrimOp, ""); +} + +TEST_F(ValuePrintingTests, vPrimOpApp) +{ + Value vPrimOpApp; + vPrimOpApp.mkPrimOpApp(nullptr, nullptr); + + test(vPrimOpApp, ""); +} + +TEST_F(ValuePrintingTests, vExternal) +{ + struct MyExternal : ExternalValueBase + { + public: + std::string showType() const override + { + return ""; + } + std::string typeOf() const override + { + return ""; + } + virtual std::ostream & print(std::ostream & str) const override + { + str << "testing-external!"; + return str; + } + } myExternal; + Value vExternal; + vExternal.mkExternal(&myExternal); + + test(vExternal, "testing-external!"); +} + +TEST_F(ValuePrintingTests, vFloat) +{ + Value vFloat; + vFloat.mkFloat(2.0); + + test(vFloat, "2"); +} + +TEST_F(ValuePrintingTests, vBlackhole) +{ + Value vBlackhole; + vBlackhole.mkBlackhole(); + test(vBlackhole, "«potential infinite recursion»"); +} + +} // namespace nix From 1400fde144e1fdde2647fda5a7f9511f25572a35 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Thu, 22 Jun 2023 12:27:19 +0800 Subject: [PATCH 22/44] libexpr: extend `Value::print` to allow limited depth --- src/libexpr/eval.cc | 20 ++++++---- src/libexpr/tests/value/print.cc | 66 ++++++++++++++++++++++++++++++++ src/libexpr/value.hh | 5 ++- 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 71fd6e6e4..da55d3f9e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -95,11 +95,16 @@ RootValue allocRootValue(Value * v) #endif } -void Value::print(const SymbolTable & symbols, std::ostream & str, - std::set * seen) const +void Value::print(const SymbolTable &symbols, std::ostream &str, + std::set *seen, int depth) const + { checkInterrupt(); + if (depth <= 0) { + str << "«too deep»"; + return; + } switch (internalType) { case tInt: str << integer; @@ -123,7 +128,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << "{ "; for (auto & i : attrs->lexicographicOrder(symbols)) { str << symbols[i->name] << " = "; - i->value->print(symbols, str, seen); + i->value->print(symbols, str, seen, depth - 1); str << "; "; } str << "}"; @@ -139,7 +144,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << "[ "; for (auto v2 : listItems()) { if (v2) - v2->print(symbols, str, seen); + v2->print(symbols, str, seen, depth - 1); else str << "(nullptr)"; str << " "; @@ -181,11 +186,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, } } - -void Value::print(const SymbolTable & symbols, std::ostream & str, bool showRepeated) const -{ +void Value::print(const SymbolTable &symbols, std::ostream &str, + bool showRepeated, int depth) const { std::set seen; - print(symbols, str, showRepeated ? nullptr : &seen); + print(symbols, str, showRepeated ? nullptr : &seen, depth); } // Pretty print types for assertion errors diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc index 2268658b2..5e96e12ec 100644 --- a/src/libexpr/tests/value/print.cc +++ b/src/libexpr/tests/value/print.cc @@ -167,4 +167,70 @@ TEST_F(ValuePrintingTests, vBlackhole) test(vBlackhole, "«potential infinite recursion»"); } +TEST_F(ValuePrintingTests, depthAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + BindingsBuilder builder2(state, state.allocBindings(10)); + builder2.insert(state.symbols.create("one"), &vOne); + builder2.insert(state.symbols.create("two"), &vTwo); + builder2.insert(state.symbols.create("nested"), &vAttrs); + + Value vNested; + vNested.mkAttrs(builder2.finish()); + + test(vNested, "{ nested = «too deep»; one = «too deep»; two = «too deep»; }", false, 1); + test(vNested, "{ nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; }", false, 2); + test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 3); + test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 4); +} + +TEST_F(ValuePrintingTests, depthList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + BindingsBuilder builder2(state, state.allocBindings(10)); + builder2.insert(state.symbols.create("one"), &vOne); + builder2.insert(state.symbols.create("two"), &vTwo); + builder2.insert(state.symbols.create("nested"), &vAttrs); + + Value vNested; + vNested.mkAttrs(builder2.finish()); + + Value vList; + state.mkList(vList, 5); + vList.bigList.elems[0] = &vOne; + vList.bigList.elems[1] = &vTwo; + vList.bigList.elems[2] = &vNested; + vList.bigList.size = 3; + + test(vList, "[ «too deep» «too deep» «too deep» ]", false, 1); + test(vList, "[ 1 2 { nested = «too deep»; one = «too deep»; two = «too deep»; } ]", false, 2); + test(vList, "[ 1 2 { nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; } ]", false, 3); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 4); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 5); +} + } // namespace nix diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 89c0c36fd..6b32e2b04 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -2,6 +2,7 @@ ///@file #include +#include #include "symbol-table.hh" #include "value/context.hh" @@ -137,11 +138,11 @@ private: friend std::string showType(const Value & v); - void print(const SymbolTable & symbols, std::ostream & str, std::set * seen) const; + void print(const SymbolTable &symbols, std::ostream &str, std::set *seen, int depth) const; public: - void print(const SymbolTable & symbols, std::ostream & str, bool showRepeated = false) const; + void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const; // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's From 97df060588e4d328a7e219107553087149a77bac Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 22 Jun 2023 14:23:25 -0400 Subject: [PATCH 23/44] Better document build failure exit codes - Improved API docs from comment - Exit codes are for `nix-build`, not just `nix-store --release` - Make note in tests so the magic numbers are not surprising Picking up where #8387 left off. --- doc/manual/src/command-ref/nix-build.md | 2 ++ .../src/command-ref/nix-store/realise.md | 31 +---------------- .../src/command-ref/status-build-failure.md | 34 +++++++++++++++++++ src/libstore/build/entry-points.cc | 10 +++--- src/libstore/build/worker.cc | 11 ++---- src/libstore/build/worker.hh | 23 ++++++++++++- tests/check.sh | 3 ++ tests/linux-sandbox.sh | 3 ++ 8 files changed, 72 insertions(+), 45 deletions(-) create mode 100644 doc/manual/src/command-ref/status-build-failure.md diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 7369ea2eb..b548edf82 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -70,6 +70,8 @@ except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](n Change the name of the symlink to the output path created from `result` to *outlink*. +{{#include ./status-build-failure.md}} + {{#include ./opt-common.md}} {{#include ./env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 6b50d2145..c19aea75e 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -54,36 +54,7 @@ The following flags are available: previous build, the new output path is left in `/nix/store/name.check.` -Special exit codes: - - - `100`\ - Generic build failure, the builder process returned with a non-zero - exit code. - - - `101`\ - Build timeout, the build was aborted because it did not complete - within the specified `timeout`. - - - `102`\ - Hash mismatch, the build output was rejected because it does not - match the [`outputHash` attribute of the - derivation](@docroot@/language/advanced-attributes.md). - - - `104`\ - Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducible. - -With the `--keep-going` flag it's possible for multiple failures to -occur, in this case the 1xx status codes are or combined using binary -or. - - 1100100 - ^^^^ - |||`- timeout - ||`-- output hash mismatch - |`--- build failure - `---- not deterministic - +{{#include ../status-build-failure.md}} {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/status-build-failure.md b/doc/manual/src/command-ref/status-build-failure.md new file mode 100644 index 000000000..06114eb29 --- /dev/null +++ b/doc/manual/src/command-ref/status-build-failure.md @@ -0,0 +1,34 @@ +# Special exit codes for build failure + +1xx status codes are used when requested builds failed. +The following codes are in use: + +- `100` Generic build failure + + The builder process returned with a non-zero exit code. + +- `101` Build timeout + + The build was aborted because it did not complete within the specified `timeout`. + +- `102` Hash mismatch + + The build output was rejected because it does not match the + [`outputHash` attribute of the derivation](@docroot@/language/advanced-attributes.md). + +- `104` Not deterministic + + The build succeeded in check mode but the resulting output is not binary reproducible. + +With the `--keep-going` flag it's possible for multiple failures to occur. +In this case the 1xx status codes are or combined using +[bitwise OR](https://en.wikipedia.org/wiki/Bitwise_operation#OR). + +``` +0b1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic +``` diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index edd6cb6d2..4aa4d6dca 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -31,11 +31,11 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); + ex->status = worker.failingExitStatus(); throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); } } @@ -102,10 +102,10 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { - goal->ex->status = worker.exitStatus(); + goal->ex->status = worker.failingExitStatus(); throw std::move(*goal->ex); } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } } @@ -128,7 +128,7 @@ void Store::repairPath(const StorePath & path) goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); worker.run(goals); } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); + throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index ee334d54a..a9ca9cbbc 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -468,16 +468,9 @@ void Worker::waitForInput() } -unsigned int Worker::exitStatus() +unsigned int Worker::failingExitStatus() { - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ + // See API docs in header for explanation unsigned int mask = 0; bool buildFailure = permanentFailure || timedOut || hashMismatch; if (buildFailure) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 63624d910..5abceca0d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -280,7 +280,28 @@ public: */ void waitForInput(); - unsigned int exitStatus(); + /*** + * The exit status in case of failure. + * + * In the case of a build failure, returned value follows this + * bitmask: + * + * ``` + * 0b1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + * ``` + * + * In other words, the failure code is at least 100 (0b1100100), but + * might also be greater. + * + * Otherwise (no build failure, but some other sort of failure by + * assumption), this returned value is 1. + */ + unsigned int failingExitStatus(); /** * Check whether the given valid path exists and has the right diff --git a/tests/check.sh b/tests/check.sh index 645b90222..e13abf747 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -18,6 +18,9 @@ clearStore nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check +# Build failure exit codes (100, 104, etc.) are from +# doc/manual/src/command-ref/status-build-failure.md + # check for dangling temporary build directories # only retain if build fails and --keep-failed is specified, or... # ...build is non-deterministic and --check and --keep-failed are both specified diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index f2a77174d..ff7d257bd 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -36,11 +36,13 @@ nix-sandbox-build dependencies.nix --check # Test that sandboxed builds with --check and -K can move .check directory to store nix-sandbox-build check.nix -A nondeterministic +# `100 + 4` means non-determinstic, see doc/manual/src/command-ref/status-build-failure.md expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log grepQuietInverse 'error: renaming' $TEST_ROOT/log grepQuiet 'may not be deterministic' $TEST_ROOT/log # Test that sandboxed builds cannot write to /etc easily +# `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' | grepQuiet "/etc/test: Permission denied" @@ -50,6 +52,7 @@ testCert () { expectation=$1 # "missing" | "present" mode=$2 # "normal" | "fixed-output" certFile=$3 # a string that can be the path to a cert file + # `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md [ "$mode" == fixed-output ] && ret=1 || ret=100 expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | grepQuiet "CERT_${expectation}_IN_SANDBOX" From 9d8c4ac446bed82eb22a989cd81982acd722c58a Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Fri, 23 Jun 2023 13:35:41 +0800 Subject: [PATCH 24/44] libexpr: remove unused token `ATTRPATH` in token declaration --- src/libexpr/parser.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 3b545fd84..5c9597f8a 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -330,7 +330,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %type ind_string_parts %type path_start string_parts string_attr %type attr -%token ID ATTRPATH +%token ID %token STR IND_STR %token INT %token FLOAT From 484290a9e05ea840f9bc6ba3cc98d64ccffe202b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Jun 2023 12:01:10 -0400 Subject: [PATCH 25/44] Use a struct not `std::pair` for `SearchPathElem` I got very confused trying to keep all the `first` and `second` straight reading the code, *especially* as there is also another `(boolean, string)` pair type also being used. Named fields is much better. There are other cleanups that we can do (for example, the existing TODO), but we can do them later. Doing them now would just make this harder to review. --- src/libexpr/eval.hh | 8 ++++++-- src/libexpr/parser.y | 33 ++++++++++++++++++--------------- src/libexpr/primops.cc | 9 ++++++--- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 8e41bdbd0..7b726a78f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -65,8 +65,12 @@ std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -// FIXME: maybe change this to an std::variant. -typedef std::pair SearchPathElem; +struct SearchPathElem +{ + std::string prefix; + // FIXME: maybe change this to an std::variant. + std::string path; +}; typedef std::list SearchPath; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 5c9597f8a..03e6fb02c 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -741,7 +741,10 @@ void EvalState::addToSearchPath(const std::string & s) path = std::string(s, pos + 1); } - searchPath.emplace_back(prefix, path); + searchPath.emplace_back(SearchPathElem { + .prefix = prefix, + .path = path, + }); } @@ -755,11 +758,11 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p { for (auto & i : searchPath) { std::string suffix; - if (i.first.empty()) + if (i.prefix.empty()) suffix = concatStrings("/", path); else { - auto s = i.first.size(); - if (path.compare(0, s, i.first) != 0 || + auto s = i.prefix.size(); + if (path.compare(0, s, i.prefix) != 0 || (path.size() > s && path[s] != '/')) continue; suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); @@ -785,47 +788,47 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) { - auto i = searchPathResolved.find(elem.second); + auto i = searchPathResolved.find(elem.path); if (i != searchPathResolved.end()) return i->second; std::pair res; - if (EvalSettings::isPseudoUrl(elem.second)) { + if (EvalSettings::isPseudoUrl(elem.path)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).tree.storePath; + store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath; res = { true, store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path) }); res = { false, "" }; } } - else if (hasPrefix(elem.second, "flake:")) { + else if (hasPrefix(elem.path, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.second); + auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", elem.path); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; res = { true, store->toRealPath(storePath) }; } else { - auto path = absPath(elem.second); + auto path = absPath(elem.path); if (pathExists(path)) res = { true, path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path) }); res = { false, "" }; } } - debug("resolved search path element '%s' to '%s'", elem.second, res.second); + debug("resolved search path element '%s' to '%s'", elem.path, res.second); - searchPathResolved[elem.second] = res; + searchPathResolved[elem.path] = res; return res; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5b2f7e8b7..9dd7bae02 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1656,7 +1656,10 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V })); } - searchPath.emplace_back(prefix, path); + searchPath.emplace_back(SearchPathElem { + .prefix = prefix, + .path = path, + }); } auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); @@ -4129,8 +4132,8 @@ void EvalState::createBaseEnv() int n = 0; for (auto & i : searchPath) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.second); - attrs.alloc("prefix").mkString(i.first); + attrs.alloc("path").mkString(i.path); + attrs.alloc("prefix").mkString(i.prefix); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v); From a7b49086c76bc181ab1cf1d3c7fba44b06d2f1dd Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Thu, 14 Oct 2021 23:44:45 +1100 Subject: [PATCH 26/44] Add `dirtyRev` and `dirtyShortRev` to `fetchGit` Fixes #4682 --- src/libexpr/primops/fetchTree.cc | 7 ++++++- src/libfetchers/git.cc | 9 ++++++++- src/nix/flake.cc | 6 ++++++ tests/fetchGit.sh | 6 ++++++ tests/flakes/flakes.sh | 5 +++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index cd7039025..bfae34a48 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -22,7 +22,7 @@ void emitTreeAttrs( { assert(input.isLocked()); - auto attrs = state.buildBindings(8); + auto attrs = state.buildBindings(10); state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); @@ -56,6 +56,11 @@ void emitTreeAttrs( } + if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) { + attrs.alloc("dirtyRev").mkString(*dirtyRev); + attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev")); + } + if (auto lastModified = input.getLastModified()) { attrs.alloc("lastModified").mkInt(*lastModified); attrs.alloc("lastModifiedDate").mkString( diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 47282f6c4..be5842d53 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -243,6 +243,13 @@ std::pair fetchFromWorkdir(ref store, Input & input, co "lastModified", workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + if (workdirInfo.hasHead) { + input.attrs.insert_or_assign("dirtyRev", chomp( + runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty"); + input.attrs.insert_or_assign("dirtyShortRev", chomp( + runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty"); + } + return {std::move(storePath), input}; } } // end namespace @@ -283,7 +290,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1eea52e15..8d39de389 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -179,6 +179,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); + if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) + j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) @@ -204,6 +206,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", rev->to_string(Base16, false)); + if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) + logger->cout( + ANSI_BOLD "Revision:" ANSI_NORMAL " %s", + *dirtyRev); if (auto revCount = flake.lockedRef.input.getRevCount()) logger->cout( ANSI_BOLD "Revisions:" ANSI_NORMAL " %s", diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index e2ccb0e97..418b4f63f 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -105,6 +105,8 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $(cat $path2/dir1/foo) = foo ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]] # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") @@ -119,6 +121,10 @@ git -C $repo commit -m 'Bla3' -a path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"rev\" (builtins.fetchGit $repo)") == "true" ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyRev\" (builtins.fetchGit $repo)") == "false" ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyShortRev\" (builtins.fetchGit $repo)") == "false" ]] + status=0 nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? [[ "$status" = "102" ]] diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index f2e216435..128f759ea 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -95,11 +95,16 @@ json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) +echo foo > $flake1Dir/foo +git -C $flake1Dir add $flake1Dir/foo +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] + echo -n '# foo' >> $flake1Dir/flake.nix flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) git -C $flake1Dir commit -a -m 'Foo' flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] [[ $hash1 != $hash2 ]] # Test 'nix build' on a flake. From 559fd7ffe7a8cb61b11bec081f14ce3c3ffb210d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 27 Jun 2023 14:58:29 +0200 Subject: [PATCH 27/44] nix flake check: improve error message if overlay is not a lambda (#8582) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * nix flake check: improve error message if overlay is not a lambda Suppose you have an overlay like this { inputs = { /* ... */ }; outputs = { flake-utils, ... }: flake-utils.lib.eachDefaultSystem (system: { overlays.default = final: prev: { }; }); } then `nix flake check` (correctly) fails because `overlays` are supposed to have the structure `overlays. = final: prev: exp`. However, the error-message is a little bit counter-intuitive: error: overlay does not take an argument named 'final' While one might guess where the error actually comes from because the trace above says `… while checking the overlay 'overlays.x86_64-linux'` this is still pretty confusing because it complains about an argument not being named `final` even though that's evidently the case. With this change, the error-message actually makes it clear what's wrong: [ma27@carsten:~/Projects/nix/tmp]$ nix flake check --extra-experimental-features 'nix-command flakes' path:$(pwd) error: … while checking flake output 'overlays' at /nix/store/clgblnxx003hyrq8qkz5ab6kgqkck6qc-source/flake.nix:4:5: 3| outputs = { ... }: { 4| overlays.x86_64-linux.snens = final: prev: { | ^ 5| kek = throw "snens"; … while checking the overlay 'overlays.x86_64-linux' at /nix/store/clgblnxx003hyrq8qkz5ab6kgqkck6qc-source/flake.nix:4:5: 3| outputs = { ... }: { 4| overlays.x86_64-linux.snens = final: prev: { | ^ 5| kek = throw "snens"; error: overlay is not a lambda, but a set instead --- src/nix/flake.cc | 6 ++++-- tests/flakes/check.sh | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8d39de389..b5f5d0cac 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -386,8 +386,10 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { state->forceValue(v, pos); - if (!v.isLambda() - || v.lambda.fun->hasFormals() + if (!v.isLambda()) { + throw Error("overlay is not a function, but %s instead", showType(v)); + } + if (v.lambda.fun->hasFormals() || !argHasName(v.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast(v.lambda.fun->body); diff --git a/tests/flakes/check.sh b/tests/flakes/check.sh index 34b82c61c..0433e5335 100644 --- a/tests/flakes/check.sh +++ b/tests/flakes/check.sh @@ -25,6 +25,18 @@ EOF (! nix flake check $flakeDir) +cat > $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "error: overlay is not a function, but a set instead" + cat > $flakeDir/flake.nix < Date: Wed, 21 Jun 2023 16:06:16 -0400 Subject: [PATCH 28/44] Generialize `showType` --- src/libexpr/eval.cc | 23 ++++++++++++----------- src/libexpr/eval.hh | 5 ++++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 71fd6e6e4..8a0ff4cce 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -211,20 +211,21 @@ const Value * getPrimOp(const Value &v) { return primOp; } -std::string_view showType(ValueType type) +std::string_view showType(ValueType type, bool withArticle) { + #define WA(a, w) withArticle ? a " " w : w switch (type) { - case nInt: return "an integer"; - case nBool: return "a Boolean"; - case nString: return "a string"; - case nPath: return "a path"; + case nInt: return WA("an", "integer"); + case nBool: return WA("a", "Boolean"); + case nString: return WA("a", "string"); + case nPath: return WA("a", "path"); case nNull: return "null"; - case nAttrs: return "a set"; - case nList: return "a list"; - case nFunction: return "a function"; - case nExternal: return "an external value"; - case nFloat: return "a float"; - case nThunk: return "a thunk"; + case nAttrs: return WA("a", "set"); + case nList: return WA("a", "list"); + case nFunction: return WA("a", "function"); + case nExternal: return WA("an", "external value"); + case nFloat: return WA("a", "float"); + case nThunk: return WA("a", "thunk"); } abort(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 7b726a78f..0c07ae081 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -704,8 +704,11 @@ struct DebugTraceStacker { /** * @return A string representing the type of the value `v`. + * + * @param withArticle Whether to begin with an english article, e.g. "an + * integer" vs "integer". */ -std::string_view showType(ValueType type); +std::string_view showType(ValueType type, bool withArticle = true); std::string showType(const Value & v); /** From 4da7c866181c699b43d3cd2fff0afd2505774be7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 27 Jun 2023 09:32:40 -0400 Subject: [PATCH 29/44] Switch example to a primop this is less ill-advised Any primop will do for this, so might as well use one that isn't impure. Co-authored-by: Robert Hensing --- doc/manual/src/language/builtin-constants.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md index e6bc7e915..eb8b53bf0 100644 --- a/doc/manual/src/language/builtin-constants.md +++ b/doc/manual/src/language/builtin-constants.md @@ -9,7 +9,8 @@ These constants are built into the Nix language evaluator: Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: ```nix - if builtins ? getEnv then builtins.getEnv "PATH" else "" + # if hasContext is not available, we assume `s` has a context + if builtins ? hasContext then builtins.hasContext s else true ``` - [`builtins.currentSystem`]{#builtins-currentSystem} (string) From d40f0e534d657e619634ba204fb8970f4a01fbc5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 27 Jun 2023 09:35:51 -0400 Subject: [PATCH 30/44] Don't say this when we still pollute the global scope Co-authored-by: Robert Hensing --- doc/manual/src/language/builtin-constants.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md index eb8b53bf0..99520ad3f 100644 --- a/doc/manual/src/language/builtin-constants.md +++ b/doc/manual/src/language/builtin-constants.md @@ -4,7 +4,7 @@ These constants are built into the Nix language evaluator: - [`builtins`]{#builtins-builtins} (attribute set) - Contains all the [built-in functions](./builtins.md) and values, in order to avoid polluting the global scope. + Contains all the [built-in functions](./builtins.md) and values. Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: From 22b278e011ab9c1328749a126514c57b89a39173 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 May 2023 13:52:45 -0400 Subject: [PATCH 31/44] Automatically document builtin constants This is done in roughly the same way builtin functions are documented. Also auto-link experimental features for primops, subsuming PR #8371. Co-authored-by: Eelco Dolstra Co-authored-by: Robert Hensing Co-authored-by: Valentin Gagarin --- .gitignore | 3 +- doc/manual/generate-builtin-constants.nix | 29 ++ doc/manual/generate-builtins.nix | 18 +- doc/manual/local.mk | 16 +- .../src/language/builtin-constants-prefix.md | 5 + .../src/language/builtin-constants-suffix.md | 1 + doc/manual/src/language/builtin-constants.md | 44 --- src/libexpr/eval.cc | 37 ++- src/libexpr/eval.hh | 87 +++++- src/libexpr/flake/flake.cc | 3 - src/libexpr/primops.cc | 283 +++++++++++++++--- src/libexpr/primops.hh | 14 +- src/libexpr/primops/fetchClosure.cc | 3 - src/libexpr/value.hh | 10 +- src/nix/main.cc | 41 ++- 15 files changed, 447 insertions(+), 147 deletions(-) create mode 100644 doc/manual/generate-builtin-constants.nix create mode 100644 doc/manual/src/language/builtin-constants-prefix.md create mode 100644 doc/manual/src/language/builtin-constants-suffix.md delete mode 100644 doc/manual/src/language/builtin-constants.md diff --git a/.gitignore b/.gitignore index 29d9106ae..969194650 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ perl/Makefile.config /doc/manual/generated/* /doc/manual/nix.json /doc/manual/conf-file.json -/doc/manual/builtins.json +/doc/manual/language.json /doc/manual/xp-features.json /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli @@ -26,6 +26,7 @@ perl/Makefile.config /doc/manual/src/command-ref/experimental-features-shortlist.md /doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md +/doc/manual/src/language/builtin-constants.md # /scripts/ /scripts/nix-profile.sh diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix new file mode 100644 index 000000000..3fc1fae42 --- /dev/null +++ b/doc/manual/generate-builtin-constants.nix @@ -0,0 +1,29 @@ +let + inherit (builtins) concatStringsSep attrValues mapAttrs; + inherit (import ./utils.nix) optionalString squash; +in + +builtinsInfo: +let + showBuiltin = name: { doc, type, impure-only }: + let + type' = optionalString (type != null) " (${type})"; + + impureNotice = optionalString impure-only '' + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + ''; + in + squash '' +
+ ${name}${type'} +
+
+ + ${doc} + + ${impureNotice} + +
+ ''; +in +concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 71f96153f..813a287f5 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,24 +1,28 @@ let - inherit (builtins) concatStringsSep attrNames; + inherit (builtins) concatStringsSep attrValues mapAttrs; + inherit (import ./utils.nix) optionalString squash; in builtinsInfo: let - showBuiltin = name: + showBuiltin = name: { doc, args, arity, experimental-feature }: let - inherit (builtinsInfo.${name}) doc args; + experimentalNotice = optionalString (experimental-feature != null) '' + This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled. + ''; in - '' + squash ''
${name} ${listArgs args}
- ${doc} + ${doc} + + ${experimentalNotice}
''; listArgs = args: concatStringsSep " " (map (s: "${s}") args); in -concatStringsSep "\n" (map showBuiltin (attrNames builtinsInfo)) - +concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index b4b7283ef..abdfd6a62 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -128,14 +128,20 @@ $(d)/xp-features.json: $(bindir)/nix $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/builtins.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp +$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix + @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; + @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp + @mv $@.tmp $@ + +$(d)/language.json: $(bindir)/nix + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ # Generate the HTML manual. @@ -167,7 +173,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md new file mode 100644 index 000000000..50f43006d --- /dev/null +++ b/doc/manual/src/language/builtin-constants-prefix.md @@ -0,0 +1,5 @@ +# Built-in Constants + +These constants are built into the Nix language evaluator: + +
diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md new file mode 100644 index 000000000..a74db2857 --- /dev/null +++ b/doc/manual/src/language/builtin-constants-suffix.md @@ -0,0 +1 @@ +
diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md deleted file mode 100644 index 99520ad3f..000000000 --- a/doc/manual/src/language/builtin-constants.md +++ /dev/null @@ -1,44 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - -- [`builtins`]{#builtins-builtins} (attribute set) - - Contains all the [built-in functions](./builtins.md) and values. - - Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: - - ```nix - # if hasContext is not available, we assume `s` has a context - if builtins ? hasContext then builtins.hasContext s else true - ``` - -- [`builtins.currentSystem`]{#builtins-currentSystem} (string) - - The built-in value `currentSystem` evaluates to the Nix platform - identifier for the Nix installation on which the expression is being - evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. - - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). - -- [`builtins.currentTime`]{#builtins-currentTime} (integer) - - Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. - Repeated references to that name will re-use the initially obtained value. - - Example: - - ```console - $ nix repl - Welcome to Nix 2.15.1 Type :? for help. - - nix-repl> builtins.currentTime - 1683705525 - - nix-repl> builtins.currentTime - 1683705525 - ``` - - The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation. - - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8a0ff4cce..aaeb419ad 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -703,28 +703,34 @@ Path EvalState::toRealPath(const Path & path, const NixStringContext & context) } -Value * EvalState::addConstant(const std::string & name, Value & v) +Value * EvalState::addConstant(const std::string & name, Value & v, Constant info) { Value * v2 = allocValue(); *v2 = v; - addConstant(name, v2); + addConstant(name, v2, info); return v2; } -void EvalState::addConstant(const std::string & name, Value * v) +void EvalState::addConstant(const std::string & name, Value * v, Constant info) { - staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); - baseEnv.values[baseEnvDispl++] = v; auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); -} + constantInfos.push_back({name2, info}); -Value * EvalState::addPrimOp(const std::string & name, - size_t arity, PrimOpFun primOp) -{ - return addPrimOp(PrimOp { .fun = primOp, .arity = arity, .name = name }); + if (!(evalSettings.pureEval && info.impureOnly)) { + /* Check the type, if possible. + + We might know the type of a thunk in advance, so be allowed + to just write it down in that case. */ + if (auto gotType = v->type(true); gotType != nThunk) + assert(info.type == gotType); + + /* Install value the base environment. */ + staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); + } } @@ -738,7 +744,10 @@ Value * EvalState::addPrimOp(PrimOp && primOp) vPrimOp->mkPrimOp(new PrimOp(primOp)); Value v; v.mkApp(vPrimOp, vPrimOp); - return addConstant(primOp.name, v); + return addConstant(primOp.name, v, { + .type = nThunk, // FIXME + .doc = primOp.doc, + }); } auto envName = symbols.create(primOp.name); @@ -764,13 +773,13 @@ std::optional EvalState::getDoc(Value & v) { if (v.isPrimOp()) { auto v2 = &v; - if (v2->primOp->doc) + if (auto * doc = v2->primOp->doc) return Doc { .pos = {}, .name = v2->primOp->name, .arity = v2->primOp->arity, .args = v2->primOp->args, - .doc = v2->primOp->doc, + .doc = doc, }; } return {}; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0c07ae081..e3676c1b7 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -25,15 +25,72 @@ struct DerivedPath; enum RepairFlag : bool; +/** + * Function that implements a primop. + */ typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); +/** + * Info about a primitive operation, and its implementation + */ struct PrimOp { - PrimOpFun fun; - size_t arity; + /** + * Name of the primop. `__` prefix is treated specially. + */ std::string name; + + /** + * Names of the parameters of a primop, for primops that take a + * fixed number of arguments to be substituted for these parameters. + */ std::vector args; + + /** + * Aritiy of the primop. + * + * If `args` is not empty, this field will be computed from that + * field instead, so it doesn't need to be manually set. + */ + size_t arity = 0; + + /** + * Optional free-form documentation about the primop. + */ const char * doc = nullptr; + + /** + * Implementation of the primop. + */ + PrimOpFun fun; + + /** + * Optional experimental for this to be gated on. + */ + std::optional experimentalFeature; +}; + +/** + * Info about a constant + */ +struct Constant +{ + /** + * Optional type of the constant (known since it is a fixed value). + * + * @todo we should use an enum for this. + */ + ValueType type = nThunk; + + /** + * Optional free-form documentation about the constant. + */ + const char * doc = nullptr; + + /** + * Whether the constant is impure, and not available in pure mode. + */ + bool impureOnly = false; }; #if HAVE_BOEHMGC @@ -513,18 +570,23 @@ public: */ std::shared_ptr staticBaseEnv; // !!! should be private + /** + * Name and documentation about every constant. + * + * Constants from primops are hard to crawl, and their docs will go + * here too. + */ + std::vector> constantInfos; + private: unsigned int baseEnvDispl = 0; void createBaseEnv(); - Value * addConstant(const std::string & name, Value & v); + Value * addConstant(const std::string & name, Value & v, Constant info); - void addConstant(const std::string & name, Value * v); - - Value * addPrimOp(const std::string & name, - size_t arity, PrimOpFun primOp); + void addConstant(const std::string & name, Value * v, Constant info); Value * addPrimOp(PrimOp && primOp); @@ -538,6 +600,10 @@ public: std::optional name; size_t arity; std::vector args; + /** + * Unlike the other `doc` fields in this file, this one should never be + * `null`. + */ const char * doc; }; @@ -740,7 +806,12 @@ struct EvalSettings : Config Setting nixPath{ this, getDefaultNixPath(), "nix-path", - "List of directories to be searched for `<...>` file references."}; + R"( + List of directories to be searched for `<...>` file references + + In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath). + )"}; Setting restrictEval{ this, false, "restrict-eval", diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 60bb6a71e..5aa44d6a1 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -788,9 +788,6 @@ static RegisterPrimOp r2({ ```nix (builtins.getFlake "github:edolstra/dwarffs").rev ``` - - This function is only available if you enable the experimental feature - `flakes`. )", .fun = prim_getFlake, .experimentalFeature = Xp::Flakes, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9dd7bae02..5dfad470a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -238,7 +238,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v } } -static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { +static RegisterPrimOp primop_scopedImport(PrimOp { .name = "scopedImport", .arity = 2, .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -692,7 +692,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a v.listElems()[n++] = i; } -static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { +static RegisterPrimOp primop_genericClosure(PrimOp { .name = "__genericClosure", .args = {"attrset"}, .arity = 1, @@ -809,7 +809,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * } } -static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { +static RegisterPrimOp primop_addErrorContext(PrimOp { .name = "__addErrorContext", .arity = 2, .fun = prim_addErrorContext, @@ -1400,7 +1400,7 @@ drvName, Bindings * attrs, Value & v) v.mkAttrs(result); } -static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { +static RegisterPrimOp primop_derivationStrict(PrimOp { .name = "derivationStrict", .arity = 1, .fun = prim_derivationStrict, @@ -1667,9 +1667,52 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); } -static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { +static RegisterPrimOp primop_findFile(PrimOp { .name = "__findFile", - .arity = 2, + .args = {"search path", "lookup path"}, + .doc = R"( + Look up the given path with the given search path. + + A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`. + `prefix` is a relative path. + `path` denotes a file system location; the exact syntax depends on the command line interface. + + Examples of search path attribute sets: + + - ``` + { + prefix = "nixos-config"; + path = "/etc/nixos/configuration.nix"; + } + ``` + + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` + + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match. + + This is the process for each entry: + If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`. + Note that the `path` may need to be downloaded at this point to look inside. + If the suffix is found inside that directory, then the entry is a match; + the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + + The syntax + + ```nix + + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + )", .fun = prim_findFile, }); @@ -2388,7 +2431,7 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * * state.mkPos(v, i->pos); } -static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info { +static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp { .name = "__unsafeGetAttrPos", .arity = 2, .fun = prim_unsafeGetAttrPos, @@ -4061,10 +4104,10 @@ static RegisterPrimOp primop_splitVersion({ RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(Info && info) +RegisterPrimOp::RegisterPrimOp(PrimOp && primOp) { if (!primOps) primOps = new PrimOps; - primOps->push_back(std::move(info)); + primOps->push_back(std::move(primOp)); } @@ -4077,54 +4120,202 @@ void EvalState::createBaseEnv() /* `builtins' must be first! */ v.mkAttrs(buildBindings(128).finish()); - addConstant("builtins", v); + addConstant("builtins", v, { + .type = nAttrs, + .doc = R"( + Contains all the [built-in functions](@docroot@/language/builtins.md) and values. + + Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: + + ```nix + # if hasContext is not available, we assume `s` has a context + if builtins ? hasContext then builtins.hasContext s else true + ``` + )", + }); v.mkBool(true); - addConstant("true", v); + addConstant("true", v, { + .type = nBool, + .doc = R"( + Primitive value. + + It can be returned by + [comparison operators](@docroot@/language/operators.md#Comparison) + and used in + [conditional expressions](@docroot@/language/constructs.md#Conditionals). + + The name `true` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let true = 1; in true + 1 + ``` + )", + }); v.mkBool(false); - addConstant("false", v); + addConstant("false", v, { + .type = nBool, + .doc = R"( + Primitive value. + + It can be returned by + [comparison operators](@docroot@/language/operators.md#Comparison) + and used in + [conditional expressions](@docroot@/language/constructs.md#Conditionals). + + The name `false` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let false = 1; in false + 1 + ``` + )", + }); v.mkNull(); - addConstant("null", v); + addConstant("null", v, { + .type = nNull, + .doc = R"( + Primitive value. + + The name `null` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let null = 1; in null + 1 + ``` + )", + }); if (!evalSettings.pureEval) { v.mkInt(time(0)); - addConstant("__currentTime", v); - - v.mkString(settings.thisSystem.get()); - addConstant("__currentSystem", v); } + addConstant("__currentTime", v, { + .type = nInt, + .doc = R"( + Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. + Repeated references to that name will re-use the initially obtained value. + + Example: + + ```console + $ nix repl + Welcome to Nix 2.15.1 Type :? for help. + + nix-repl> builtins.currentTime + 1683705525 + + nix-repl> builtins.currentTime + 1683705525 + ``` + + The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second. + )", + .impureOnly = true, + }); + + if (!evalSettings.pureEval) { + v.mkString(settings.thisSystem.get()); + } + addConstant("__currentSystem", v, { + .type = nString, + .doc = R"( + The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: + + ```nix + builtins.derivation { + # ... + system = builtins.currentSystem; + } + ``` + + It can be overridden in order to create derivations for different system than the current one: + + ```console + $ nix-instantiate --system "mips64-linux" --eval --expr 'builtins.currentSystem' + "mips64-linux" + ``` + )", + .impureOnly = true, + }); v.mkString(nixVersion); - addConstant("__nixVersion", v); + addConstant("__nixVersion", v, { + .type = nString, + .doc = R"( + The version of Nix. + + For example, where the command line returns the current Nix version, + + ```shell-session + $ nix --version + nix (Nix) 2.16.0 + ``` + + the Nix language evaluator returns the same value: + + ```nix-repl + nix-repl> builtins.nixVersion + "2.16.0" + ``` + )", + }); v.mkString(store->storeDir); - addConstant("__storeDir", v); + addConstant("__storeDir", v, { + .type = nString, + .doc = R"( + Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. + + This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md): + + ```shell-session + $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir + "/blah" + ``` + )", + }); /* Language version. This should be increased every time a new language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ v.mkInt(6); - addConstant("__langVersion", v); + addConstant("__langVersion", v, { + .type = nInt, + .doc = R"( + The current version of the Nix language. + )", + }); // Miscellaneous if (evalSettings.enableNativeCode) { - addPrimOp("__importNative", 2, prim_importNative); - addPrimOp("__exec", 1, prim_exec); + addPrimOp({ + .name = "__importNative", + .arity = 2, + .fun = prim_importNative, + }); + addPrimOp({ + .name = "__exec", + .arity = 1, + .fun = prim_exec, + }); } addPrimOp({ - .fun = evalSettings.traceVerbose ? prim_trace : prim_second, - .arity = 2, .name = "__traceVerbose", .args = { "e1", "e2" }, + .arity = 2, .doc = R"( Evaluate *e1* and print its abstract syntax representation on standard error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", + .fun = evalSettings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ @@ -4136,26 +4327,46 @@ void EvalState::createBaseEnv() attrs.alloc("prefix").mkString(i.prefix); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } - addConstant("__nixPath", v); + addConstant("__nixPath", v, { + .type = nList, + .doc = R"( + The search path used to resolve angle bracket path lookups. + + Angle bracket expressions can be + [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) + using this and + [`builtins.findFile`](./builtins.html#builtins-findFile): + + ```nix + + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + )", + }); if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) - if (!primOp.experimentalFeature - || experimentalFeatureSettings.isEnabled(*primOp.experimentalFeature)) + if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) { - addPrimOp({ - .fun = primOp.fun, - .arity = std::max(primOp.args.size(), primOp.arity), - .name = primOp.name, - .args = primOp.args, - .doc = primOp.doc, - }); + auto primOpAdjusted = primOp; + primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); + addPrimOp(std::move(primOpAdjusted)); } /* Add a wrapper around the derivation primop that computes the - `drvPath' and `outPath' attributes lazily. */ + `drvPath' and `outPath' attributes lazily. + + Null docs because it is documented separately. + */ auto vDerivation = allocValue(); - addConstant("derivation", vDerivation); + addConstant("derivation", vDerivation, { + .type = nFunction, + }); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 73b7b866c..930e7f32a 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -10,17 +10,7 @@ namespace nix { struct RegisterPrimOp { - struct Info - { - std::string name; - std::vector args; - size_t arity = 0; - const char * doc; - PrimOpFun fun; - std::optional experimentalFeature; - }; - - typedef std::vector PrimOps; + typedef std::vector PrimOps; static PrimOps * primOps; /** @@ -28,7 +18,7 @@ struct RegisterPrimOp * will get called during EvalState initialization, so there * may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp(Info && info); + RegisterPrimOp(PrimOp && primOp); }; /* These primops are disabled without enableNativeCode, but plugins diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 4cf1f1e0b..bae849f61 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -154,9 +154,6 @@ static RegisterPrimOp primop_fetchClosure({ specifying a binary cache from which the path can be fetched. Also, requiring a content-addressed final store path avoids the need for users to configure binary cache public keys. - - This function is only available if you enable the experimental - feature `fetch-closure`. )", .fun = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 89c0c36fd..44c8071c2 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -218,8 +218,11 @@ public: /** * Returns the normal type of a Value. This only returns nThunk if * the Value hasn't been forceValue'd + * + * @param invalidIsThunk Instead of aborting an an invalid (probably + * 0, so uninitialized) internal type, return `nThunk`. */ - inline ValueType type() const + inline ValueType type(bool invalidIsThunk = false) const { switch (internalType) { case tInt: return nInt; @@ -234,7 +237,10 @@ public: case tFloat: return nFloat; case tThunk: case tApp: case tBlackhole: return nThunk; } - abort(); + if (invalidIsThunk) + return nThunk; + else + abort(); } /** diff --git a/src/nix/main.cc b/src/nix/main.cc index ce0bed2a3..650c79d14 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -352,7 +352,7 @@ void mainWrapped(int argc, char * * argv) return; } - if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { + if (argc == 2 && std::string(argv[1]) == "__dump-language") { experimentalFeatureSettings.experimentalFeatures = { Xp::Flakes, Xp::FetchClosure, @@ -360,17 +360,34 @@ void mainWrapped(int argc, char * * argv) evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); auto res = nlohmann::json::object(); - auto builtins = state.baseEnv.values[0]->attrs; - for (auto & builtin : *builtins) { - auto b = nlohmann::json::object(); - if (!builtin.value->isPrimOp()) continue; - auto primOp = builtin.value->primOp; - if (!primOp->doc) continue; - b["arity"] = primOp->arity; - b["args"] = primOp->args; - b["doc"] = trim(stripIndentation(primOp->doc)); - res[state.symbols[builtin.name]] = std::move(b); - } + res["builtins"] = ({ + auto builtinsJson = nlohmann::json::object(); + auto builtins = state.baseEnv.values[0]->attrs; + for (auto & builtin : *builtins) { + auto b = nlohmann::json::object(); + if (!builtin.value->isPrimOp()) continue; + auto primOp = builtin.value->primOp; + if (!primOp->doc) continue; + b["arity"] = primOp->arity; + b["args"] = primOp->args; + b["doc"] = trim(stripIndentation(primOp->doc)); + b["experimental-feature"] = primOp->experimentalFeature; + builtinsJson[state.symbols[builtin.name]] = std::move(b); + } + std::move(builtinsJson); + }); + res["constants"] = ({ + auto constantsJson = nlohmann::json::object(); + for (auto & [name, info] : state.constantInfos) { + auto c = nlohmann::json::object(); + if (!info.doc) continue; + c["doc"] = trim(stripIndentation(info.doc)); + c["type"] = showType(info.type, false); + c["impure-only"] = info.impureOnly; + constantsJson[name] = std::move(c); + } + std::move(constantsJson); + }); logger->cout("%s", res); return; } From 80c9259756811c1165167db1bb66c1fef0accb65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 27 Jun 2023 12:01:08 +0200 Subject: [PATCH 32/44] Allow to sign path as unprivileged user User can now sign path as unprivileged/allowed user refs #1708 --- doc/manual/src/release-notes/rl-next.md | 3 +++ src/libstore/daemon.cc | 2 -- tests/nixos/authorization.nix | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index bde9057c6..8479b166a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,6 @@ # Release X.Y (202?-??-??) - [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand + +- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 75c3d2aca..ad3dee1a2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -864,8 +864,6 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); StringSet sigs = readStrings(from); logger->startWork(); - if (!trusted) - throw Error("you are not privileged to add signatures"); store->addSignatures(path, sigs); logger->stopWork(); to << 1; diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix index 7e8744dd9..fdeae06ed 100644 --- a/tests/nixos/authorization.nix +++ b/tests/nixos/authorization.nix @@ -75,5 +75,20 @@ su --login bob -c '(! nix-store --verify --repair 2>&1)' | tee diag 1>&2 grep -F "you are not privileged to repair paths" diag """) + + machine.succeed(""" + set -x + su --login mallory -c ' + nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 + (! nix store sign --key-file sk1 ${pathFour} 2>&1)' | tee diag 1>&2 + grep -F "cannot open connection to remote store 'daemon'" diag + """) + + machine.succeed(""" + su --login bob -c ' + nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 + nix store sign --key-file sk1 ${pathFour} + ' + """) ''; } From 2ccc02515f4a52d59f3473493f254942209fc81b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 27 Jun 2023 18:22:42 -0400 Subject: [PATCH 33/44] Trailing commas in redirects This avoids diff noise when more are added. Unlike with JSON, this is allowed in JS. --- doc/manual/redirects.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index d75cd231a..f54f6ea6a 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -330,23 +330,23 @@ const redirects = { "ssec-relnotes-2.0": "release-notes/rl-2.0.html", "ssec-relnotes-2.1": "release-notes/rl-2.1.html", "ssec-relnotes-2.2": "release-notes/rl-2.2.html", - "ssec-relnotes-2.3": "release-notes/rl-2.3.html" + "ssec-relnotes-2.3": "release-notes/rl-2.3.html", }, "language/values.html": { "simple-values": "#primitives", "lists": "#list", "strings": "#string", "lists": "#list", - "attribute-sets": "#attribute-set" + "attribute-sets": "#attribute-set", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", - "uninstalling": "uninstall.html" + "uninstalling": "uninstall.html", } "contributing/hacking.html": { - "nix-with-flakes": "#building-nix-with-flakes" - "classic-nix": "#building-nix" + "nix-with-flakes": "#building-nix-with-flakes", + "classic-nix": "#building-nix", } }; From ca49e13414b4d1d9b2cdd72e36920e28c4ef117e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 21 Jun 2023 23:33:33 -0400 Subject: [PATCH 34/44] Split testing into its own page in the contribution guide `hacking.md` has gotten really big! --- doc/manual/redirects.js | 8 ++ doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/contributing/hacking.md | 165 ------------------------ doc/manual/src/contributing/testing.md | 167 +++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 165 deletions(-) create mode 100644 doc/manual/src/contributing/testing.md diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index f54f6ea6a..dcdb5d6e9 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -347,6 +347,14 @@ const redirects = { "contributing/hacking.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", + "running-tests": "testing.html#running-tests", + "unit-tests": "testing.html#unit-tests", + "functional-tests": "testing.html#functional-tests", + "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests", + "integration-tests": "testing.html#integration-tests", + "installer-tests": "testing.html#installer-tests", + "one-time-setup": "testing.html#one-time-setup", + "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", } }; diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index bbb603713..1bd8fa774 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -104,6 +104,7 @@ - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Testing](contributing/testing.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 6c6f2eb52..7b2440971 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -190,171 +190,6 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Running tests - -### Unit-tests - -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. - -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. - -### Functional tests - -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. -Each test is a bash script. - -The whole test suite can be run with: - -```shell-session -$ make install && make installcheck -ran test tests/foo.sh... [PASS] -ran test tests/bar.sh... [PASS] -... -``` - -Individual tests can be run with `make`: - -```shell-session -$ make tests/${testName}.sh.test -ran test tests/${testName}.sh... [PASS] -``` - -or without `make`: - -```shell-session -$ ./mk/run-test.sh tests/${testName}.sh -ran test tests/${testName}.sh... [PASS] -``` - -To see the complete output, one can also run: - -```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -+ foo -output from foo -+ bar -output from bar -... -``` - -The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. - -#### Debugging failing functional tests - -When a functional test fails, it usually does so somewhere in the middle of the script. - -To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. - -For example, if the script looks like: - -```bash -foo -nix blah blub -bar -``` -edit it like so: - -```diff - foo --nix blah blub -+gdb --args nix blah blub - bar -``` - -Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: - -```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -... -+ gdb blash blub -GNU gdb (GDB) 12.1 -... -(gdb) -``` - -One can debug the Nix invocation in all the usual ways. -For example, enter `run` to start the Nix invocation. - -### Integration tests - -The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. -These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. -Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). - -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` - -### Installer tests - -After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. - -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - -- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - - `x86_64-linux` - - `armv6l-linux` - - `armv7l-linux` - - `x86_64-darwin` - -- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. - -#### One-time setup - -1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). -2. At cachix.org: - - Create or log in to an account. - - Create a Cachix cache using the format `-nix-install-tests`. - - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new Cachix auth token and copy the generated value. -3. At github.com: - - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN`. - - Paste the copied value of the Cachix cache auth token. - -#### Using the CI-generated installer for manual testing - -After the CI run completes, you can check the output to extract the installer URL: -1. Click into the detailed view of the CI run. -2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). -3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the value of `install_url` -5. To generate an install command, plug this `install_url` and your GitHub username into this template: - - ```console - curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve - ``` - - - ### Checking links in the manual The build checks for broken internal links. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md new file mode 100644 index 000000000..e5f80a928 --- /dev/null +++ b/doc/manual/src/contributing/testing.md @@ -0,0 +1,167 @@ +# Running tests + +## Unit-tests + +The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined +under `src/{library_name}/tests` using the +[googletest](https://google.github.io/googletest/) and +[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. + +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. + +## Functional tests + +The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +Each test is a bash script. + +The whole test suite can be run with: + +```shell-session +$ make install && make installcheck +ran test tests/foo.sh... [PASS] +ran test tests/bar.sh... [PASS] +... +``` + +Individual tests can be run with `make`: + +```shell-session +$ make tests/${testName}.sh.test +ran test tests/${testName}.sh... [PASS] +``` + +or without `make`: + +```shell-session +$ ./mk/run-test.sh tests/${testName}.sh +ran test tests/${testName}.sh... [PASS] +``` + +To see the complete output, one can also run: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh ++ foo +output from foo ++ bar +output from bar +... +``` + +The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. + +### Debugging failing functional tests + +When a functional test fails, it usually does so somewhere in the middle of the script. + +To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. + +For example, if the script looks like: + +```bash +foo +nix blah blub +bar +``` +edit it like so: + +```diff + foo +-nix blah blub ++gdb --args nix blah blub + bar +``` + +Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh +... ++ gdb blash blub +GNU gdb (GDB) 12.1 +... +(gdb) +``` + +One can debug the Nix invocation in all the usual ways. +For example, enter `run` to start the Nix invocation. + +## Integration tests + +The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. +These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. +Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). + +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` + +## Installer tests + +After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. + +Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): + +- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: + - `x86_64-linux` + - `armv6l-linux` + - `armv7l-linux` + - `x86_64-darwin` + +- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. + +### One-time setup + +1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). +2. At cachix.org: + - Create or log in to an account. + - Create a Cachix cache using the format `-nix-install-tests`. + - Navigate to the new cache > Settings > Auth Tokens. + - Generate a new Cachix auth token and copy the generated value. +3. At github.com: + - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. + - Name the secret `CACHIX_AUTH_TOKEN`. + - Paste the copied value of the Cachix cache auth token. + +## Working on documentation + +### Using the CI-generated installer for manual testing + +After the CI run completes, you can check the output to extract the installer URL: +1. Click into the detailed view of the CI run. +2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). +3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) +4. Copy the value of `install_url` +5. To generate an install command, plug this `install_url` and your GitHub username into this template: + + ```console + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + ``` + + + From 3468cbaf479e3a2e9b3c7d6b00b2cb1976cce43e Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Wed, 28 Jun 2023 22:38:44 +0800 Subject: [PATCH 35/44] libexpr: fix leaking `es2` in stripIndentation (parser.y) --- src/libexpr/parser.y | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 5c9597f8a..ed6421ced 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -275,7 +275,12 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, } /* If this is a single string, then don't do a concatenation. */ - return es2->size() == 1 && dynamic_cast((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2); + if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { + auto *const result = (*es2)[0].second; + delete es2; + return result; + } + return new ExprConcatStrings(pos, true, es2); } From 685f1bb3864d52b80e74f920a150f8ffc7a479c3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Jun 2023 15:10:07 +0200 Subject: [PATCH 36/44] labeler.yml: tests -> with-tests --- .github/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index fce0d3aeb..12120bdb3 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -16,7 +16,7 @@ "new-cli": - src/nix/**/* -"tests": +"with-tests": # Unit tests - src/*/tests/**/* # Functional and integration tests From eebfe989a5dc3aac622b9b5f2edef4461d8968c1 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 30 Jun 2023 21:11:59 +0800 Subject: [PATCH 37/44] linkOrCopy: Fallback upon cross-device link error (EXDEV) Fix building derivations in local chroot store on OpenAFS, where hard linking accross directories causes cross-device link error (EXDEV). --- src/libstore/build/local-derivation-goal.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ea7c52098..ee66ee500 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -395,8 +395,9 @@ static void linkOrCopy(const Path & from, const Path & to) bind-mount in this case? It can also fail with EPERM in BeegFS v7 and earlier versions + or fail with EXDEV in OpenAFS which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) + if (errno != EMLINK && errno != EPERM && errno != EXDEV) throw SysError("linking '%s' to '%s'", to, from); copyPath(from, to); } From a6c17097d2ee76a347d86c42a37fd6de0d0fd3ef Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Jun 2023 23:36:27 +0200 Subject: [PATCH 38/44] tests: Don't install test-libstoreconsumer program Sorry about that. Fixes https://github.com/NixOS/nix/issues/8616 --- tests/test-libstoreconsumer/local.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test-libstoreconsumer/local.mk b/tests/test-libstoreconsumer/local.mk index cd2d0c7f8..edc140723 100644 --- a/tests/test-libstoreconsumer/local.mk +++ b/tests/test-libstoreconsumer/local.mk @@ -2,6 +2,9 @@ programs += test-libstoreconsumer test-libstoreconsumer_DIR := $(d) +# do not install +test-libstoreconsumer_INSTALL_DIR := + test-libstoreconsumer_SOURCES := \ $(wildcard $(d)/*.cc) \ From 87b82db8812736922de851a27681dd0b2955c2f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 10:33:58 +0200 Subject: [PATCH 39/44] nix profile list: Improve readability of the output --- src/nix/profile-list.md | 46 ++++++++++++++++++++++++++++++----------- src/nix/profile.cc | 14 +++++++++---- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index fa786162f..5d7fcc0ec 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -6,26 +6,48 @@ R""( ```console # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 - 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 - 2 flake:blender-bin#packages.x86_64-linux.default github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#packages.x86_64-linux.default /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + Index: 0 + Flake attribute: legacyPackages.x86_64-linux.gdb + Original flake URL: flake:nixpkgs + Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca + Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1 + + Index: 1 + Flake attribute: packages.x86_64-linux.default + Original flake URL: flake:blender-bin + Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender + Store paths: /nix/store/i798sxl3j40wpdi1rgf391id1b5klw7g-blender-bin-3.1.2 ``` + Note that you can unambiguously rebuild a package from a profile + through its locked flake URL and flake attribute, e.g. + + ```console + # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default + ``` + + will build the package with index 1 shown above. + # Description This command shows what packages are currently installed in a -profile. The output consists of one line per package, with the -following fields: +profile. For each installed package, it shows the following +information: -* An integer that can be used to unambiguously identify the package in - invocations of `nix profile remove` and `nix profile upgrade`. +* `Index`: An integer that can be used to unambiguously identify the + package in invocations of `nix profile remove` and `nix profile + upgrade`. -* The original ("unlocked") flake reference and output attribute path - used at installation time. +* `Flake attribute`: The flake output attribute path that provides the + package (e.g. `packages.x86_64-linux.hello`). -* The locked flake reference to which the unlocked flake reference was - resolved. +* `Original flake URL`: The original ("unlocked") flake reference + specified by the user when the package was first installed via `nix + profile install`. -* The store path(s) of the package. +* `Locked flake URL`: The locked flake reference to which the original + flake reference was resolved. + +* `Store paths`: The store path(s) of the package. )"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index f3b73f10d..b0cedb24a 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -655,10 +655,16 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); - logger->cout("%d %s %s %s", i, - element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", - element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", - concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + if (i) logger->cout(""); + logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + i, + element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + if (element.source) { + logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); + logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->resolvedRef.to_string()); + } + logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } }; From 3c90340fe64fb18e66f02a648ebce8ad629482a5 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Mon, 3 Jul 2023 16:02:54 +0800 Subject: [PATCH 40/44] libexpr: use `thread_local` to make the parser thread-safe If we call `adjustLoc`, the global variable `prev_yylloc` is shared between threads and racy. Currently, nix itself does not concurrently parsing files, but this is helpful for libexpr users. (The parser is thread-safe except this.) --- src/libexpr/lexer.l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 462b3b602..a3a8608d9 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -36,7 +36,7 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) #define CUR_POS makeCurPos(*yylloc, data) // backup to recover from yyless(0) -YYLTYPE prev_yylloc; +thread_local YYLTYPE prev_yylloc; static void initLoc(YYLTYPE * loc) { From b8e8f27159251091141ccd6db12dd2d07204771a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 10:35:28 +0200 Subject: [PATCH 41/44] Rename 'resolvedRef' to 'lockedRef' 'resolvedRef' was incorrect, since a resolved ref is one after registry resolution, which may still be unlocked (e.g. 'nixpkgs' -> 'github:NixOS/nixpkgs'). --- src/libcmd/installable-flake.cc | 2 +- src/libcmd/installable-flake.hh | 2 +- src/nix/profile.cc | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index eb944240b..4da9b131b 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -151,7 +151,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() }, ExtraPathInfoFlake::Flake { .originalRef = flakeRef, - .resolvedRef = getLockedFlake()->flake.lockedRef, + .lockedRef = getLockedFlake()->flake.lockedRef, }), }}; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 7ac4358d2..314918c14 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -19,7 +19,7 @@ struct ExtraPathInfoFlake : ExtraPathInfoValue */ struct Flake { FlakeRef originalRef; - FlakeRef resolvedRef; + FlakeRef lockedRef; }; Flake flake; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b0cedb24a..ae2e8f4f1 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -21,7 +21,7 @@ struct ProfileElementSource { FlakeRef originalRef; // FIXME: record original attrpath. - FlakeRef resolvedRef; + FlakeRef lockedRef; std::string attrPath; ExtendedOutputsSpec outputs; @@ -181,7 +181,7 @@ struct ProfileManifest obj["priority"] = element.priority; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); - obj["url"] = element.source->resolvedRef.to_string(); + obj["url"] = element.source->lockedRef.to_string(); obj["attrPath"] = element.source->attrPath; obj["outputs"] = element.source->outputs; } @@ -349,7 +349,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile if (auto * info2 = dynamic_cast(&*info)) { element.source = ProfileElementSource { .originalRef = info2->flake.originalRef, - .resolvedRef = info2->flake.resolvedRef, + .lockedRef = info2->flake.lockedRef, .attrPath = info2->value.attrPath, .outputs = info2->value.extendedOutputsSpec, }; @@ -588,14 +588,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf assert(infop); auto & info = *infop; - if (element.source->resolvedRef == info.flake.resolvedRef) continue; + if (element.source->lockedRef == info.flake.lockedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef); + element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); element.source = ProfileElementSource { .originalRef = installable->flakeRef, - .resolvedRef = info.flake.resolvedRef, + .lockedRef = info.flake.lockedRef, .attrPath = info.value.attrPath, .outputs = installable->extendedOutputsSpec, }; @@ -662,7 +662,7 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro if (element.source) { logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); - logger->cout("Locked flake URL: %s", element.source->resolvedRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); } logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } From a353412c43e776f4fdc709dc7affd1dbb9d588be Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 10:44:26 +0200 Subject: [PATCH 42/44] nix profile list: Add --json flag This just dumps the profile manifest to stdout. --- src/nix/profile.cc | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ae2e8f4f1..b833b5192 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -168,7 +168,7 @@ struct ProfileManifest } } - std::string toJSON(Store & store) const + nlohmann::json toJSON(Store & store) const { auto array = nlohmann::json::array(); for (auto & element : elements) { @@ -190,7 +190,7 @@ struct ProfileManifest nlohmann::json json; json["version"] = 2; json["elements"] = array; - return json.dump(); + return json; } StorePath build(ref store) @@ -210,7 +210,7 @@ struct ProfileManifest buildProfile(tempDir, std::move(pkgs)); - writeFile(tempDir + "/manifest.json", toJSON(*store)); + writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); /* Add the symlink tree to the store. */ StringSink sink; @@ -635,7 +635,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile, MixJSON { std::string description() override { @@ -653,18 +653,22 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro { ProfileManifest manifest(*getEvalState(), *profile); - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - if (i) logger->cout(""); - logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", - i, - element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); - if (element.source) { - logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); - logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); - logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); + if (json) { + std::cout << manifest.toJSON(*store).dump() << "\n"; + } else { + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (i) logger->cout(""); + logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + i, + element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + if (element.source) { + logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); + logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); + } + logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } - logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } }; From 5fbfbb4c7c246090df3debf7688f5ee9f01a4cbb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 13:53:56 +0200 Subject: [PATCH 43/44] Fix test --- tests/nix-profile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 9da3f802b..7c478a0cd 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -47,8 +47,9 @@ cp ./config.nix $flake1Dir/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 -nix profile list | grep '0 - - .*-foo-1.0' +nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0' nix profile install $flake1Dir -L +nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash' [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) From 82d6699976aa51cb6c06272d3455b32299b2c4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Tue, 4 Jul 2023 10:39:08 +0200 Subject: [PATCH 44/44] Document the path flakeref format (#8640) * Document the path flakeref format Fix https://github.com/NixOS/nix/issues/8482 Co-authored-by: Valentin Gagarin --- src/nix/flake.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 456fd0ea1..92f477917 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -71,8 +71,6 @@ inputs.nixpkgs = { Here are some examples of flake references in their URL-like representation: -* `.`: The flake in the current directory. -* `/home/alice/src/patchelf`: A flake in some other directory. * `nixpkgs`: The `nixpkgs` entry in the flake registry. * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` entry in the flake registry, with its Git revision overridden to a @@ -93,6 +91,23 @@ Here are some examples of flake references in their URL-like representation: * `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball flake. +## Path-like syntax + +Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity). + +The semantic of such a path is as follows: + +* If the directory is part of a Git repository, then the input will be treated as a `git+file:` URL, otherwise it will be treated as a `path:` url; +* If the directory doesn't contain a `flake.nix` file, then Nix will search for such a file upwards in the file system hierarchy until it finds any of: + 1. The Git repository root, or + 2. The filesystem root (/), or + 3. A folder on a different mount point. + +### Examples + +* `.`: The flake to which the current directory belongs to. +* `/home/alice/src/patchelf`: A flake in some other directory. + ## Flake reference attributes The following generic flake reference attributes are supported: