From 02bff90e7bbc04244b3d566d82c945156cb0885b Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 1 Sep 2022 22:02:38 +0200 Subject: [PATCH 01/66] nix flake show: don't evaluate derivations for foreign systems by default --- src/nix/flake.cc | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3967f1102..5232534c6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -957,6 +957,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; + bool showForeign = false; CmdFlakeShow() { @@ -965,6 +966,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); + addFlag({ + .longName = "foreign", + .description = "Show the contents of outputs for foreign systems.", + .handler = {&showForeign, true} + }); } std::string description() override @@ -985,6 +991,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = std::make_shared(lockFlake()); + auto localSystem = std::string(settings.thisSystem.get()); std::functioncout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--foreign' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--foreign' to show)", concatStringsSep(".", attrPathS))); + } + } else { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } } else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { From 6da4cc92d8c546939818b65ba4f1b4ce65d88d6e Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 7 Sep 2022 20:28:30 +0200 Subject: [PATCH 02/66] showForeign -> showAllSystems --- src/nix/flake.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5232534c6..bb8be872b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -957,7 +957,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; - bool showForeign = false; + bool showAllSystems = false; CmdFlakeShow() { @@ -967,9 +967,9 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .handler = {&showLegacy, true} }); addFlag({ - .longName = "foreign", - .description = "Show the contents of outputs for foreign systems.", - .handler = {&showForeign, true} + .longName = "all-systems", + .description = "Show the contents of outputs for all systems.", + .handler = {&showAllSystems, true} }); } @@ -1082,11 +1082,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells")) ) { - if (!showForeign && std::string(attrPathS[1]) != localSystem) { + if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { if (!json) - logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--foreign' to show)", headerPrefix)); + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--foreign' to show)", concatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); } } else { if (visitor.isDerivation()) From c5fd34a14e2a271b66839b35f500caeae3368be2 Mon Sep 17 00:00:00 2001 From: Graham Bennett Date: Sat, 22 Oct 2022 14:24:25 +0100 Subject: [PATCH 03/66] Build with C++20 --- Makefile | 2 +- perl/Makefile | 2 +- src/libcmd/nix-cmd.pc.in | 2 +- src/libexpr/nix-expr.pc.in | 2 +- src/libmain/nix-main.pc.in | 2 +- src/libstore/nix-store.pc.in | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c1a1ce2c7..42d11638b 100644 --- a/Makefile +++ b/Makefile @@ -36,4 +36,4 @@ endif include mk/lib.mk -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src +GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++20 -I src diff --git a/perl/Makefile b/perl/Makefile index 708f86882..2d759e6fc 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,6 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -I ../src +GLOBAL_CXXFLAGS += -g -Wall -std=c++20 -I ../src -include Makefile.config diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in index 1761a9f41..a21d93f1d 100644 --- a/src/libcmd/nix-cmd.pc.in +++ b/src/libcmd/nix-cmd.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixcmd -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++20 diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 80f7a492b..95d452ca8 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++20 diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 37b03dcd4..b46ce1990 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++20 diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 6d67b1e03..385169a13 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++20 From 4563e803633d44ec57dfc4515f0c42c8855c018b Mon Sep 17 00:00:00 2001 From: Graham Bennett Date: Sat, 22 Oct 2022 14:25:35 +0100 Subject: [PATCH 04/66] Fix C++20 warnings --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/build/worker.cc | 2 +- src/libstore/globals.cc | 6 +++--- src/libutil/args.cc | 2 +- src/libutil/config.cc | 8 ++++---- src/libutil/config.hh | 12 ++++++++---- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a26770c79..9ab941075 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -378,7 +378,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); getFile(narInfoFile, - {[=](std::future> fut) { + {[=,this](std::future> fut) { try { auto data = fut.get(); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b192fbc77..75406bec9 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -276,7 +276,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) + if (awake.empty() && 0U == settings.maxBuildJobs) { if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ff658c428..c264014be 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -218,19 +218,19 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s .longName = name, .description = "Enable sandboxing.", .category = category, - .handler = {[=]() { override(smEnabled); }} + .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, .description = "Disable sandboxing.", .category = category, - .handler = {[=]() { override(smDisabled); }} + .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, - .handler = {[=]() { override(smRelaxed); }} + .handler = {[this]() { override(smRelaxed); }} }); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 753980fd4..2930913d6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -324,7 +324,7 @@ MultiCommand::MultiCommand(const Commands & commands_) expectArgs({ .label = "subcommand", .optional = true, - .handler = {[=](std::string s) { + .handler = {[=,this](std::string s) { assert(!command); auto i = commands.find(s); if (i == commands.end()) { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 9bb412b4f..b349f2d80 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -209,7 +209,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s); }}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, }); if (isAppendable()) @@ -218,7 +218,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s, true); }}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, }); } @@ -270,13 +270,13 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(true); }} + .handler = {[this]() { override(true); }} }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(false); }} + .handler = {[this]() { override(false); }} }); } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 79ec0f9cf..7ac43c854 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -250,11 +250,15 @@ public: operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } - bool operator ==(const T & v2) const { return value == v2; } - bool operator !=(const T & v2) const { return value != v2; } - void operator =(const T & v) { assign(v); } + template + bool operator ==(const U & v2) const { return value == v2; } + template + bool operator !=(const U & v2) const { return value != v2; } + template + void operator =(const U & v) { assign(v); } virtual void assign(const T & v) { value = v; } - void setDefault(const T & v) { if (!overridden) value = v; } + template + void setDefault(const U & v) { if (!overridden) value = v; } void set(const std::string & str, bool append = false) override; From 15f7fa59bec1dbb4fa94bc1c2f9018b1f14f0d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 23 Sep 2022 23:30:29 +0200 Subject: [PATCH 05/66] `unsafeDiscardReferences` Adds a new boolean structured attribute `outputChecks..unsafeDiscardReferences` which disables scanning an output for runtime references. __structuredAttrs = true; outputChecks.out.unsafeDiscardReferences = true; This is useful when creating filesystem images containing their own embedded Nix store: they are self-contained blobs of data with no runtime dependencies. Setting this attribute requires the experimental feature `discard-references` to be enabled. --- .../src/language/advanced-attributes.md | 6 ++--- doc/manual/src/release-notes/rl-next.md | 12 +++++++++ src/libstore/build/local-derivation-goal.cc | 27 ++++++++++++++++--- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 1 + tests/check-refs.nix | 9 +++++++ tests/check-refs.sh | 9 +++++++ 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2e7e80ed0..f02425b13 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -207,13 +207,13 @@ Derivations can declare some infrequently used optional attributes. the hash in either hexadecimal or base-32 notation. (See the [`nix-hash` command](../command-ref/nix-hash.md) for information about converting to and from base-32 notation.) - + - [`__contentAddressed`]{#adv-attr-__contentAddressed} If this **experimental** attribute is set to true, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivation` experimental feature is enabled. - + This only has an effect if the `ca-derivations` experimental feature is enabled. + Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). - [`passAsFile`]{#adv-attr-passAsFile}\ diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 9f491efc8..51320d50a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -21,3 +21,15 @@ * Error traces have been reworked to provide detailed explanations and more accurate error locations. A short excerpt of the trace is now shown by default when an error occurs. + +* In derivations that use structured attributes, you can now use `unsafeDiscardReferences` + to disable scanning a given output for runtime dependencies: + ```nix + __structuredAttrs = true; + outputChecks.out.unsafeDiscardReferences = true; + ``` + This is useful e.g. when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + + This requires the `discard-references` experimental feature. diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9d869d513..65340d6ce 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2319,11 +2319,30 @@ DrvOutputs LocalDerivationGoal::registerOutputs() buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { + if (auto output = get(*outputChecks, outputName)) { + if (auto unsafeDiscardReferences = get(*output, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (!unsafeDiscardReferences->is_boolean()) + throw Error("attribute 'outputChecks.\"%s\".unsafeDiscardReferences' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = unsafeDiscardReferences->get(); + } + } + } + } - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e0902971e..58d762ebb 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,6 +16,7 @@ std::map stringifiedXpFeatures = { { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, { Xp::Cgroups, "cgroups" }, + { Xp::DiscardReferences, "discard-references" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index af775feb0..ac372e03e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -25,6 +25,7 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, + DiscardReferences, }; /** diff --git a/tests/check-refs.nix b/tests/check-refs.nix index 9d90b0920..a5e40c9e5 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -67,4 +67,13 @@ rec { disallowedReferences = [test5]; }; + test11 = makeTest 11 { + __structuredAttrs = true; + outputChecks.out = { + unsafeDiscardReferences = true; + allowedReferences = []; + }; + buildCommand = ''echo ${dep} > "''${outputs[out]}"''; + }; + } diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 16bbabc40..65a72552a 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -40,3 +40,12 @@ nix-build -o $RESULT check-refs.nix -A test7 # test10 should succeed (no disallowed references). nix-build -o $RESULT check-refs.nix -A test10 + +if isDaemonNewer 2.12pre20230103; then + enableFeatures discard-references + restartDaemon + + # test11 should succeed. + test11=$(nix-build -o $RESULT check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] +fi From 3c968191f1d54a172d2b245d080c67012f766f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 3 Jan 2023 18:53:01 +0100 Subject: [PATCH 06/66] move `unsafeDiscardReferences` out of `outputChecks` It's not a check. --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libstore/build/local-derivation-goal.cc | 14 ++++++-------- tests/check-refs.nix | 6 ++---- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 51320d50a..1e8a035b0 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -26,7 +26,7 @@ to disable scanning a given output for runtime dependencies: ```nix __structuredAttrs = true; - outputChecks.out.unsafeDiscardReferences = true; + unsafeDiscardReferences.out = true; ``` This is useful e.g. when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 65340d6ce..d4ea68f93 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2321,14 +2321,12 @@ DrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { - if (auto output = get(*outputChecks, outputName)) { - if (auto unsafeDiscardReferences = get(*output, "unsafeDiscardReferences")) { - settings.requireExperimentalFeature(Xp::DiscardReferences); - if (!unsafeDiscardReferences->is_boolean()) - throw Error("attribute 'outputChecks.\"%s\".unsafeDiscardReferences' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); - discardReferences = unsafeDiscardReferences->get(); - } + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get(); } } } diff --git a/tests/check-refs.nix b/tests/check-refs.nix index a5e40c9e5..99d69a226 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -69,10 +69,8 @@ rec { test11 = makeTest 11 { __structuredAttrs = true; - outputChecks.out = { - unsafeDiscardReferences = true; - allowedReferences = []; - }; + unsafeDiscardReferences.out = true; + outputChecks.out.allowedReferences = []; buildCommand = ''echo ${dep} > "''${outputs[out]}"''; }; From 763c1dfc2b0875251385d9b228c0fc1bfd654bf4 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 16 Jan 2023 11:35:12 -0800 Subject: [PATCH 07/66] Expand installation.md Changes the `quick-start.md` to recommend a multi-user install, since single-user is not supported on MacOS and https://nixos.org/download.html recommends multi-user. Expands `installation.md` to reflect wording on https://nixos.org/download.html --- doc/manual/src/installation/installation.md | 40 +++++++++++++++++++-- doc/manual/src/quick-start.md | 12 +++---- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/installation.md index b40c5b95f..dafdeb667 100644 --- a/doc/manual/src/installation/installation.md +++ b/doc/manual/src/installation/installation.md @@ -1,2 +1,38 @@ -This section describes how to install and configure Nix for first-time -use. +# Installation + +This section describes how to install and configure Nix for first-time use. + +The current recommended option on Linux and MacOS is [multi-user](#multi-user). + +## Multi-user + +This installation offers better sharing, improved isolation, and more security +over a single user installation. + +This option requires either: + +* Linux running systemd, with SELinux disabled +* MacOS + +```console +$ bash <(curl -L https://nixos.org/nix/install) --daemon +``` + +## Single-user + +> Single-user is not supported on Mac. + +This installation has less requirements than the multi-user install, however it +cannot offer equivalent sharing, isolation, or security. + +This option is suitable for systems without systemd. + +```console +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon +``` + +## Distributions + +The Nix community maintains installers for several distributions. + +They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository. diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index b54e73500..e11549984 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -4,16 +4,16 @@ This chapter is for impatient people who don't like reading documentation. For more in-depth information you are kindly referred to subsequent chapters. -1. Install single-user Nix by running the following: +1. Install Nix by running the following: ```console - $ bash <(curl -L https://nixos.org/nix/install) + $ bash <(curl -L https://nixos.org/nix/install) --daemon ``` - This will install Nix in `/nix`. The install script will create - `/nix` using `sudo`, so make sure you have sufficient rights. (For - other installation methods, see - [here](installation/installation.md).) + The install script will use `sudo`, so make sure you have sufficient rights. + On Linux, `--daemon` can be omitted for a single-user install. + + For other installation methods, see [here](installation/installation.md). 1. See what installable packages are currently available in the channel: From a5919f4754be6f4a9fe091e0ee5538ad85af0050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 08/66] =?UTF-8?q?Move=20the=20default=20profiles=20to=20th?= =?UTF-8?q?e=20user=E2=80=99s=20home?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than using `/nix/var/nix/{profiles,gcroots}/per-user/`, put the user profiles and gcroots under `$XDG_DATA_DIR/nix/{profiles,gcroots}`. This means that the daemon no longer needs to manage these paths itself (they are fully handled client-side). In particular, it doesn’t have to `chown` them anymore (removing one need for root). This does change the layout of the gc-roots created by nix-env, and is likely to break some stuff, so I’m not sure how to properly handle that. --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/local-store.cc | 16 ------------- src/libstore/local-store.hh | 2 -- src/libstore/profiles.cc | 26 ++++++++++++++++----- src/libstore/profiles.hh | 4 ++++ src/libstore/store-api.hh | 3 --- src/libutil/util.cc | 18 ++++++++------ src/libutil/util.hh | 3 +++ src/nix-channel/nix-channel.cc | 4 +++- src/nix/daemon.cc | 1 - tests/remote-store.sh | 4 ---- 11 files changed, 42 insertions(+), 41 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9ab9b17fe..4fec8fb88 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1517,7 +1517,7 @@ void LocalDerivationGoal::startDaemon() try { daemon::processConnection(store, from, to, daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); + [&](Store & store) {}); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index be21e3ca0..82edaa9bf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -201,8 +201,6 @@ LocalStore::LocalStore(const Params & params) throw SysError("could not set permissions on '%s' to 755", perUserDir); } - createUser(getUserName(), getuid()); - /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { @@ -1824,20 +1822,6 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - std::optional> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 06d36a7d5..a84eb7c26 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -281,8 +281,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void createUser(const std::string & userName, uid_t userId) override; - // XXX: Make a generic `Store` method FixedOutputHash hashCAPath( const FileIngestionMethod & method, diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 3e4188188..29ce13b8d 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -280,16 +280,30 @@ std::string optimisticLockProfile(const Path & profile) } +Path profilesDir() +{ + auto profileRoot = getDataDir() + "/nix/profiles"; + createDirs(profileRoot); + return profileRoot; +} + + Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + // Migrate from the “old-style” profiles stored under `/nix/var`: + // If the link exists and points to the old location, rewrite it to the + // new one (otherwise keep-it as-it-is as it might have been + // intentionnally changed, in which case we shouldn’t touch it) + auto legacyProfile = getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); + if (!pathExists(profileLink) || + (isLink(profileLink) && + readLink(profileLink) == legacyProfile) + ) { + replaceSymlink(profilesDir() + "/profile", profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 408ca039c..73667a798 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -68,6 +68,10 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ std::string optimisticLockProfile(const Path & profile); +/* Creates and returns the path to a directory suitable for storing the user’s + profiles. */ +Path profilesDir(); + /* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create it. */ Path getDefaultProfile(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9eab4b4e5..5807392a7 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -657,9 +657,6 @@ public: return toRealPath(printStorePath(storePath)); } - virtual void createUser(const std::string & userName, uid_t userId) - { } - /* * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 993dc1cb6..40faa4bf2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -537,6 +537,16 @@ std::string getUserName() return name; } +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} Path getHome() { @@ -558,13 +568,7 @@ Path getHome() } } if (!homeDir) { - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; + homeDir = getHomeOf(geteuid()); if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9b149de80..266da0ae3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -137,6 +137,9 @@ void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); +/* Return the given user's home directory from /etc/passwd. */ +Path getHomeOf(uid_t userId); + /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index cf52b03b4..263d85eea 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,11 @@ +#include "profiles.hh" #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "util.hh" #include #include @@ -166,7 +168,7 @@ static int main_nix_channel(int argc, char ** argv) nixDefExpr = home + "/.nix-defexpr"; // Figure out the name of the channels profile. - profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); + profile = profilesDir() + "/channels"; enum { cNone, diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index c527fdb0a..19fbbf155 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -248,7 +248,6 @@ static void daemonLoop() querySetting("build-users-group", "") == "") throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); #endif - store.createUser(user, peer.uid); }); exit(0); diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 31210ab47..1ae126794 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -30,7 +30,3 @@ NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 cmp $TEST_ROOT/d1 $TEST_ROOT/d2 killDaemon - -user=$(whoami) -[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] -[ -e $NIX_STATE_DIR/profiles/per-user/$user ] From 0601050755ba39f52fdd2a0fc0478baaf9b1c7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 09/66] Migrate the old profiles to the new location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that we don’t just create the new profiles directory, but that we also migrate every existing profile to it. --- src/libstore/profiles.cc | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 29ce13b8d..d185a898c 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -293,17 +293,29 @@ Path getDefaultProfile() Path profileLink = getHome() + "/.nix-profile"; try { // Migrate from the “old-style” profiles stored under `/nix/var`: - // If the link exists and points to the old location, rewrite it to the - // new one (otherwise keep-it as-it-is as it might have been - // intentionnally changed, in which case we shouldn’t touch it) + // If the link exists and points to the old location, then: + // - Rewrite it to point to the new location + // - For every generation of the old default profile, create a symlink + // from the new directory to it (so that all the previous profiles + // and generations are still available). auto legacyProfile = getuid() == 0 ? settings.nixStateDir + "/profiles/default" : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); - if (!pathExists(profileLink) || - (isLink(profileLink) && - readLink(profileLink) == legacyProfile) - ) { - replaceSymlink(profilesDir() + "/profile", profileLink); + auto newProfile = profilesDir() + "/profile"; + if (!pathExists(profileLink) + || (isLink(profileLink) + && readLink(profileLink) == legacyProfile)) { + warn("Migrating the default profile"); + replaceSymlink(newProfile, profileLink); + replaceSymlink(legacyProfile, newProfile); + if (pathExists(legacyProfile)) { + for (auto & oldGen : findGenerations(legacyProfile).first) { + replaceSymlink( + oldGen.path, + dirOf(newProfile) + "/" + + std::string(baseNameOf(oldGen.path))); + } + } } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { From 1f02aa4098a61e60062e3ecdb713810e24efa25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 10/66] Test the migration of the user profiles --- tests/common.sh.in | 2 +- tests/local.mk | 1 + tests/user-envs-migration.sh | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/user-envs-migration.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index 73c2d2309..74bbbc8ca 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -62,7 +62,7 @@ readLink() { } clearProfiles() { - profiles="$NIX_STATE_DIR"/profiles + profiles="$HOME"/.local/share/nix/profiles rm -rf $profiles } diff --git a/tests/local.mk b/tests/local.mk index 5ac1ede32..b900e1534 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ fetchMercurial.sh \ gc-auto.sh \ user-envs.sh \ + user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ ca/build.sh \ diff --git a/tests/user-envs-migration.sh b/tests/user-envs-migration.sh new file mode 100644 index 000000000..467c28fbb --- /dev/null +++ b/tests/user-envs-migration.sh @@ -0,0 +1,35 @@ +# Test that the migration of user environments +# (https://github.com/NixOS/nix/pull/5226) does preserve everything + +source common.sh + +if isDaemonNewer "2.4pre20211005"; then + exit 99 +fi + + +killDaemon +unset NIX_REMOTE + +clearStore +clearProfiles +rm -rf ~/.nix-profile + +# Fill the environment using the older Nix +PATH_WITH_NEW_NIX="$PATH" +export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" + +nix-env -f user-envs.nix -i foo-1.0 +nix-env -f user-envs.nix -i bar-0.1 + +# Migrate to the new profile dir, and ensure that everything’s there +export PATH="$PATH_WITH_NEW_NIX" +nix-env -q # Trigger the migration +( [[ -L ~/.nix-profile ]] && \ + [[ $(readlink ~/.nix-profile) == ~/.local/share/nix/profiles/profile ]] ) || \ + fail "The nix profile should point to the new location" + +(nix-env -q | grep foo && nix-env -q | grep bar && \ + [[ -e ~/.nix-profile/bin/foo ]] && \ + [[ $(nix-env --list-generations | wc -l) == 2 ]]) || + fail "The nix profile should have the same content as before the migration" From c80621dbacc501f8bf43863438a3df087436d13e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 17 Jan 2023 14:15:00 +0100 Subject: [PATCH 11/66] Don't try to migrate existing profiles Doing so would be more dangerous than useful, better leave them as-is if they already exist --- src/libstore/profiles.cc | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index d185a898c..ff62d0972 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -292,30 +292,9 @@ Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - // Migrate from the “old-style” profiles stored under `/nix/var`: - // If the link exists and points to the old location, then: - // - Rewrite it to point to the new location - // - For every generation of the old default profile, create a symlink - // from the new directory to it (so that all the previous profiles - // and generations are still available). - auto legacyProfile = getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); - auto newProfile = profilesDir() + "/profile"; - if (!pathExists(profileLink) - || (isLink(profileLink) - && readLink(profileLink) == legacyProfile)) { - warn("Migrating the default profile"); - replaceSymlink(newProfile, profileLink); - replaceSymlink(legacyProfile, newProfile); - if (pathExists(legacyProfile)) { - for (auto & oldGen : findGenerations(legacyProfile).first) { - replaceSymlink( - oldGen.path, - dirOf(newProfile) + "/" - + std::string(baseNameOf(oldGen.path))); - } - } + auto profile = profilesDir() + "/profile"; + if (!pathExists(profileLink)) { + replaceSymlink(profile, profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { From 6bdf4edb77bea3fdc006e9b8131a43f4dcd0ee42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 17 Jan 2023 14:15:53 +0100 Subject: [PATCH 12/66] Keep the default profile the same MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's used as the “system” profile in a bunch of places, so better not touch it. Besides, it doesn't hurt to keep it since it's owned by root any way, so it doesn't have the `chown` problem that the user profiles had and that led to wanting to move them on the client-side. --- src/libstore/profiles.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ff62d0972..b202351ce 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -292,7 +292,10 @@ Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - auto profile = profilesDir() + "/profile"; + auto profile = + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : profilesDir() + "/profile"; if (!pathExists(profileLink)) { replaceSymlink(profile, profileLink); } From 3272a2d26fd24647509cddfc2267310416a41cd1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 18 Jan 2023 12:16:27 +0100 Subject: [PATCH 13/66] maintainers: Document backporting quirk Closes #7623 --- maintainers/backporting.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 maintainers/backporting.md diff --git a/maintainers/backporting.md b/maintainers/backporting.md new file mode 100644 index 000000000..fe860bc61 --- /dev/null +++ b/maintainers/backporting.md @@ -0,0 +1,8 @@ + +# Backporting + +Backports are handled by the backport action. + +Since GitHub Actions can not trigger actions, the backport PR needs to be re-triggered by another actor. This is achieved by closing and reopening the backport PR. + +This specifically affects the `installer_test` check, but note that it only runs after the other tests, so it may take a while to appear. From 4d50995effdaf1d04453293d1afa56e9c8ce6f24 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Fri, 20 Jan 2023 10:31:26 +0100 Subject: [PATCH 14/66] Fix url parsing for urls using `file+` `file+https://example.org/test.mp4` should not be rejected with `unexpected authority`. --- src/libutil/tests/url.cc | 21 +++++++++++++++++++++ src/libutil/url.cc | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index c3b233797..e0c438b4d 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -99,6 +99,27 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parsesFilePlusHttpsUrl) { + auto s = "file+https://www.example.org/video.mp4"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "file+https://www.example.org/video.mp4", + .base = "https://www.example.org/video.mp4", + .scheme = "file+https", + .authority = "www.example.org", + .path = "/video.mp4", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) { + auto s = "file://www.example.org/video.mp4"; + ASSERT_THROW(parseURL(s), Error); + } TEST(parseURL, parseIPv4Address) { auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 5b7abeb49..4e43455e1 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -30,13 +30,13 @@ ParsedURL parseURL(const std::string & url) auto & query = match[6]; auto & fragment = match[7]; - auto isFile = scheme.find("file") != std::string::npos; + auto transportIsFile = parseUrlScheme(scheme).transport == "file"; - if (authority && *authority != "" && isFile) + if (authority && *authority != "" && transportIsFile) throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); - if (isFile && path.empty()) + if (transportIsFile && path.empty()) path = "/"; return ParsedURL{ From 74026bb1014460d511534da8a955bee6948716ee Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 14:43:26 +0100 Subject: [PATCH 15/66] tests: Move NixOS tests to tests/nixos This will allow contributors to find them more easily. --- flake.nix | 14 +++++++------- tests/{ => nixos/containers}/containers.nix | 0 tests/{ => nixos/containers}/id-test.nix | 0 tests/{ => nixos/containers}/systemd-nspawn.nix | 0 tests/{ => nixos}/github-flakes.nix | 0 tests/{ => nixos}/nix-copy-closure.nix | 0 tests/{ => nixos}/nss-preload.nix | 0 tests/{ => nixos}/remote-builds.nix | 0 tests/{ => nixos}/setuid.nix | 0 tests/{ => nixos}/sourcehut-flakes.nix | 0 10 files changed, 7 insertions(+), 7 deletions(-) rename tests/{ => nixos/containers}/containers.nix (100%) rename tests/{ => nixos/containers}/id-test.nix (100%) rename tests/{ => nixos/containers}/systemd-nspawn.nix (100%) rename tests/{ => nixos}/github-flakes.nix (100%) rename tests/{ => nixos}/nix-copy-closure.nix (100%) rename tests/{ => nixos}/nss-preload.nix (100%) rename tests/{ => nixos}/remote-builds.nix (100%) rename tests/{ => nixos}/setuid.nix (100%) rename tests/{ => nixos}/sourcehut-flakes.nix (100%) diff --git a/flake.nix b/flake.nix index 573373c42..0dc8c4c2e 100644 --- a/flake.nix +++ b/flake.nix @@ -475,37 +475,37 @@ }; # System tests. - tests.remoteBuilds = import ./tests/remote-builds.nix { + tests.remoteBuilds = import ./tests/nixos/remote-builds.nix { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }; - tests.nix-copy-closure = import ./tests/nix-copy-closure.nix { + tests.nix-copy-closure = import ./tests/nixos/nix-copy-closure.nix { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }; - tests.nssPreload = (import ./tests/nss-preload.nix rec { + tests.nssPreload = (import ./tests/nixos/nss-preload.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.githubFlakes = (import ./tests/github-flakes.nix rec { + tests.githubFlakes = (import ./tests/nixos/github-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec { + tests.sourcehutFlakes = (import ./tests/nixos/sourcehut-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.containers = (import ./tests/containers.nix rec { + tests.containers = (import ./tests/nixos/containers/containers.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; @@ -514,7 +514,7 @@ tests.setuid = nixpkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] (system: - import ./tests/setuid.nix rec { + import ./tests/nixos/setuid.nix rec { inherit nixpkgs system; overlay = self.overlays.default; }); diff --git a/tests/containers.nix b/tests/nixos/containers/containers.nix similarity index 100% rename from tests/containers.nix rename to tests/nixos/containers/containers.nix diff --git a/tests/id-test.nix b/tests/nixos/containers/id-test.nix similarity index 100% rename from tests/id-test.nix rename to tests/nixos/containers/id-test.nix diff --git a/tests/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix similarity index 100% rename from tests/systemd-nspawn.nix rename to tests/nixos/containers/systemd-nspawn.nix diff --git a/tests/github-flakes.nix b/tests/nixos/github-flakes.nix similarity index 100% rename from tests/github-flakes.nix rename to tests/nixos/github-flakes.nix diff --git a/tests/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix similarity index 100% rename from tests/nix-copy-closure.nix rename to tests/nixos/nix-copy-closure.nix diff --git a/tests/nss-preload.nix b/tests/nixos/nss-preload.nix similarity index 100% rename from tests/nss-preload.nix rename to tests/nixos/nss-preload.nix diff --git a/tests/remote-builds.nix b/tests/nixos/remote-builds.nix similarity index 100% rename from tests/remote-builds.nix rename to tests/nixos/remote-builds.nix diff --git a/tests/setuid.nix b/tests/nixos/setuid.nix similarity index 100% rename from tests/setuid.nix rename to tests/nixos/setuid.nix diff --git a/tests/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix similarity index 100% rename from tests/sourcehut-flakes.nix rename to tests/nixos/sourcehut-flakes.nix From 261c25601d9a4efd5245e3ef161fb52bf0543083 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 15:32:31 +0100 Subject: [PATCH 16/66] Use the official, documented NixOS runTest interface --- flake.nix | 55 ++++++++++----------------- tests/nixos/containers/containers.nix | 11 ++---- tests/nixos/github-flakes.nix | 15 ++------ tests/nixos/nix-copy-closure.nix | 17 +++++---- tests/nixos/nss-preload.nix | 16 +++----- tests/nixos/remote-builds.nix | 14 ++----- tests/nixos/setuid.nix | 11 +++--- tests/nixos/sourcehut-flakes.nix | 14 ++----- 8 files changed, 56 insertions(+), 97 deletions(-) diff --git a/flake.nix b/flake.nix index 0dc8c4c2e..31d7f1fcf 100644 --- a/flake.nix +++ b/flake.nix @@ -401,6 +401,18 @@ }; }; + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}; + }; + _module.args.nixpkgs = nixpkgs; + }; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -475,49 +487,22 @@ }; # System tests. - tests.remoteBuilds = import ./tests/nixos/remote-builds.nix { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }; + tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; - tests.nix-copy-closure = import ./tests/nixos/nix-copy-closure.nix { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }; + tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; - tests.nssPreload = (import ./tests/nixos/nss-preload.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; - tests.githubFlakes = (import ./tests/nixos/github-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; - tests.sourcehutFlakes = (import ./tests/nixos/sourcehut-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; - tests.containers = (import ./tests/nixos/containers/containers.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; tests.setuid = nixpkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] - (system: - import ./tests/nixos/setuid.nix rec { - inherit nixpkgs system; - overlay = self.overlays.default; - }); + (system: runNixOSTestFor system ./tests/nixos/setuid.nix); + # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index a4856b2df..c8ee78a4a 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -1,12 +1,7 @@ # Test whether we can run a NixOS container inside a Nix build using systemd-nspawn. -{ nixpkgs, system, overlay }: +{ lib, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ({ +{ name = "containers"; nodes = @@ -65,4 +60,4 @@ makeTest ({ host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; -}) +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index a8b036b17..e4d347691 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -1,14 +1,9 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - +{ lib, config, nixpkgs, ... }: let + pkgs = config.nodes.client.nixpkgs.pkgs; # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate. - cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } + cert = pkgs.runCommand "cert" { nativeBuildInputs = [ pkgs.openssl ]; } '' mkdir -p $out @@ -92,8 +87,6 @@ let ''; in -makeTest ( - { name = "github-flakes"; @@ -207,4 +200,4 @@ makeTest ( client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") ''; -}) +} diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index 2dc164ae4..66cbfb033 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -1,13 +1,16 @@ # Test ‘nix-copy-closure’. -{ nixpkgs, system, overlay }: +{ lib, config, nixpkgs, hostPkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +let + pkgs = config.nodes.client.nixpkgs.pkgs; -makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pkgs.tmux; in { + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { name = "nix-copy-closure"; nodes = @@ -74,4 +77,4 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pk # ) # client.succeed("nix-store --check-validity ${pkgC}") ''; -}) +} diff --git a/tests/nixos/nss-preload.nix b/tests/nixos/nss-preload.nix index 5a6ff3f68..cef62e95b 100644 --- a/tests/nixos/nss-preload.nix +++ b/tests/nixos/nss-preload.nix @@ -1,11 +1,9 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +{ lib, config, nixpkgs, ... }: let + + pkgs = config.nodes.client.nixpkgs.pkgs; + nix-fetch = pkgs.writeText "fetch.nix" '' derivation { # This derivation is an copy from what is available over at @@ -41,9 +39,7 @@ let ''; in -makeTest ( - -rec { +{ name = "nss-preload"; nodes = { @@ -122,4 +118,4 @@ rec { nix-build ${nix-fetch} >&2 """) ''; -}) +} diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 9f88217fe..696cd2652 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -1,15 +1,9 @@ # Test Nix's remote build feature. -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ( +{ config, lib, hostPkgs, ... }: let + pkgs = config.nodes.client.nixpkgs.pkgs; # The configuration of the remote builders. builder = @@ -75,7 +69,7 @@ in # Create an SSH key on the client. subprocess.run([ - "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" ], capture_output=True, check=True) client.succeed("mkdir -p -m 700 /root/.ssh") client.copy_from_host("key", "/root/.ssh/id_ed25519") @@ -109,4 +103,4 @@ in builder1.block() client.succeed("nix-build ${expr nodes.client.config 4}") ''; -}) +} diff --git a/tests/nixos/setuid.nix b/tests/nixos/setuid.nix index 6784615e4..2b66320dd 100644 --- a/tests/nixos/setuid.nix +++ b/tests/nixos/setuid.nix @@ -1,13 +1,12 @@ # Verify that Linux builds cannot create setuid or setgid binaries. -{ nixpkgs, system, overlay }: +{ lib, config, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +let + pkgs = config.nodes.machine.nixpkgs.pkgs; -makeTest { +in +{ name = "setuid"; nodes.machine = diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index b77496ab6..a76fed020 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -1,12 +1,8 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") -{ - inherit system; - extraConfigurations = [{ nixpkgs.overlays = [ overlay ]; }]; -}; +{ lib, config, hostPkgs, nixpkgs, ... }: let + pkgs = config.nodes.sourcehut.nixpkgs.pkgs; + # Generate a fake root CA and a fake git.sr.ht certificate. cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } '' @@ -64,8 +60,6 @@ let in -makeTest ( - { name = "sourcehut-flakes"; @@ -164,4 +158,4 @@ makeTest ( client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") ''; - }) +} From 3c08a3e6b68135af126a05e9e3be417b604edded Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 15:35:46 +0100 Subject: [PATCH 17/66] PR template: Specify path to integration tests --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a268d7caf..344f9405f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -21,7 +21,7 @@ Maintainers: tick if completed or explain if not relevant - [ ] tests, as appropriate - functional tests - `tests/**.sh` - unit tests - `src/*/tests` - - integration tests + - integration tests - `tests/nixos/*` - [ ] documentation in the manual - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made From 79c084cb598221f5910c429127e8da33cff6a206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 27 Jan 2023 10:13:05 +0100 Subject: [PATCH 18/66] Add a test for `nix flake show` --- tests/flakes/common.sh | 4 ++++ tests/flakes/show.sh | 39 +++++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 44 insertions(+) create mode 100644 tests/flakes/show.sh diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index c333733c2..fdcdd4aef 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -20,6 +20,10 @@ writeSimpleFlake() { foo = import ./simple.nix; default = foo; }; + packages.someOtherSystem = rec { + foo = import ./simple.nix; + default = foo; + }; # To test "nix flake init". legacyPackages.x86_64-linux.hello = import ./simple.nix; diff --git a/tests/flakes/show.sh b/tests/flakes/show.sh new file mode 100644 index 000000000..995de8dc3 --- /dev/null +++ b/tests/flakes/show.sh @@ -0,0 +1,39 @@ +source ./common.sh + +flakeDir=$TEST_ROOT/flake +mkdir -p "$flakeDir" + +writeSimpleFlake "$flakeDir" +cd "$flakeDir" + + +# By default: Only show the packages content for the current system and no +# legacyPackages at all +nix flake show --json > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default == {}; +assert show_output.packages.${builtins.currentSystem}.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--all-systems`, show the packages for all systems +nix flake show --json --all-systems > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--legacy`, show the legacy packages +nix flake show --json --legacy > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple"; +true +' diff --git a/tests/local.mk b/tests/local.mk index 5ac1ede32..c8ee82e1a 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -113,6 +113,7 @@ nix_tests = \ store-ping.sh \ fetchClosure.sh \ completions.sh \ + flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ toString-path.sh From 1cba5984a68a489c4a56691032e4c87991c678f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 25 Jan 2023 15:10:35 +0100 Subject: [PATCH 19/66] getDefaultNixPath: actually respect `{restrict,pure}-eval` Previously, getDefaultNixPath was called too early: at initialisation time, before CLI and config have been processed, when `restrictEval` and `pureEval` both have their default value `false`. Call it when initialising the EvalState instead, and use `setDefault`. --- src/libexpr/eval.cc | 36 +++++++++++++++++++++--------------- src/libexpr/eval.hh | 4 ++-- tests/nix_path.sh | 5 +++++ tests/restricted.sh | 3 +++ 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1828b8c2e..a48968656 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -519,6 +519,7 @@ EvalState::EvalState( static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); /* Initialise the Nix expression search path. */ + evalSettings.nixPath.setDefault(evalSettings.getDefaultNixPath()); if (!evalSettings.pureEval) { for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); @@ -2472,30 +2473,35 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { EvalSettings::EvalSettings() { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); } +/* impure => NIX_PATH or a default path + * restrict-eval => NIX_PATH + * pure-eval => empty + */ Strings EvalSettings::getDefaultNixPath() { - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; + if (pureEval) + return {}; + + auto var = getEnv("NIX_PATH"); + if (var) { + return parseNixPath(*var); + } else if (restrictEval) { + return {}; + } else { + Strings res; + auto add = [&](const Path & p, const std::optional & s = std::nullopt) { + if (pathExists(p)) + res.push_back(s ? *s + "=" + p : p); + }; - if (!evalSettings.restrictEval && !evalSettings.pureEval) { add(getHome() + "/.nix-defexpr/channels"); add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); add(settings.nixStateDir + "/profiles/per-user/root/channels"); - } - return res; + return res; + } } bool EvalSettings::isPseudoUrl(std::string_view s) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e4d5906bd..876a6ae0e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -570,7 +570,7 @@ struct EvalSettings : Config { EvalSettings(); - static Strings getDefaultNixPath(); + Strings getDefaultNixPath(); static bool isPseudoUrl(std::string_view s); @@ -580,7 +580,7 @@ struct EvalSettings : Config "Whether builtin functions that allow executing native code should be enabled."}; Setting nixPath{ - this, getDefaultNixPath(), "nix-path", + this, {}, "nix-path", "List of directories to be searched for `<...>` file references."}; Setting restrictEval{ diff --git a/tests/nix_path.sh b/tests/nix_path.sh index 2b222b4a1..d16fb4bb2 100644 --- a/tests/nix_path.sh +++ b/tests/nix_path.sh @@ -12,3 +12,8 @@ nix-instantiate --eval -E '' --restrict-eval [[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] [[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +unset NIX_PATH + +[[ $(nix-instantiate --option nix-path by-relative-path=. --find-file by-relative-path/simple.nix) = "$PWD/simple.nix" ]] +[[ $(NIX_PATH= nix-instantiate --option nix-path by-relative-path=. --find-file by-relative-path/simple.nix) = "$PWD/simple.nix" ]] diff --git a/tests/restricted.sh b/tests/restricted.sh index 9bd16cf51..3b6ee2af1 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -17,6 +17,9 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh From dba9173a1d8cb1dd40e5922d009cb3a434e081c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 27 Jan 2023 15:25:07 +0100 Subject: [PATCH 20/66] Document default `nix-path` value --- src/libexpr/eval.hh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 876a6ae0e..2340ef67b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -581,7 +581,14 @@ struct EvalSettings : Config Setting nixPath{ this, {}, "nix-path", - "List of directories to be searched for `<...>` file references."}; + R"( + List of directories to be searched for `<...>` file references. + + If [pure evaluation](#conf-pure-eval) is disabled, + this is initialised using the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable, or, if it is unset and [restricted evaluation](#conf-restrict-eval) + is disabled, a default search path including the user's and `root`'s channels. + )"}; Setting restrictEval{ this, false, "restrict-eval", From 02e81cdf6276dbaaa5ecb720fb3244d5cf2c5e8e Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 27 Jan 2023 23:59:48 +0100 Subject: [PATCH 21/66] apply showAllSystems to legacyPackages as well --- src/nix/flake.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2e4d2cf3a..d2f68c37c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1121,6 +1121,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else { logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); } + } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } } else { if (visitor.isDerivation()) showDerivation(); From 6772e9ed0ac7853cefdeb6b354c81cba531829f2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 29 Jan 2023 12:31:34 -0500 Subject: [PATCH 22/66] Allow programs to have custom names Logic modeled after that for libraries. --- mk/programs.mk | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mk/programs.mk b/mk/programs.mk index 0fc1990f7..204409332 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -3,6 +3,9 @@ programs-list := # Build a program with symbolic name $(1). The program is defined by # various variables prefixed by ‘$(1)_’: # +# - $(1)_NAME: the name of the program (e.g. ‘foo’); defaults to +# $(1). +# # - $(1)_DIR: the directory where the (non-installed) program will be # placed. # @@ -23,11 +26,12 @@ programs-list := # - $(1)_INSTALL_DIR: the directory where the program will be # installed; defaults to $(bindir). define build-program + $(1)_NAME ?= $(1) _d := $(buildprefix)$$($(1)_DIR) _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - $(1)_PATH := $$(_d)/$(1) + $(1)_PATH := $$(_d)/$$($(1)_NAME) $$(eval $$(call create-dir, $$(_d))) @@ -38,7 +42,7 @@ define build-program ifdef $(1)_INSTALL_DIR - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME) $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) From ec0c0efec6d7eed868b424434748b97651e02bcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 29 Jan 2023 12:37:32 -0500 Subject: [PATCH 23/66] Allow unit test infra to be reused across libs' tests This allows using Arbitrary "instances" defined in libstore-tests in libexpr-tests, something we will leverage in a moment. --- .gitignore | 6 +++--- src/libexpr/tests/error_traces.cc | 2 +- src/libexpr/tests/json.cc | 2 +- .../tests/{libexprtests.hh => libexpr.hh} | 8 ++++---- src/libexpr/tests/local.mk | 4 +++- src/libexpr/tests/primops.cc | 2 +- src/libexpr/tests/trivial.cc | 2 +- src/libexpr/tests/value/context.cc | 2 +- .../tests/{libstoretests.hh => libstore.hh} | 0 src/libstore/tests/local.mk | 18 ++++++++++++++++-- src/libstore/tests/path.cc | 8 ++------ src/libstore/tests/path.hh | 15 +++++++++++++++ src/libutil/tests/local.mk | 2 ++ 13 files changed, 50 insertions(+), 21 deletions(-) rename src/libexpr/tests/{libexprtests.hh => libexpr.hh} (95%) rename src/libstore/tests/{libstoretests.hh => libstore.hh} (100%) create mode 100644 src/libstore/tests/path.hh diff --git a/.gitignore b/.gitignore index 8e0db013f..47815fde6 100644 --- a/.gitignore +++ b/.gitignore @@ -37,14 +37,14 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libexpr-tests +/src/libexpr/tests/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libstore-tests +/src/libstore/tests/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libutil-tests +/src/libutil/tests/libnixutil-tests /src/nix/nix diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 5e2213f69..24e95ac39 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -1,7 +1,7 @@ #include #include -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index f1ea1b197..411bc0ac3 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" #include "value-to-json.hh" namespace nix { diff --git a/src/libexpr/tests/libexprtests.hh b/src/libexpr/tests/libexpr.hh similarity index 95% rename from src/libexpr/tests/libexprtests.hh rename to src/libexpr/tests/libexpr.hh index 03e468fbb..8534d9567 100644 --- a/src/libexpr/tests/libexprtests.hh +++ b/src/libexpr/tests/libexpr.hh @@ -7,18 +7,19 @@ #include "eval-inline.hh" #include "store-api.hh" +#include "tests/libstore.hh" namespace nix { - class LibExprTest : public ::testing::Test { + class LibExprTest : public LibStoreTest { public: static void SetUpTestSuite() { - initLibStore(); + LibStoreTest::SetUpTestSuite(); initGC(); } protected: LibExprTest() - : store(openStore("dummy://")) + : LibStoreTest() , state({}, store) { } @@ -36,7 +37,6 @@ namespace nix { return state.symbols.create(value); } - ref store; EvalState state; }; diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index e483575a4..6b766d82a 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -2,6 +2,8 @@ check: libexpr-tests_RUN programs += libexpr-tests +libexpr-tests_NAME := libnixexpr-tests + libexpr-tests_DIR := $(d) libexpr-tests_INSTALL_DIR := @@ -12,6 +14,6 @@ libexpr-tests_SOURCES := \ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -libexpr-tests_LIBS = libexpr libutil libstore libfetchers +libexpr-tests_LIBS = libstore-tests libexpr libutil libstore libfetchers libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index 9cdcf64a1..e1d3ac503 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -1,7 +1,7 @@ #include #include -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { class CaptureLogger : public Logger diff --git a/src/libexpr/tests/trivial.cc b/src/libexpr/tests/trivial.cc index 8ce276e52..171727ac7 100644 --- a/src/libexpr/tests/trivial.cc +++ b/src/libexpr/tests/trivial.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { // Testing of trivial expressions diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index d5c9d3bce..0bcd79625 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -1,6 +1,6 @@ #include "value/context.hh" -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { diff --git a/src/libstore/tests/libstoretests.hh b/src/libstore/tests/libstore.hh similarity index 100% rename from src/libstore/tests/libstoretests.hh rename to src/libstore/tests/libstore.hh diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index a2cf8a0cf..c24712ccf 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -1,6 +1,20 @@ -check: libstore-tests_RUN +check: libstore-tests-exe_RUN -programs += libstore-tests +programs += libstore-tests-exe + +libstore-tests-exe_NAME = libnixstore-tests + +libstore-tests-exe_DIR := $(d) + +libstore-tests-exe_INSTALL_DIR := + +libstore-tests-exe_LIBS = libstore-tests + +libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libstore-tests + +libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 8ea252c92..2fc6d7e48 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -7,7 +7,8 @@ #include "path-regex.hh" #include "store-api.hh" -#include "libstoretests.hh" +#include "tests/libstore.hh" +#include "tests/path.hh" namespace nix { @@ -73,11 +74,6 @@ void showValue(const StorePath & p, std::ostream & os) { namespace rc { using namespace nix; -template<> -struct Arbitrary { - static Gen arbitrary(); -}; - Gen Arbitrary::arbitrary() { auto len = *gen::inRange(1, StorePath::MaxPathLen); diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh new file mode 100644 index 000000000..0ac9bb9f2 --- /dev/null +++ b/src/libstore/tests/path.hh @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include "path.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 815e18560..4f2a3f294 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -2,6 +2,8 @@ check: libutil-tests_RUN programs += libutil-tests +libutil-tests_NAME := libnixutil-tests + libutil-tests_DIR := $(d) libutil-tests_INSTALL_DIR := From ecd3e4ebd777dca694d0c60431641f054ec2d1ce Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 29 Jan 2023 13:52:38 -0500 Subject: [PATCH 24/66] More property tests Also put proper comparison methods on `DerivedPath` and `NixStringContextElem`, which is needed for the tests but good in general. --- src/libexpr/tests/local.mk | 2 +- src/libexpr/tests/value/context.cc | 42 +++++++++++++++++++++++- src/libexpr/tests/value/context.hh | 15 +++++++++ src/libexpr/value/context.hh | 17 +++++++--- src/libstore/derived-path.hh | 11 ++++--- src/libstore/tests/derived-path.cc | 52 ++++++++++++++++++++++++++++++ src/libstore/tests/derived-path.hh | 18 +++++++++++ src/libstore/tests/local.mk | 2 +- src/libstore/tests/outputs-spec.cc | 32 ++++++++++++++++++ src/libstore/tests/outputs-spec.hh | 17 ++++++++++ src/libstore/tests/path.cc | 23 ++++++++++--- src/libstore/tests/path.hh | 17 ++++++++-- src/libutil/tests/hash.cc | 22 ++++++++++++- src/libutil/tests/hash.hh | 15 +++++++++ src/libutil/tests/local.mk | 18 +++++++++-- 15 files changed, 279 insertions(+), 24 deletions(-) create mode 100644 src/libexpr/tests/value/context.hh create mode 100644 src/libstore/tests/derived-path.cc create mode 100644 src/libstore/tests/derived-path.hh create mode 100644 src/libstore/tests/outputs-spec.hh create mode 100644 src/libutil/tests/hash.hh diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 6b766d82a..3e5504f71 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -14,6 +14,6 @@ libexpr-tests_SOURCES := \ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -libexpr-tests_LIBS = libstore-tests libexpr libutil libstore libfetchers +libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0bcd79625..75934ab31 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -1,6 +1,10 @@ -#include "value/context.hh" +#include +#include +#include +#include "tests/path.hh" #include "tests/libexpr.hh" +#include "tests/value/context.hh" namespace nix { @@ -70,3 +74,39 @@ TEST_F(NixStringContextElemTest, built) { } } + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, 2)) { + case 0: + return gen::just((NixStringContextElem) NixStringContextElem::Opaque { + .path = *gen::arbitrary(), + }); + case 1: + return gen::just((NixStringContextElem) NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); + default: + return gen::just((NixStringContextElem) NixStringContextElem::Built { + .drvPath = *gen::arbitrary(), + .output = (*gen::arbitrary()).name, + }); + } +} + +} + +namespace nix { + +RC_GTEST_FIXTURE_PROP( + NixStringContextElemTest, + prop_round_rip, + (const NixStringContextElem & o)) +{ + RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store()))); +} + +} diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh new file mode 100644 index 000000000..83505962f --- /dev/null +++ b/src/libexpr/tests/value/context.hh @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index d8008c436..721563cba 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,9 +1,10 @@ #pragma once #include "util.hh" +#include "comparator.hh" #include "path.hh" -#include +#include #include @@ -31,7 +32,9 @@ class Store; Encoded as just the path: ‘’. */ struct NixStringContextElem_Opaque { - StorePath path; + StorePath path; + + GENERATE_CMP(NixStringContextElem_Opaque, me->path); }; /* Path to a derivation and its entire build closure. @@ -43,7 +46,9 @@ struct NixStringContextElem_Opaque { Encoded in the form ‘=’. */ struct NixStringContextElem_DrvDeep { - StorePath drvPath; + StorePath drvPath; + + GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); }; /* Derivation output. @@ -51,8 +56,10 @@ struct NixStringContextElem_DrvDeep { Encoded in the form ‘!!’. */ struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; + StorePath drvPath; + std::string output; + + GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); }; using _NixStringContextElem_Raw = std::variant< diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4edff7467..9e0cce377 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -4,8 +4,9 @@ #include "path.hh" #include "realisation.hh" #include "outputs-spec.hh" +#include "comparator.hh" -#include +#include #include @@ -27,8 +28,7 @@ struct DerivedPathOpaque { std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); - bool operator < (const DerivedPathOpaque & b) const - { return path < b.path; } + GENERATE_CMP(DerivedPathOpaque, me->path); }; /** @@ -51,8 +51,7 @@ struct DerivedPathBuilt { static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(ref store) const; - bool operator < (const DerivedPathBuilt & b) const - { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } + GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); }; using _DerivedPathRaw = std::variant< @@ -96,6 +95,8 @@ struct BuiltPathBuilt { nlohmann::json toJSON(ref store) const; static BuiltPathBuilt parse(const Store & store, std::string_view); + + GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); }; using _BuiltPathRaw = std::variant< diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc new file mode 100644 index 000000000..09887d1f0 --- /dev/null +++ b/src/libstore/tests/derived-path.cc @@ -0,0 +1,52 @@ +#include + +#include +#include +#include + +#include "tests/derived-path.hh" +#include "tests/libstore.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, 1)) { + case 0: + return gen::just((DerivedPath) DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); + default: + return gen::just((DerivedPath) DerivedPath::Built { + .drvPath = *gen::arbitrary(), + .outputs = *gen::arbitrary(), + }); + } +} + +} + +namespace nix { + +class DerivedPathTest : public LibStoreTest +{ +}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathTest, force_init) +{ +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); +} + +} diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh new file mode 100644 index 000000000..0161449ef --- /dev/null +++ b/src/libstore/tests/derived-path.hh @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +#include "tests/path.hh" +#include "tests/outputs-spec.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index c24712ccf..03becc7d1 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -24,6 +24,6 @@ libstore-tests_SOURCES := $(wildcard $(d)/*.cc) libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil -libstore-tests_LIBS = libstore libutil +libstore-tests_LIBS = libutil-tests libstore libutil libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc index 06e4cabbd..984d1d963 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/src/libstore/tests/outputs-spec.cc @@ -2,6 +2,7 @@ #include #include +#include namespace nix { @@ -199,3 +200,34 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON } + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, 1)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + default: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + } +} + +} + +namespace nix { + +RC_GTEST_PROP( + OutputsSpec, + prop_round_rip, + (const OutputsSpec & o)) +{ + RC_ASSERT(o == OutputsSpec::parse(o.to_string())); +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh new file mode 100644 index 000000000..2d455c817 --- /dev/null +++ b/src/libstore/tests/outputs-spec.hh @@ -0,0 +1,17 @@ +#pragma once + +#include + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 2fc6d7e48..430aa0099 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -7,6 +7,7 @@ #include "path-regex.hh" #include "store-api.hh" +#include "tests/hash.hh" #include "tests/libstore.hh" #include "tests/path.hh" @@ -74,12 +75,14 @@ void showValue(const StorePath & p, std::ostream & os) { namespace rc { using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - auto len = *gen::inRange(1, StorePath::MaxPathLen); + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - std::string pre { HASH_PART "-" }; - pre.reserve(pre.size() + len); + std::string pre; + pre.reserve(len); for (size_t c = 0; c < len; ++c) { switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { @@ -114,7 +117,17 @@ Gen Arbitrary::arbitrary() } } - return gen::just(StorePath { pre }); + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); } } // namespace rc diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh index 0ac9bb9f2..d7f1a8988 100644 --- a/src/libstore/tests/path.hh +++ b/src/libstore/tests/path.hh @@ -1,12 +1,25 @@ #pragma once -#include +#include -#include "path.hh" +#include + +namespace nix { + +struct StorePathName { + std::string name; +}; + +} namespace rc { using namespace nix; +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + template<> struct Arbitrary { static Gen arbitrary(); diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index 412c03030..e4e928b3b 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -1,5 +1,12 @@ -#include "hash.hh" +#include + +#include #include +#include + +#include + +#include "tests/hash.hh" namespace nix { @@ -73,3 +80,16 @@ namespace nix { "c7d329eeb6dd26545e96e55b874be909"); } } + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/src/libutil/tests/hash.hh new file mode 100644 index 000000000..9e9650e6e --- /dev/null +++ b/src/libutil/tests/hash.hh @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 4f2a3f294..167915439 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -2,7 +2,19 @@ check: libutil-tests_RUN programs += libutil-tests -libutil-tests_NAME := libnixutil-tests +libutil-tests-exe_NAME = libnixutil-tests + +libutil-tests-exe_DIR := $(d) + +libutil-tests-exe_INSTALL_DIR := + +libutil-tests-exe_LIBS = libutil-tests + +libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libutil-tests + +libutil-tests_NAME = libnixutil-tests libutil-tests_DIR := $(d) @@ -10,8 +22,8 @@ libutil-tests_INSTALL_DIR := libutil-tests_SOURCES := $(wildcard $(d)/*.cc) -libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr +libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil -libutil-tests_LDFLAGS := $(GTEST_LIBS) +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) From 7408776b003d8a7670c5d8eb393f22899a662b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 10:32:23 +0100 Subject: [PATCH 25/66] Fix the release-notes Slightly butchered during the merge --- doc/manual/src/release-notes/rl-next.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2f0f89cc6..7e8344e63 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,5 @@ +# Release X.Y (202?-??-??) + * A new function `builtins.readFileType` is available. It is similar to `builtins.readDir` but acts on a single file or directory. @@ -6,7 +8,7 @@ discovered by making multiple syscalls. This change makes these operations lazy such that these lookups will only be performed if the attribute is used. This optimization affects a minority of filesystems and operating systems. - + * In derivations that use structured attributes, you can now use `unsafeDiscardReferences` to disable scanning a given output for runtime dependencies: ```nix @@ -17,4 +19,4 @@ their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. - This requires the `discard-references` experimental feature. \ No newline at end of file + This requires the `discard-references` experimental feature. From bc6e65e26fd54f9c27a0344b86fc813820a7e9e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 10:44:10 +0100 Subject: [PATCH 26/66] Fix the flakes/show test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't hardcode “x86_64-linux” as this won't work too nicely on other platforms --- tests/flakes/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index fdcdd4aef..9d79080cd 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -26,7 +26,7 @@ writeSimpleFlake() { }; # To test "nix flake init". - legacyPackages.x86_64-linux.hello = import ./simple.nix; + legacyPackages.$system.hello = import ./simple.nix; }; } EOF From 3ac9f1658a482b3bf94b4a78263ea8e1ebea5b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 11:21:52 +0100 Subject: [PATCH 27/66] Fix the flakes init test Things leading to another... --- tests/flakes/init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/flakes/init.sh b/tests/flakes/init.sh index 36cb9956a..2d4c77ba1 100644 --- a/tests/flakes/init.sh +++ b/tests/flakes/init.sh @@ -41,8 +41,8 @@ cat > $templatesDir/trivial/flake.nix < Date: Mon, 30 Jan 2023 14:49:45 +0100 Subject: [PATCH 28/66] doc: add `__structuredAttrs`, `outputChecks`, `unsafeDiscardReferences` --- .../src/language/advanced-attributes.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index f02425b13..a164b3997 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -255,3 +255,67 @@ Derivations can declare some infrequently used optional attributes. > substituted. Thus it is usually a good idea to align `system` with > `builtins.currentSystem` when setting `allowSubstitutes` to > `false`. For most trivial derivations this should be the case. + + - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ + If the special attribute `__structuredAttrs` is set to `true`, the other derivation + attributes are serialised in JSON format and made available to the + builder via the file `.attrs.json` in the builder’s temporary + directory. This obviates the need for `passAsFile` since JSON files + have no size restrictions, unlike process environments. + + It also makes it possible to tweak derivation settings in a structured way; see + [`outputChecks`](#adv-attr-outputChecks) for example. + + As a convenience to Bash builders, + Nix writes a script named `.attrs.sh` to the builder’s directory + that initialises shell variables corresponding to all attributes + that are representable in Bash. This includes non-nested + (associative) arrays. For example, the attribute `hardening.format = true` + ends up as the Bash associative array element `${hardening[format]}`. + + - [`outputChecks`]{#adv-attr-outputChecks}\ + When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` + attribute allows defining checks per-output. + + In addition to + [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), + the following attributes are available: + + - `maxSize` defines the maximum size of the output path. + - `maxClosureSize` defines the maximum size of the output's closure. + - `ignoreSelfRefs` controls whether self-references should be considered when + checking for allowed references/requisites. + + ```nix + __structuredAttrs = true; + + outputChecks.out = { + # The closure of 'out' must not be larger than 256 MiB. + maxClosureSize = 256 * 1024 * 1024; + + # It must not refer to the C compiler or to the 'dev' output. + disallowedRequisites = [ stdenv.cc "dev" ]; + }; + + outputChecks.dev = { + # The 'dev' output must not be larger than 128 KiB. + maxSize = 128 * 1024; + }; + ``` + + - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ + When using [structured attributes](#adv-attr-structuredAttrs), the **experimental** + attribute `unsafeDiscardReferences` is a per-output boolean which, if set to `true`, + disables scanning the build output for runtime dependencies altogether. + + ```nix + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + ``` + + This is useful, for example, when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + + This is only allowed if the `discard-references` experimental feature is enabled. From f3e272ba02c3167b65a635389394f97a733440ca Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 09:35:25 -0500 Subject: [PATCH 29/66] Avoid some `StorePath` <-> `Path` round trips Avoid needless work and throwing away invariants. These conversions date back to when `StorePath` was in Rust and there were issues with it missing utility methods. --- src/libstore/store-api.cc | 8 ++++---- src/nix/verify.cc | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5130409d4..31bd9318f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -742,13 +742,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -760,7 +760,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index efa2434dc..0b306cc11 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -81,14 +81,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -173,7 +173,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); From 02e745ba5b3dca3b53d88a49d86f5b22d047a08d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 10:12:44 -0500 Subject: [PATCH 30/66] Separate `path.hh` from `content-address.hh` It is good to separate concerns; `StorePath` (in general) has nothing to do with `ContentAddress` anyways. This reduces the diff from #3746. --- src/libstore/path.hh | 5 ++--- src/libstore/realisation.hh | 3 +++ src/libstore/store-api.hh | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 6a8f027f9..1e5579b90 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include + #include "types.hh" namespace nix { @@ -66,8 +67,6 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; -typedef std::map> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 62561fce3..48d0283de 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,5 +1,8 @@ #pragma once +#include + +#include "hash.hh" #include "path.hh" #include #include "comparator.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5807392a7..4d8db3596 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -87,6 +87,8 @@ enum BuildMode { bmNormal, bmRepair, bmCheck }; struct BuildResult; +typedef std::map> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; From 560142fec0debefe7f983e60663dc3db4bdd79a9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 10:55:08 -0500 Subject: [PATCH 31/66] Make per-variant Arbitrary impls too This is a nice idea that @roberth requested. If we could factor our a generic `std::variant` impl as a follow-up it would be even better! --- src/libexpr/tests/value/context.cc | 35 +++++++++++++++++++++--------- src/libexpr/tests/value/context.hh | 15 +++++++++++++ src/libstore/tests/derived-path.cc | 24 ++++++++++++++------ src/libstore/tests/derived-path.hh | 10 +++++++++ 4 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 75934ab31..083359b7a 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -78,22 +78,37 @@ TEST_F(NixStringContextElemTest, built) { namespace rc { using namespace nix; +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::Built { + .drvPath = *gen::arbitrary(), + .output = (*gen::arbitrary()).name, + }); +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, 2)) { case 0: - return gen::just((NixStringContextElem) NixStringContextElem::Opaque { - .path = *gen::arbitrary(), - }); + return gen::just(*gen::arbitrary()); case 1: - return gen::just((NixStringContextElem) NixStringContextElem::DrvDeep { - .drvPath = *gen::arbitrary(), - }); + return gen::just(*gen::arbitrary()); default: - return gen::just((NixStringContextElem) NixStringContextElem::Built { - .drvPath = *gen::arbitrary(), - .output = (*gen::arbitrary()).name, - }); + return gen::just(*gen::arbitrary()); } } diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh index 83505962f..54d21760e 100644 --- a/src/libexpr/tests/value/context.hh +++ b/src/libexpr/tests/value/context.hh @@ -7,6 +7,21 @@ namespace rc { using namespace nix; +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + template<> struct Arbitrary { static Gen arbitrary(); diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 09887d1f0..d1ac2c5e7 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -10,18 +10,28 @@ namespace rc { using namespace nix; +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = *gen::arbitrary(), + .outputs = *gen::arbitrary(), + }); +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, 1)) { case 0: - return gen::just((DerivedPath) DerivedPath::Opaque { - .path = *gen::arbitrary(), - }); + return gen::just(*gen::arbitrary()); default: - return gen::just((DerivedPath) DerivedPath::Built { - .drvPath = *gen::arbitrary(), - .outputs = *gen::arbitrary(), - }); + return gen::just(*gen::arbitrary()); } } diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 0161449ef..3bc812440 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -10,6 +10,16 @@ namespace rc { using namespace nix; +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + template<> struct Arbitrary { static Gen arbitrary(); From b26562c629604b76c58b789ee520581cc5549433 Mon Sep 17 00:00:00 2001 From: dramforever Date: Mon, 23 Jan 2023 23:06:57 +0800 Subject: [PATCH 32/66] InstallableFlake: Handle missing attr in getCursors Handle the case where none of getActualAttrPaths() actually exists, in which case instead of returning an empty vector. This fixes the case where the user misspells the attribute name in nix search. Instead of getting no search results, now it shows an error with suggestions. Also remove InstallableFlake::getCursor() override since it's now equivalent to the base class version. --- src/libcmd/installables.cc | 44 +++++++++++--------------------------- src/libcmd/installables.hh | 4 ---- 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5090ea6d2..f8632a535 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -696,46 +696,28 @@ InstallableFlake::getCursors(EvalState & state) std::vector> res; - for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back(ref(*attr)); - } - - return res; -} - -ref InstallableFlake::getCursor(EvalState & state) -{ - auto lockedFlake = getLockedFlake(); - - auto cache = openEvalCache(state, lockedFlake); - auto root = cache->getRoot(); - Suggestions suggestions; - auto attrPaths = getActualAttrPaths(); for (auto & attrPath : attrPaths) { debug("trying flake output attribute '%s'", attrPath); - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) { + res.push_back(ref(*attr)); + } else { + suggestions += attr.getSuggestions(); } - - return *attrOrSuggestions; } - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(attrPaths)); + if (res.size() == 0) + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); + + return res; } std::shared_ptr InstallableFlake::getLockedFlake() const diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 3d12639b0..d489315df 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -198,10 +198,6 @@ struct InstallableFlake : InstallableValue std::vector> getCursors(EvalState & state) override; - /* Get a cursor to the first attrpath in getActualAttrPaths() that - exists, or throw an exception with suggestions if none exists. */ - ref getCursor(EvalState & state) override; - std::shared_ptr getLockedFlake() const; FlakeRef nixpkgsFlakeRef() const override; From 377d5eb38815d8290ab444c81c7826f46ba79844 Mon Sep 17 00:00:00 2001 From: dramforever Date: Mon, 30 Jan 2023 23:56:27 +0800 Subject: [PATCH 33/66] Installable::getCursors: Cleanup - Clarify doc comments, Installables::getCursors returns non-empty vector - Use vector::at in Installable::getCursor instead of checking for empty vector and throwing an exception with error message. --- src/libcmd/installables.cc | 7 +++---- src/libcmd/installables.hh | 8 ++++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f8632a535..24f458f1a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -379,10 +379,9 @@ Installable::getCursors(EvalState & state) ref Installable::getCursor(EvalState & state) { - auto cursors = getCursors(state); - if (cursors.empty()) - throw Error("cannot find flake attribute '%s'", what()); - return cursors[0]; + /* Although getCursors should return at least one element, in case it doesn't, + bound check to avoid an undefined behavior for vector[0] */ + return getCursors(state).at(0); } static StorePath getDeriver( diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index d489315df..da6a3addd 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -103,9 +103,13 @@ struct Installable return {}; } + /* Get a cursor to each value this Installable could refer to. However + if none exists, throw exception instead of returning empty vector. */ virtual std::vector> getCursors(EvalState & state); + /* Get the first and most preferred cursor this Installable could refer + to, or throw an exception if none exists. */ virtual ref getCursor(EvalState & state); @@ -193,8 +197,8 @@ struct InstallableFlake : InstallableValue std::pair toValue(EvalState & state) override; - /* Get a cursor to every attrpath in getActualAttrPaths() that - exists. */ + /* Get a cursor to every attrpath in getActualAttrPaths() + that exists. However if none exists, throw an exception. */ std::vector> getCursors(EvalState & state) override; From 6b779e4b07c42e5780feda5daf6dedc6c49e50d7 Mon Sep 17 00:00:00 2001 From: dramforever Date: Mon, 23 Jan 2023 23:07:34 +0800 Subject: [PATCH 34/66] Fix extra "." in CmdSearch::getDefaultFlakeAttrPaths No other getDefaultFlakeAttrPaths implementation has this trailing dot, and the dot can show up in error messages like: error: flake '...' does not provide attribute 'packages.x86_64-linux.', ... --- src/nix/search.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index d2a31607d..4fa1e7837 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -56,8 +56,8 @@ struct CmdSearch : InstallableCommand, MixJSON Strings getDefaultFlakeAttrPaths() override { return { - "packages." + settings.thisSystem.get() + ".", - "legacyPackages." + settings.thisSystem.get() + "." + "packages." + settings.thisSystem.get(), + "legacyPackages." + settings.thisSystem.get() }; } From e21aa43212a4326631ecb14a673b68f0b3bbc58e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 11:14:25 -0500 Subject: [PATCH 35/66] Delete dead code The references set seems to have been unused since `LegacySSHStore` references were first created in caa5793b4a74049ee37dd88eb1c5b785456ce40d. The method decls never were upstream, and accidentally added by me in 062533f7cdb74026096ca8c7d5b6e393893d59ef (probably due to `git rerere`). Sorry! This reduces the diff from #3746. --- src/libstore/legacy-ssh-store.cc | 1 - src/libstore/path-info.hh | 6 ------ 2 files changed, 7 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e1a4e13a3..2c9dd2680 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -134,7 +134,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor /* Hash will be set below. FIXME construct ValidPathInfo at end. */ auto info = std::make_shared(path, Hash::dummy); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b4b54e593..a7fcbd232 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -81,12 +81,6 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were From 16fa8c2e0e1c03c5cdf759358574a7030bb0dbf7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 12:08:57 -0500 Subject: [PATCH 36/66] Fix configure.ac rapidcheck tests - `AC_LANG_PUSH(C++)` is needed for the header check - The library check is hopeless (without lots of third-party macros I don't feel like getting) because name mangling Pkg-config would make all this easier. I previously opened https://github.com/emil-e/rapidcheck/issues/302, I should write a PR too. --- configure.ac | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 0066bc389..09b3651b9 100644 --- a/configure.ac +++ b/configure.ac @@ -276,8 +276,11 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # Look for rapidcheck. # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 +AC_LANG_PUSH(C++) AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include ]) -AC_CHECK_LIB([rapidcheck], []) +dnl No good for C++ libs with mangled symbols +dnl AC_CHECK_LIB([rapidcheck], []) +AC_LANG_POP(C++) # Look for nlohmann/json. From c5d4c50abaa5977db7dd7da2112f9ac289d4171e Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Mon, 30 Jan 2023 23:34:35 -0800 Subject: [PATCH 37/66] manual: Document that the store timestamp is now 1, not 0 Commit 14bc3ce3d6d5745717fa19b8b43b5fdd117ff757 (0.13~43) changed the timestamps in the Nix store from 0 to 1. Update the nix-store man page to match. Signed-off-by: Anders Kaseorg --- doc/manual/src/command-ref/nix-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 403cf285d..31fdd7806 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -633,7 +633,7 @@ written to standard output. A NAR archive is like a TAR or Zip archive, but it contains only the information that Nix considers important. For instance, timestamps are -elided because all files in the Nix store have their timestamp set to 0 +elided because all files in the Nix store have their timestamp set to 1 anyway. Likewise, all permissions are left out except for the execute bit, because all files in the Nix store have 444 or 555 permission. From 28648ed78413c9a681501e37473cfebe39bedb1b Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 31 Jan 2023 13:24:23 +0100 Subject: [PATCH 38/66] nix store ping: add --json flag --- src/nix/ping-store.cc | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 3c3b7bb45..0d7a75075 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -2,9 +2,11 @@ #include "shared.hh" #include "store-api.hh" +#include + using namespace nix; -struct CmdPingStore : StoreCommand +struct CmdPingStore : StoreCommand, MixJSON { std::string description() override { @@ -20,10 +22,19 @@ struct CmdPingStore : StoreCommand void run(ref store) override { - notice("Store URL: %s", store->getUri()); - store->connect(); - if (auto version = store->getVersion()) - notice("Version: %s", *version); + if (!json) { + notice("Store URL: %s", store->getUri()); + store->connect(); + if (auto version = store->getVersion()) + notice("Version: %s", *version); + } else { + nlohmann::json res; + res["url"] = store->getUri(); + store->connect(); + if (auto version = store->getVersion()) + res["version"] = *version; + logger->cout("%s", res); + } } }; From 4f4a6074e41bb635bc35ac14000b11d341536650 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 31 Jan 2023 15:10:39 +0100 Subject: [PATCH 39/66] nix store ping: try to print json if connect() fails aswell --- src/nix/ping-store.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 0d7a75075..5c44510ab 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "finally.hh" #include @@ -29,11 +30,13 @@ struct CmdPingStore : StoreCommand, MixJSON notice("Version: %s", *version); } else { nlohmann::json res; + Finally printRes([&]() { + logger->cout("%s", res); + }); res["url"] = store->getUri(); store->connect(); if (auto version = store->getVersion()) res["version"] = *version; - logger->cout("%s", res); } } }; From 4757b3f04e0a4568ab40ab86b9251945874a5662 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 31 Jan 2023 15:10:54 +0100 Subject: [PATCH 40/66] tests/store-ping: test nix store ping --json --- tests/store-ping.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/store-ping.sh b/tests/store-ping.sh index f9427cf0a..9846c7d3d 100644 --- a/tests/store-ping.sh +++ b/tests/store-ping.sh @@ -1,13 +1,17 @@ source common.sh STORE_INFO=$(nix store ping 2>&1) +STORE_INFO_JSON=$(nix store ping --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3) echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" + [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ fail "nix store ping on a non-existent store should fail" + +[[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] From 60d48eda2354dfa6d23ed3feb7b23ac25b34edcf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 31 Jan 2023 18:16:31 +0100 Subject: [PATCH 41/66] nix flake show: Ignore empty attrsets For frameworks it's important that structures are as lazy as possible to prevent infinite recursions, performance issues and errors that aren't related to the thing to evaluate. As a consequence, they have to emit more attributes than strictly (sic) necessary. However, these attributes with empty values are not useful to the user so we omit them. --- src/nix/flake.cc | 62 +++++++++++++++++++++++++++++++++++++++++++- tests/flakes/show.sh | 27 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d2f68c37c..c025bc7a6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1002,6 +1002,61 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto flake = std::make_shared(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); + std::function &attrPath, + const Symbol &attr)> hasContent; + + // For frameworks it's important that structures are as lazy as possible + // to prevent infinite recursions, performance issues and errors that + // aren't related to the thing to evaluate. As a consequence, they have + // to emit more attributes than strictly (sic) necessary. + // However, these attributes with empty values are not useful to the user + // so we omit them. + hasContent = [&]( + eval_cache::AttrCursor & visitor, + const std::vector &attrPath, + const Symbol &attr) -> bool + { + auto attrPath2(attrPath); + attrPath2.push_back(attr); + auto attrPathS = state->symbols.resolve(attrPath2); + const auto & attrName = state->symbols[attr]; + + auto visitor2 = visitor.getAttr(attrName); + + if ((attrPathS[0] == "apps" + || attrPathS[0] == "checks" + || attrPathS[0] == "devShells" + || attrPathS[0] == "legacyPackages" + || attrPathS[0] == "packages") + && (attrPathS.size() == 1 || attrPathS.size() == 2)) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + if ((attrPathS.size() == 1) + && (attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "overlays" + )) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + // If we don't recognize it, it's probably content + return true; + }; + std::function & attrPath, @@ -1027,7 +1082,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON { if (!json) logger->cout("%s", headerPrefix); - auto attrs = visitor.getAttrs(); + std::vector attrs; + for (const auto &attr : visitor.getAttrs()) { + if (hasContent(visitor, attrPath, attr)) + attrs.push_back(attr); + } + for (const auto & [i, attr] : enumerate(attrs)) { const auto & attrName = state->symbols[attr]; bool last = i + 1 == attrs.size(); diff --git a/tests/flakes/show.sh b/tests/flakes/show.sh index 995de8dc3..dd13264b9 100644 --- a/tests/flakes/show.sh +++ b/tests/flakes/show.sh @@ -37,3 +37,30 @@ in assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple"; true ' + +# Test that attributes are only reported when they have actual content +cat >flake.nix < show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output == { }; +true +' From 57a4258426260da9faf8c1ffd003edac330d50b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Feb 2023 20:27:35 +0100 Subject: [PATCH 42/66] Remove an unused capture --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 31bd9318f..601efa1cc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -744,7 +744,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); From e8ca49f6efeb54a9e1ed624f658e6790a3121ac9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Feb 2023 20:34:44 +0100 Subject: [PATCH 43/66] Fix clang compilation --- src/libexpr/flake/lockfile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a3ed90e1f..a74e68c9c 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -219,7 +219,7 @@ std::optional LockFile::isUnlocked() const visit(root); for (auto & i : nodes) { - if (i == root) continue; + if (i == ref(root)) continue; auto node = i.dynamic_pointer_cast(); if (node && !node->lockedRef.input.isLocked()) return node->lockedRef; From 32ca59649b0af8d8d77905274fe63de3a3fc0d1e Mon Sep 17 00:00:00 2001 From: Jamie Quigley Date: Wed, 1 Feb 2023 20:43:26 +0000 Subject: [PATCH 44/66] nix-shell: Colour the prompt red if the user is root This matches the nixos prompt colours - green for standard user, red for root --- src/nix-build/nix-build.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 049838bb1..da76c2ace 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -536,7 +536,9 @@ static void main_nix_build(int argc, char * * argv) "SHELL=%5%; " "BASH=%5%; " "set +e; " - R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" + + (getuid() == 0 ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" + : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " From 92edc3836904c2448387be11330a6e7c62593873 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 1 Feb 2023 20:05:56 -0500 Subject: [PATCH 45/66] Don't send plugin-files to the daemon. This is radically unsafe and the daemon has already loaded its plugins anyway. Fixes cachix/devenv#276 --- src/libstore/remote-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ff57a77ca..d1296627a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -266,6 +266,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(settings.experimentalFeatures.name); + overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; From 895c525d04901022b7e0c4586ee12774fb4f8c6f Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 2 Feb 2023 06:03:45 -0500 Subject: [PATCH 46/66] daemon: Warn on old clients passing unexpected plugin-files. The setting itself was already ignored due to exception trying to set pluginFiles. --- src/libstore/daemon.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e2a7dab35..d6621ec0b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -236,6 +236,10 @@ struct ClientSettings // the daemon, as that could cause some pretty weird stuff if (parseFeatures(tokenizeString(value)) != settings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); + } else if (name == settings.pluginFiles.name) { + if (tokenizeString(value) != settings.pluginFiles.get()) + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name From 51013da9211b4f18ca97b7194788121c92094bb1 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 28 Jan 2023 11:48:23 +0100 Subject: [PATCH 47/66] perl: run `initLibStore()` on `openStore()` Since #7478 it's mandatory that `initLibStore()` is called for store operations. However that's not the case when running `openStore()` in Perl using the perl-bindings. That breaks e.g. `hydra-eval-jobset` when built against Nix 2.13 which uses small portions of the store API. --- perl/lib/Nix/Store.xs | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 54ad1680c..de91dc28d 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -26,6 +26,7 @@ static ref store() static std::shared_ptr _store; if (!_store) { try { + initLibStore(); loadConfFile(); settings.lockCPU = false; _store = openStore(); From 479c0117840a5dc710019db006c5940b29d98dcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Feb 2023 12:02:03 -0500 Subject: [PATCH 48/66] Get rid of the `authHook` parameter on `processConnection` This is (morally) dead code. As @edolstra pointed out in https://github.com/NixOS/nix/pull/5226#discussion_r1073470813, this is no longer needed. I created this in 8d4162ff9e940ea9e2f97b07f3030a722695901a, so it is fitting that I now destroy it :). --- src/libstore/build/local-derivation-goal.cc | 3 +-- src/libstore/daemon.cc | 7 +------ src/libstore/daemon.hh | 7 +------ src/nix/daemon.cc | 11 ++--------- 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 572f71045..8ff83f748 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1516,8 +1516,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) {}); + daemon::NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d6621ec0b..d1f69fe2d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -985,8 +985,7 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - std::function authHook) + RecursiveFlag recursive) { auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; @@ -1029,10 +1028,6 @@ void processConnection( try { - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - authHook(*store); - tunnelLogger->stopWork(); to.flush(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67755d54e..8c765615c 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -13,11 +13,6 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - /* Arbitrary hook to check authorization / initialize user data / whatever - after the protocol has been negotiated. The idea is that this function - and everything it calls doesn't know about this stuff, and the - `nix-daemon` handles that instead. */ - std::function authHook); + RecursiveFlag recursive); } diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 19fbbf155..96568fb8f 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -241,14 +241,7 @@ static void daemonLoop() // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - }); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive); exit(0); }, options); @@ -301,7 +294,7 @@ static void runDaemon(bool stdio) /* Auth hook is empty because in this mode we blindly trust the standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); } } else daemonLoop(); From a47e055e090a85884413dfc7889897f4e3d831fb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Feb 2023 13:20:44 -0500 Subject: [PATCH 49/66] Move `trustedUsers` and `allowedUsers` to separate config struct These settings are not needed for libstore at all, they are just used by the nix daemon *command* for authorization on unix domain sockets. My moving them to a new configuration struct just in that file, we avoid them leaking anywhere else. Also, it is good to break up the mammoth `Settings` struct in general. Issue #5638 tracks this. The message is not changed because I do not want to regress in convenience to the user. Just saying "this connection is not trusted" doesn't tell them out to fix the issue. The ideal thing to do would be to somehow parameterize `processCommand` on how the error should be displayed, so different sorts of connections can display different information to the user based on how authentication is performed for the connection in question. This, however, is a good bit more work, so it is left for the future. This came up with me thinking about the tcp:// store (#5265). The larger project is not TCP *per se*, but the idea that it should be possible for something else to manage access control to services like the Nix Daemon, and those services simply trust or trust the incoming connection as they are told. This is a more capability-oriented way of thinking about trust than "every server implements its own auth separately" as we are used to today. Its very great that libstore itself already implements just this model, and so via this refactor I basically want to "enshrine" that so it continues to be the case. --- src/libstore/daemon.cc | 13 +++++++++++-- src/libstore/globals.hh | 34 ++-------------------------------- src/nix/daemon.cc | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d6621ec0b..591693a64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -529,7 +529,14 @@ static void performOp(TunnelLogger * logger, ref store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); } @@ -546,7 +553,9 @@ static void performOp(TunnelLogger * logger, ref store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation; see above. */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index c3ccb5e11..42981219d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -279,8 +279,8 @@ public: If the build users group is empty, builds will be performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used in - multi-user settings with untrusted users. + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. Defaults to `nixbld` when running as root, *empty* otherwise. )", @@ -696,24 +696,6 @@ public: )", {"trusted-binary-caches"}}; - Setting trustedUsers{ - this, {"root"}, "trusted-users", - R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. - - > **Warning** - > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. - )"}; - Setting ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( @@ -736,18 +718,6 @@ public: mismatch if the build isn't reproducible. )"}; - /* ?Who we trust to use the daemon in safe ways */ - Setting allowedUsers{ - this, {"*"}, "allowed-users", - R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. - - Note that trusted users are always allowed to connect. - )"}; - Setting printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 19fbbf155..b391ebed5 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -34,6 +34,43 @@ using namespace nix; using namespace nix::daemon; +struct UserSettings : Config { + + Setting trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of names of users (separated by whitespace) that have + additional rights when connecting to the Nix daemon, such as the + ability to specify additional binary caches, or to import unsigned + NARs. You can also specify groups by prefixing them with `@`; for + instance, `@wheel` means all users in the `wheel` group. The default + is `root`. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to + > giving that user root access to the system. For example, the user + > can set `sandbox-paths` and thereby obtain read access to + > directories that are otherwise inacessible to them. + )"}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list of names of users (separated by whitespace) that are allowed + to connect to the Nix daemon. As with the `trusted-users` option, + you can specify groups by prefixing them with `@`. Also, you can + allow all users by specifying `*`. The default is `*`. + + Note that trusted users are always allowed to connect. + )"}; +}; + +UserSettings userSettings; + +static GlobalConfig::Register rSettings(&userSettings); + #ifndef __linux__ #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) @@ -203,8 +240,8 @@ static void daemonLoop() struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - Strings trustedUsers = settings.trustedUsers; - Strings allowedUsers = settings.allowedUsers; + Strings trustedUsers = userSettings.trustedUsers; + Strings allowedUsers = userSettings.allowedUsers; if (matchUser(user, group, trustedUsers)) trusted = Trusted; From b8e96351f761ebc688f2f2f6a13345576c008396 Mon Sep 17 00:00:00 2001 From: Peter Becich Date: Sun, 5 Feb 2023 12:00:48 -0800 Subject: [PATCH 50/66] fix `clang11Stdenv` dev shell documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clang11StdenvPackages` does not exist ``` │ └───x86_64-linux │ ├───ccacheStdenv: development environment 'nix' │ ├───clang11Stdenv: development environment 'nix' │ ├───clangStdenv: development environment 'nix' │ ├───default: development environment 'nix' │ ├───gccStdenv: development environment 'nix' │ ├───libcxxStdenv: development environment 'nix' │ └───stdenv: development environment 'nix' ``` --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9dbafcc0a..e2046acc8 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -45,13 +45,13 @@ To get a shell with a different compilation environment (e.g. stdenv, gccStdenv, clangStdenv, clang11Stdenv, ccacheStdenv): ```console -$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages +$ nix-shell -A devShells.x86_64-linux.clang11Stdenv ``` or if you have a flake-enabled nix: ```console -$ nix develop .#clang11StdenvPackages +$ nix develop .#clang11Stdenv ``` Note: you can use `ccacheStdenv` to drastically improve rebuild From 6fdce7a9df0af7ea6266ded6c287b1dfc9d3eada Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 22:00:49 +0000 Subject: [PATCH 51/66] Bump cachix/install-nix-action from 18 to 19 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 18 to 19. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v18...v19) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dafba6d85..325579a5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 if: needs.check_secrets.outputs.cachix == 'true' @@ -58,7 +58,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - uses: cachix/cachix-action@v12 with: name: '${{ env.CACHIX_NAME }}' @@ -77,7 +77,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -102,7 +102,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 From 9670cac1ab0119628916dfd5a7df5751ee27530a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 7 Feb 2023 11:11:50 +0100 Subject: [PATCH 52/66] Backport PR title: Bot-based -> Automatic In hope to reduce friction in contributor-facing communication, use a more common word. "bot" needless jargon for "something the machine does itself". --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index ca5af260f..558cfa804 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -27,6 +27,6 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} github_workspace: ${{ github.workspace }} pull_description: |- - Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. # should be kept in sync with `uses` version: v0.0.5 From 2915db7b28ffab0aee9cfc3759f1ad0ee11b4cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 7 Feb 2023 11:38:09 +0100 Subject: [PATCH 53/66] doc: fixup 7714 --- .../src/language/advanced-attributes.md | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index a164b3997..5a63236e5 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -260,7 +260,7 @@ Derivations can declare some infrequently used optional attributes. If the special attribute `__structuredAttrs` is set to `true`, the other derivation attributes are serialised in JSON format and made available to the builder via the file `.attrs.json` in the builder’s temporary - directory. This obviates the need for `passAsFile` since JSON files + directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, unlike process environments. It also makes it possible to tweak derivation settings in a structured way; see @@ -282,11 +282,13 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the output path. + - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. + Example: + ```nix __structuredAttrs = true; @@ -305,9 +307,20 @@ Derivations can declare some infrequently used optional attributes. ``` - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ - When using [structured attributes](#adv-attr-structuredAttrs), the **experimental** - attribute `unsafeDiscardReferences` is a per-output boolean which, if set to `true`, - disables scanning the build output for runtime dependencies altogether. + > **Warning** + > This is an experimental feature. + > + > To enable it, add the following to [nix.conf](../command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = discard-references + > ``` + + When using [structured attributes](#adv-attr-structuredAttrs), the + attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. + If set to `true`, it disables scanning the output for runtime dependencies. + + Example: ```nix __structuredAttrs = true; @@ -317,5 +330,3 @@ Derivations can declare some infrequently used optional attributes. This is useful, for example, when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. - - This is only allowed if the `discard-references` experimental feature is enabled. From c20394245d602d43c7a601253bebdf0fab498b52 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Jan 2023 15:48:38 +0100 Subject: [PATCH 54/66] add "instantiate" to glossary --- doc/manual/src/glossary.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 6004df833..d0aff34e2 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -19,6 +19,13 @@ [store derivation]: #gloss-store-derivation + - [instantiate]{#gloss-instantiate}, instantiation\ + Translate a [derivation] into a [store derivation]. + + See [`nix-instantiate`](./command-ref/nix-instantiate.md). + + [instantiate]: #gloss-instantiate + - [realise]{#gloss-realise}, realisation\ Ensure a [store path] is [valid][validity]. From 0a82d2ca491507f727d44a96f07d48934862da1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:48:31 +0100 Subject: [PATCH 55/66] Ask release notes for incompatible changes, not bug fixes --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 344f9405f..db69e51db 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -25,4 +25,4 @@ Maintainers: tick if completed or explain if not relevant - [ ] documentation in the manual - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made - - [ ] new feature or bug fix: updated release notes + - [ ] new feature or incompatible change: updated release notes From fb2f7f5dcc6b37a4f39f59d9f477d3fa57d79095 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Jan 2023 17:31:27 +0100 Subject: [PATCH 56/66] Fix auto-uid-allocation in Docker containers This didn't work because sandboxing doesn't work in Docker. However, the sandboxing check is done lazily - after clone(CLONE_NEWNS) fails, we retry with sandboxing disabled. But at that point, we've already done UID allocation under the assumption that user namespaces are enabled. So let's get rid of the "goto fallback" logic and just detect early whether user / mount namespaces are enabled. This commit also gets rid of a compatibility hack for some ancient Linux kernels (<2.13). --- src/libstore/build/local-derivation-goal.cc | 82 +++++---------------- src/libutil/namespaces.cc | 63 ++++++++++++++++ src/libutil/namespaces.hh | 9 +++ 3 files changed, 91 insertions(+), 63 deletions(-) create mode 100644 src/libutil/namespaces.cc create mode 100644 src/libutil/namespaces.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8ff83f748..16955e326 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -16,6 +16,7 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "namespaces.hh" #include #include @@ -167,7 +168,8 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() { +void LocalDerivationGoal::tryLocalBuild() +{ unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { state = &DerivationGoal::tryToBuild; @@ -205,6 +207,17 @@ void LocalDerivationGoal::tryLocalBuild() { #endif } + #if __linux__ + if (useChroot) { + if (!mountNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support mount namespaces, which are required for sandboxing"); + debug("auto-disabling sandboxing because mount namespaces are not available"); + useChroot = false; + } + } + #endif + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); @@ -888,12 +901,7 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.create(); - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; + usingUserNamespace = userNamespacesSupported(); Pid helper = startProcess([&]() { @@ -920,64 +928,15 @@ void LocalDerivationGoal::startBuilder() flags |= CLONE_NEWUSER; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (child == -1) { - switch(errno) { - case EPERM: - case EINVAL: { - int errno_ = errno; - if (!userNamespacesEnabled && errno==EPERM) - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); - if (userNamespacesEnabled) { - Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; - if (pathExists(procSysKernelUnprivilegedUsernsClone) - && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); - } - } - Path procSelfNsUser = "/proc/self/ns/user"; - if (!pathExists(procSelfNsUser)) - notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); - /* Mention sandbox-fallback in the error message so the user - knows that having it disabled contributed to the - unrecoverability of this failure */ - throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); - } - default: - throw SysError("creating sandboxed builder process using clone()"); - } - } + + if (child == -1) + throw SysError("creating sandboxed builder process using clone()"); writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); }); - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -1045,9 +1004,6 @@ void LocalDerivationGoal::startBuilder() } else #endif { -#if __linux__ - fallback: -#endif pid = startProcess([&]() { runChild(); }); diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc new file mode 100644 index 000000000..0c3c3cbdd --- /dev/null +++ b/src/libutil/namespaces.cc @@ -0,0 +1,63 @@ +#include "namespaces.hh" +#include "util.hh" + +#if __linux__ + +namespace nix { + +bool userNamespacesSupported() +{ + static bool res = [&]() -> bool + { + if (!pathExists("/proc/self/ns/user")) { + notice("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + return false; + } + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + if (!pathExists(maxUserNamespaces) || + trim(readFile(maxUserNamespaces)) == "0") + { + notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/user/max_user_namespaces'"); + return false; + } + + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") + { + notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/kernel/unprivileged_userns_clone'"); + return false; + } + + Pid pid = startProcess([&]() + { + auto res = unshare(CLONE_NEWUSER); + _exit(res ? 1 : 0); + }); + + return pid.wait() == 0; + }(); + return res; +} + +bool mountNamespacesSupported() +{ + static bool res = [&]() -> bool + { + bool useUserNamespace = userNamespacesSupported(); + + Pid pid = startProcess([&]() + { + auto res = unshare(CLONE_NEWNS | (useUserNamespace ? CLONE_NEWUSER : 0)); + _exit(res ? 1 : 0); + }); + + return pid.wait() == 0; + }(); + return res; +} + +} + +#endif diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh new file mode 100644 index 000000000..4ed6cb683 --- /dev/null +++ b/src/libutil/namespaces.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +bool userNamespacesSupported(); + +bool mountNamespacesSupported(); + +} From bc1d9fd8b5a14334af1d0455e6b4d595cae959d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 15:25:56 +0100 Subject: [PATCH 57/66] Check whether we can use PID namespaces In unprivileged podman containers, /proc is not fully visible (there are other filesystems mounted on subdirectories of /proc). Therefore we can't mount a new /proc in the sandbox that matches the PID namespace of the sandbox. So this commit automatically disables sandboxing if /proc is not fully visible. --- src/libstore/build/local-derivation-goal.cc | 6 ++-- src/libutil/namespaces.cc | 37 ++++++++++++++++++--- src/libutil/namespaces.hh | 2 ++ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 16955e326..a99da0b5f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -209,10 +209,10 @@ void LocalDerivationGoal::tryLocalBuild() #if __linux__ if (useChroot) { - if (!mountNamespacesSupported()) { + if (!mountNamespacesSupported() || !pidNamespacesSupported()) { if (!settings.sandboxFallback) - throw Error("this system does not support mount namespaces, which are required for sandboxing"); - debug("auto-disabling sandboxing because mount namespaces are not available"); + throw Error("this system does not support the kernel namespaces that are required for sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); useChroot = false; } } diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 0c3c3cbdd..222f0d11b 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,5 +1,8 @@ #include "namespaces.hh" #include "util.hh" +#include "finally.hh" + +#include #if __linux__ @@ -7,10 +10,10 @@ namespace nix { bool userNamespacesSupported() { - static bool res = [&]() -> bool + static auto res = [&]() -> bool { if (!pathExists("/proc/self/ns/user")) { - notice("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y"); return false; } @@ -18,7 +21,7 @@ bool userNamespacesSupported() if (!pathExists(maxUserNamespaces) || trim(readFile(maxUserNamespaces)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/user/max_user_namespaces'"); + debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'"); return false; } @@ -26,7 +29,7 @@ bool userNamespacesSupported() if (pathExists(procSysKernelUnprivilegedUsernsClone) && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/kernel/unprivileged_userns_clone'"); + debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'"); return false; } @@ -43,7 +46,7 @@ bool userNamespacesSupported() bool mountNamespacesSupported() { - static bool res = [&]() -> bool + static auto res = [&]() -> bool { bool useUserNamespace = userNamespacesSupported(); @@ -58,6 +61,30 @@ bool mountNamespacesSupported() return res; } +bool pidNamespacesSupported() +{ + static auto res = [&]() -> bool + { + /* Check whether /proc is fully visible, i.e. there are no + filesystems mounted on top of files inside /proc. If this + is not the case, then we cannot mount a new /proc inside + the sandbox that matches the sandbox's PID namespace. + See https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */ + auto fp = fopen("/proc/mounts", "r"); + if (!fp) return false; + Finally delFP = [&]() { fclose(fp); }; + + while (auto ent = getmntent(fp)) + if (hasPrefix(std::string_view(ent->mnt_dir), "/proc/")) { + debug("PID namespaces do not work because /proc is not fully visible; disabling sandboxing"); + return false; + } + + return true; + }(); + return res; +} + } #endif diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 4ed6cb683..ad7bb559e 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -6,4 +6,6 @@ bool userNamespacesSupported(); bool mountNamespacesSupported(); +bool pidNamespacesSupported(); + } From d834de2894b5addc5a4a8c5088debd56a8517db1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 16:52:01 +0100 Subject: [PATCH 58/66] Fix macOS build --- src/libutil/namespaces.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 222f0d11b..b1cdbfe03 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,11 +1,11 @@ +#if __linux__ + #include "namespaces.hh" #include "util.hh" #include "finally.hh" #include -#if __linux__ - namespace nix { bool userNamespacesSupported() From 4e61877b5c64096fa3ea63bf5ead7e17e1ddef66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 16:52:31 +0100 Subject: [PATCH 59/66] More #ifdef --- src/libutil/namespaces.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index ad7bb559e..34e54d5ad 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -2,10 +2,14 @@ namespace nix { +#if __linux__ + bool userNamespacesSupported(); bool mountNamespacesSupported(); bool pidNamespacesSupported(); +#endif + } From c5c0617d6fe6810c35ec56d3116ef523f3f38904 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:59:46 +0100 Subject: [PATCH 60/66] Mention --no-sandbox if sandboxing is unsupported --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a99da0b5f..e1cc504f8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -211,7 +211,7 @@ void LocalDerivationGoal::tryLocalBuild() if (useChroot) { if (!mountNamespacesSupported() || !pidNamespacesSupported()) { if (!settings.sandboxFallback) - throw Error("this system does not support the kernel namespaces that are required for sandboxing"); + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); useChroot = false; } From 0a70b411e1afaa22d8b01560de908246042daf10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 23:01:39 +0100 Subject: [PATCH 61/66] Print debug message if a namespace test fails --- src/libutil/namespaces.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index b1cdbfe03..fdd52d92b 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -39,7 +39,12 @@ bool userNamespacesSupported() _exit(res ? 1 : 0); }); - return pid.wait() == 0; + bool supported = pid.wait() == 0; + + if (!supported) + debug("user namespaces do not work on this system"); + + return supported; }(); return res; } @@ -56,7 +61,12 @@ bool mountNamespacesSupported() _exit(res ? 1 : 0); }); - return pid.wait() == 0; + bool supported = pid.wait() == 0; + + if (!supported) + debug("mount namespaces do not work on this system"); + + return supported; }(); return res; } From 32db5e63494436b67d71d9393220dcafc34e5227 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Thu, 26 Jan 2023 22:50:36 +0100 Subject: [PATCH 62/66] docs: Fix broken anchor link --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 1f918bd4d..208d65839 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -116,7 +116,7 @@ The result is a string. [store path]: ../glossary.md#gloss-store-path [store]: ../glossary.md#gloss-store -[Path and string concatenation]: #path-and-string-concatenation +[String and path concatenation]: #string-and-path-concatenation ## Update @@ -133,7 +133,7 @@ If an attribute name is present in both, the attribute value from the latter is Comparison is -- [arithmetic] for [number]s +- [arithmetic] for [number]s - lexicographic for [string]s and [path]s - item-wise lexicographic for [list]s: elements at the same index in both lists are compared according to their type and skipped if they are equal. From d910dfe97892dcf26e83b6d591d46fbdfe2c0944 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 31 Jan 2023 14:20:51 +0100 Subject: [PATCH 63/66] docs: Fix formatting of || operator This is a workaround for [mdBook#2000](https://github.com/rust-lang/mdBook/issues/2000) --- doc/manual/src/language/operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 208d65839..90b325597 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -24,7 +24,7 @@ | [Equality] | *expr* `==` *expr* | none | 11 | | Inequality | *expr* `!=` *expr* | none | 11 | | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | -| Logical disjunction (`OR`) | *bool* `\|\|` *bool* | left | 13 | +| Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | none | 14 | [string]: ./values.md#type-string From 9aeaf98c4b0e99a3af45517a2a54f7715a396542 Mon Sep 17 00:00:00 2001 From: David Dunn <26876072+doubledup@users.noreply.github.com> Date: Wed, 8 Feb 2023 00:32:27 +0200 Subject: [PATCH 64/66] Make install command in documentation compatible with fish shell (#7474) Use a pipe for all install commands Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 2 +- doc/manual/src/installation/env-variables.md | 4 ++-- doc/manual/src/installation/installing-binary.md | 8 ++++---- doc/manual/src/quick-start.md | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index e2046acc8..53f49602b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -219,7 +219,7 @@ After the CI run completes, you can check the output to extract the installer UR 5. To generate an install command, plug this `install_url` and your GitHub username into this template: ```console - sh <(curl -L ) --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve ```