Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

src/libexpr/flake/lockfile.cc

@@ -234,6 +234,11 @@ bool LockFile::operator ==(const LockFile & other) const
     return toJSON() == other.toJSON();
 }
 
+bool LockFile::operator !=(const LockFile & other) const
+{
+    return !(*this == other);
+}
+
 InputPath parseInputPath(std::string_view s)
 {
     InputPath path;

src/libexpr/flake/lockfile.hh

@@ -73,6 +73,9 @@ struct LockFile
     std::optional<FlakeRef> isUnlocked() const;
 
     bool operator ==(const LockFile & other) const;
+    // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2
+    // that is still the default on aarch64-linux)
+    bool operator !=(const LockFile & other) const;
 
     std::shared_ptr<Node> findInput(const InputPath & path);
 

src/libutil/config.cc

@@ -70,17 +70,10 @@ void AbstractConfig::reapplyUnknownSettings()
         set(s.first, s.second);
 }
 
-// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features.
-static bool applicable(const Config::SettingData & sd)
-{
-    return !sd.isAlias
-        && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature);
-}
-
 void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
     for (auto & opt : _settings)
-        if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden))
+        if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
@@ -154,7 +147,7 @@ nlohmann::json Config::toJSON()
 {
     auto res = nlohmann::json::object();
     for (auto & s : _settings)
-        if (applicable(s.second))
+        if (!s.second.isAlias)
             res.emplace(s.first, s.second.setting->toJSON());
     return res;
 }
@@ -163,7 +156,7 @@ std::string Config::toKeyValue()
 {
     auto res = std::string();
     for (auto & s : _settings)
-        if (applicable(s.second))
+        if (s.second.isAlias)
             res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
     return res;
 }
@@ -171,9 +164,6 @@ std::string Config::toKeyValue()
 void Config::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & s : _settings) {
-        /* We do include args for settings gated on disabled
-           experimental-features. The args themselves however will also be
-           gated on any experimental feature the underlying setting is. */
         if (!s.second.isAlias)
             s.second.setting->convertToArg(args, category);
     }

src/libutil/config.hh

@@ -371,8 +371,23 @@ extern GlobalConfig globalConfig;
 
 struct ExperimentalFeatureSettings : Config {
 
-    Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
-        "Experimental Nix features to enable."};
+    Setting<std::set<ExperimentalFeature>> experimentalFeatures{
+        this, {}, "experimental-features",
+        R"(
+          Experimental features that are enabled.
+
+          Example:
+
+          ```
+          experimental-features = nix-command flakes
+          ```
+
+          The following experimental features are available:
+
+          {{#include experimental-features-shortlist.md}}
+
+          Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md).
+        )"};
 
     /**
      * Check whether the given experimental feature is enabled.

src/libutil/experimental-features.cc

@@ -5,30 +5,218 @@
 
 namespace nix {
 
-std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
-    { Xp::CaDerivations, "ca-derivations" },
-    { Xp::ImpureDerivations, "impure-derivations" },
-    { Xp::Flakes, "flakes" },
-    { Xp::NixCommand, "nix-command" },
-    { Xp::RecursiveNix, "recursive-nix" },
-    { Xp::NoUrlLiterals, "no-url-literals" },
-    { Xp::FetchClosure, "fetch-closure" },
-    { Xp::ReplFlake, "repl-flake" },
-    { Xp::AutoAllocateUids, "auto-allocate-uids" },
-    { Xp::Cgroups, "cgroups" },
-    { Xp::DiscardReferences, "discard-references" },
-    { Xp::NixTesting, "nix-testing" },
+struct ExperimentalFeatureDetails
+{
+    ExperimentalFeature tag;
+    std::string_view name;
+    std::string_view description;
 };
 
+constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
+    {
+        .tag = Xp::CaDerivations,
+        .name = "ca-derivations",
+        .description = R"(
+            Allow derivations to be content-addressed in order to prevent
+            rebuilds when changes to the derivation do not result in changes to
+            the derivation's output. See
+            [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed)
+            for details.
+        )",
+    },
+    {
+        .tag = Xp::ImpureDerivations,
+        .name = "impure-derivations",
+        .description = R"(
+            Allow derivations to produce non-fixed outputs by setting the
+            `__impure` derivation attribute to `true`. An impure derivation can
+            have differing outputs each time it is built.
+
+            Example:
+
+            ```
+            derivation {
+              name = "impure";
+              builder = /bin/sh;
+              __impure = true; # mark this derivation as impure
+              args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ];
+              system = builtins.currentSystem;
+            }
+            ```
+
+            Each time this derivation is built, it can produce a different
+            output (as the builder outputs random bytes to `$out`).  Impure
+            derivations also have access to the network, and only fixed-output
+            or other impure derivations can rely on impure derivations. Finally,
+            an impure derivation cannot also be
+            [content-addressed](#xp-feature-ca-derivations).
+        )",
+    },
+    {
+        .tag = Xp::Flakes,
+        .name = "flakes",
+        .description = R"(
+            Enable flakes. See the manual entry for [`nix
+            flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details.
+        )",
+    },
+    {
+        .tag = Xp::NixCommand,
+        .name = "nix-command",
+        .description = R"(
+            Enable the new `nix` subcommands. See the manual on
+            [`nix`](@docroot@/command-ref/new-cli/nix.md) for details.
+        )",
+    },
+    {
+        .tag = Xp::RecursiveNix,
+        .name = "recursive-nix",
+        .description = R"(
+            Allow derivation builders to call Nix, and thus build derivations
+            recursively.
+
+            Example:
+
+            ```
+            with import <nixpkgs> {};
+
+            runCommand "foo"
+              {
+                 buildInputs = [ nix jq ];
+                 NIX_PATH = "nixpkgs=${<nixpkgs>}";
+              }
+              ''
+                hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "recursive-hello"; })')
+
+                mkdir -p $out/bin
+                ln -s $hello/bin/hello $out/bin/hello
+              ''
+            ```
+
+            An important restriction on recursive builders is disallowing
+            arbitrary substitutions. For example, running
+
+            ```
+            nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
+            ```
+
+            in the above `runCommand` script would be disallowed, as this could
+            lead to derivations with hidden dependencies or breaking
+            reproducibility by relying on the current state of the Nix store. An
+            exception would be if
+            `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were
+            already in the build inputs or built by a previous recursive Nix
+            call.
+        )",
+    },
+    {
+        .tag = Xp::NoUrlLiterals,
+        .name = "no-url-literals",
+        .description = R"(
+            Disallow unquoted URLs as part of the Nix language syntax. The Nix
+            language allows for URL literals, like so:
+
+            ```
+            $ nix repl
+            Welcome to Nix 2.15.0. Type :? for help.
+
+            nix-repl> http://foo
+            "http://foo"
+            ```
+
+            But enabling this experimental feature will cause the Nix parser to
+            throw an error when encountering a URL literal:
+
+            ```
+            $ nix repl --extra-experimental-features 'no-url-literals'
+            Welcome to Nix 2.15.0. Type :? for help.
+
+            nix-repl> http://foo
+            error: URL literals are disabled
+
+            at «string»:1:1:
+
+            1| http://foo
+             | ^
+
+            ```
+
+            While this is currently an experimental feature, unquoted URLs are
+            being deprecated and their usage is discouraged.
+
+            The reason is that, as opposed to path literals, URLs have no
+            special properties that distinguish them from regular strings, URLs
+            containing parameters have to be quoted anyway, and unquoted URLs
+            may confuse external tooling.
+        )",
+    },
+    {
+        .tag = Xp::FetchClosure,
+        .name = "fetch-closure",
+        .description = R"(
+            Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language.
+        )",
+    },
+    {
+        .tag = Xp::ReplFlake,
+        .name = "repl-flake",
+        .description = R"(
+            Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands.
+        )",
+    },
+    {
+        .tag = Xp::AutoAllocateUids,
+        .name = "auto-allocate-uids",
+        .description = R"(
+            Allows Nix to automatically pick UIDs for builds, rather than creating
+            `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details.
+        )",
+    },
+    {
+        .tag = Xp::Cgroups,
+        .name = "cgroups",
+        .description = R"(
+            Allows Nix to execute builds inside cgroups. See
+            the [`use-cgroups`](#conf-use-cgroups) setting for details.
+        )",
+    },
+    {
+        .tag = Xp::DiscardReferences,
+        .name = "discard-references",
+        .description = R"(
+            Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations
+            that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for
+            runtime dependencies.
+        )",
+    },
+    {
+        .tag = Xp::NixTesting,
+        .name = "nix-testing",
+        .description = R"(
+            A "permanent" experimental feature for extra features we just need
+            for testing. Not actually an "experiment" in the sense of being
+            prospective functionality for regular users.
+        )",
+    },
+}};
+
+static_assert(
+    []() constexpr {
+        for (auto [index, feature] : enumerate(xpFeatureDetails))
+            if (index != (size_t)feature.tag)
+                return false;
+        return true;
+    }(),
+    "array order does not match enum tag order");
+
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
 {
     using ReverseXpMap = std::map<std::string_view, ExperimentalFeature>;
 
-    static auto reverseXpMap = []()
-    {
+    static std::unique_ptr<ReverseXpMap> reverseXpMap = []() {
         auto reverseXpMap = std::make_unique<ReverseXpMap>();
-        for (auto & [feature, name] : stringifiedXpFeatures)
-            (*reverseXpMap)[name] = feature;
+        for (auto & xpFeature : xpFeatureDetails)
+            (*reverseXpMap)[xpFeature.name] = xpFeature.tag;
         return reverseXpMap;
     }();
 
@@ -38,20 +226,27 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str
         return std::nullopt;
 }
 
-std::string_view showExperimentalFeature(const ExperimentalFeature feature)
+std::string_view showExperimentalFeature(const ExperimentalFeature tag)
 {
-    const auto ret = get(stringifiedXpFeatures, feature);
-    assert(ret);
-    return *ret;
+    assert((size_t)tag < xpFeatureDetails.size());
+    return xpFeatureDetails[(size_t)tag].name;
+}
+
+nlohmann::json documentExperimentalFeatures() 
+{
+    StringMap res;
+    for (auto & xpFeature : xpFeatureDetails)
+        res[std::string { xpFeature.name }] =
+            trim(stripIndentation(xpFeature.description));
+    return (nlohmann::json) res;
 }
 
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
 {
     std::set<ExperimentalFeature> res;
-    for (auto & rawFeature : rawFeatures) {
+    for (auto & rawFeature : rawFeatures)
         if (auto feature = parseExperimentalFeature(rawFeature))
             res.insert(*feature);
-    }
     return res;
 }
 

src/libutil/experimental-features.hh

@@ -11,8 +11,9 @@ namespace nix {
 /**
  * The list of available experimental features.
  *
- * If you update this, don’t forget to also change the map defining their
- * string representation in the corresponding `.cc` file.
+ * If you update this, don’t forget to also change the map defining
+ * their string representation and documentation in the corresponding
+ * `.cc` file as well.
  */
 enum struct ExperimentalFeature
 {
@@ -27,11 +28,6 @@ enum struct ExperimentalFeature
     AutoAllocateUids,
     Cgroups,
     DiscardReferences,
-
-    /**
-     * A "permanent" experimental feature for extra features we just
-     * need for testing.
-     **/
     NixTesting,
 };
 
@@ -40,26 +36,52 @@ enum struct ExperimentalFeature
  */
 using Xp = ExperimentalFeature;
 
+/**
+ * Parse an experimental feature (enum value) from its name. Experimental
+ * feature flag names are hyphenated and do not contain spaces.
+ */
 const std::optional<ExperimentalFeature> parseExperimentalFeature(
         const std::string_view & name);
+
+/**
+ * Show the name of an experimental feature. This is the opposite of
+ * parseExperimentalFeature().
+ */
 std::string_view showExperimentalFeature(const ExperimentalFeature);
 
+/**
+ * Compute the documentation of all experimental features.
+ *
+ * See `doc/manual` for how this information is used.
+ */
+nlohmann::json documentExperimentalFeatures();
+
+/**
+ * Shorthand for `str << showExperimentalFeature(feature)`.
+ */
 std::ostream & operator<<(
         std::ostream & str,
         const ExperimentalFeature & feature);
 
 /**
- * Parse a set of strings to the corresponding set of experimental features,
- * ignoring (but warning for) any unkwown feature.
+ * Parse a set of strings to the corresponding set of experimental
+ * features, ignoring (but warning for) any unknown feature.
  */
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> &);
 
+/**
+ * An experimental feature was required for some (experimental)
+ * operation, but was not enabled.
+ */
 class MissingExperimentalFeature : public Error
 {
 public:
+    /**
+     * The experimental feature that was required but not enabled.
+     */
     ExperimentalFeature missingFeature;
 
-    MissingExperimentalFeature(ExperimentalFeature);
+    MissingExperimentalFeature(ExperimentalFeature missingFeature);
 };
 
 /**

src/libutil/util.hh

@@ -916,16 +916,16 @@ constexpr auto enumerate(T && iterable)
     {
         size_t i;
         TIter iter;
-        bool operator != (const iterator & other) const { return iter != other.iter; }
-        void operator ++ () { ++i; ++iter; }
-        auto operator * () const { return std::tie(i, *iter); }
+        constexpr bool operator != (const iterator & other) const { return iter != other.iter; }
+        constexpr void operator ++ () { ++i; ++iter; }
+        constexpr auto operator * () const { return std::tie(i, *iter); }
     };
 
     struct iterable_wrapper
     {
         T iterable;
-        auto begin() { return iterator{ 0, std::begin(iterable) }; }
-        auto end() { return iterator{ 0, std::end(iterable) }; }
+        constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; }
+        constexpr auto end() { return iterator{ 0, std::end(iterable) }; }
     };
 
     return iterable_wrapper{ std::forward<T>(iterable) };

src/nix/main.cc

@@ -375,6 +375,11 @@ void mainWrapped(int argc, char * * argv)
         return;
     }
 
+    if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") {
+        logger->cout(documentExperimentalFeatures().dump());
+        return;
+    }
+
     Finally printCompletions([&]()
     {
         if (completions) {