Merge remote-tracking branch 'nixos/master' into readFile-scan-references

src/build-remote/build-remote.cc

@@ -186,12 +186,12 @@ static int main_build_remote(int argc, char * * argv)
                         // build the hint template.
                         std::string errorText =
                             "Failed to find a machine for remote build!\n"
-                            "derivation: %s\nrequired (system, features): (%s, %s)";
+                            "derivation: %s\nrequired (system, features): (%s, [%s])";
                         errorText += "\n%s available machines:";
                         errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
 
                         for (unsigned int i = 0; i < machines.size(); ++i)
-                            errorText += "\n(%s, %s, %s, %s)";
+                            errorText += "\n([%s], %s, [%s], [%s])";
 
                         // add the template values.
                         std::string drvstr;

src/libcmd/command.cc

@@ -88,7 +88,8 @@ EvalCommand::EvalCommand()
 {
     addFlag({
         .longName = "debugger",
-        .description = "start an interactive environment if evaluation fails",
+        .description = "Start an interactive environment if evaluation fails.",
+        .category = MixEvalArgs::category,
         .handler = {&startReplOnEvalErrors, true},
     });
 }
@@ -225,7 +226,7 @@ MixProfile::MixProfile()
 {
     addFlag({
         .longName = "profile",
-        .description = "The profile to update.",
+        .description = "The profile to operate on.",
         .labels = {"path"},
         .handler = {&profile},
         .completer = completePath

src/libcmd/common-eval-args.cc

@@ -13,8 +13,6 @@ namespace nix {
 
 MixEvalArgs::MixEvalArgs()
 {
-    auto category = "Common evaluation options";
-
     addFlag({
         .longName = "arg",
         .description = "Pass the value *expr* as the argument *name* to Nix functions.",

src/libcmd/common-eval-args.hh

@@ -10,6 +10,8 @@ class Bindings;
 
 struct MixEvalArgs : virtual Args
 {
+    static constexpr auto category = "Common evaluation options";
+
     MixEvalArgs();
 
     Bindings * getAutoArgs(EvalState & state);

src/libcmd/repl.cc

@@ -1050,7 +1050,7 @@ struct CmdRepl : InstallablesCommand
         evalSettings.pureEval = false;
     }
 
-    void prepare()
+    void prepare() override
     {
         if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) {
             warn("future versions of Nix will require using `--file` to load a file");

src/libexpr/eval.cc

@@ -904,7 +904,7 @@ void EvalState::throwEvalError(const char * s, const std::string & s2,
     const std::string & s3)
 {
     debugThrowLastTrace(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[noPos]
     }));
 }
@@ -913,7 +913,7 @@ void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::stri
     const std::string & s3)
 {
     debugThrowLastTrace(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[pos]
     }));
 }
@@ -922,7 +922,7 @@ void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::stri
     const std::string & s3, Env & env, Expr & expr)
 {
     debugThrow(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[pos]
     }), env, expr);
 }

src/libexpr/flake/call-flake.nix

@@ -43,7 +43,7 @@ let
 
           outputs = flake.outputs (inputs // { self = result; });
 
-          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
+          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; };
         in
           if node.flake or true then
             assert builtins.isFunction flake.outputs;

src/libexpr/primops.cc

@@ -3826,8 +3826,8 @@ static RegisterPrimOp primop_parseDrvName({
     .args = {"s"},
     .doc = R"(
       Split the string *s* into a package name and version. The package
-      name is everything up to but not including the first dash followed
-      by a digit, and the version is everything following that dash. The
+      name is everything up to but not including the first dash not followed
+      by a letter, and the version is everything following that dash. The
       result is returned in a set `{ name, version }`. Thus,
       `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name =
       "nix"; version = "0.12pre12876"; }`.

src/libfetchers/git.cc

@@ -485,6 +485,10 @@ struct GitInputScheme : InputScheme
                 }
                 input.attrs.insert_or_assign("ref", *head);
                 unlockedAttrs.insert_or_assign("ref", *head);
+            } else {
+                if (!input.getRev()) {
+                    unlockedAttrs.insert_or_assign("ref", input.getRef().value());
+                }
             }
 
             if (auto res = getCache()->lookup(store, unlockedAttrs)) {

src/libfetchers/github.cc

@@ -262,17 +262,20 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
     DownloadUrl getDownloadUrl(const Input & input) const override
     {
-        // FIXME: use regular /archive URLs instead? api.github.com
-        // might have stricter rate limits.
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        auto url = fmt(
-            host == "github.com"
-            ? "https://api.%s/repos/%s/%s/tarball/%s"
-            : "https://%s/api/v3/repos/%s/%s/tarball/%s",
-            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+        Headers headers = makeHeadersWithAuthTokens(host);
+        // If we have no auth headers then we default to the public archive
+        // urls so we do not run into rate limits.
+        const auto urlFmt =
+            host != "github.com"
+            ? "https://%s/api/v3/repos/%s/%s/tarball/%s"
+            : headers.empty()
+            ? "https://%s/%s/%s/archive/%s.tar.gz"
+            : "https://api.%s/repos/%s/%s/tarball/%s";
+
+        const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(host);
         return DownloadUrl { url, headers };
     }
 

src/libmain/common-args.cc

@@ -32,6 +32,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
     addFlag({
         .longName = "option",
         .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
+        .category = miscCategory,
         .labels = {"name", "value"},
         .handler = {[](std::string name, std::string value) {
             try {

src/libmain/common-args.hh

@@ -6,6 +6,7 @@ namespace nix {
 
 //static constexpr auto commonArgsCategory = "Miscellaneous common options";
 static constexpr auto loggingCategory = "Logging-related options";
+static constexpr auto miscCategory = "Miscellaneous global options";
 
 class MixCommonArgs : public virtual Args
 {

src/libmain/progress-bar.cc

@@ -503,7 +503,7 @@ public:
         return s[0];
     }
 
-    virtual void setPrintBuildLogs(bool printBuildLogs)
+    void setPrintBuildLogs(bool printBuildLogs) override
     {
         this->printBuildLogs = printBuildLogs;
     }

src/libmain/shared.cc

@@ -33,6 +33,7 @@
 
 namespace nix {
 
+char * * savedArgv;
 
 static bool gcWarning = true;
 

src/libmain/shared.hh

@@ -113,5 +113,25 @@ struct PrintFreed
 /* Install a SIGSEGV handler to detect stack overflows. */
 void detectStackOverflow();
 
+/* Pluggable behavior to run in case of a stack overflow.
+
+   Default value: defaultStackOverflowHandler.
+
+   This is called by the handler installed by detectStackOverflow().
+
+   This gives Nix library consumers a limit opportunity to report the error
+   condition. The handler should exit the process.
+   See defaultStackOverflowHandler() for a reference implementation.
+
+   NOTE: Use with diligence, because this runs in the signal handler, with very
+   limited stack space and a potentially a corrupted heap, all while the failed
+   thread is blocked indefinitely. All functions called must be reentrant. */
+extern std::function<void(siginfo_t * info, void * ctx)> stackOverflowHandler;
+
+/* The default, robust implementation of stackOverflowHandler.
+
+   Prints an error message directly to stderr using a syscall instead of the
+   logger. Exits the process immediately after. */
+void defaultStackOverflowHandler(siginfo_t * info, void * ctx);
 
 }

src/libmain/stack.cc

@@ -1,4 +1,5 @@
 #include "error.hh"
+#include "shared.hh"
 
 #include <cstring>
 #include <cstddef>
@@ -29,9 +30,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
         ptrdiff_t diff = (char *) info->si_addr - sp;
         if (diff < 0) diff = -diff;
         if (diff < 4096) {
-            char msg[] = "error: stack overflow (possible infinite recursion)\n";
-            [[gnu::unused]] auto res = write(2, msg, strlen(msg));
-            _exit(1); // maybe abort instead?
+            nix::stackOverflowHandler(info, ctx);
         }
     }
 
@@ -67,5 +66,12 @@ void detectStackOverflow()
 #endif
 }
 
+std::function<void(siginfo_t * info, void * ctx)> stackOverflowHandler(defaultStackOverflowHandler);
+
+void defaultStackOverflowHandler(siginfo_t * info, void * ctx) {
+    char msg[] = "error: stack overflow (possible infinite recursion)\n";
+    [[gnu::unused]] auto res = write(2, msg, strlen(msg));
+    _exit(1); // maybe abort instead?
+}
 
 }

src/libstore/binary-cache-store.cc

@@ -331,6 +331,17 @@ bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
     return fileExists(narInfoFileFor(storePath));
 }
 
+std::optional<StorePath> BinaryCacheStore::queryPathFromHashPart(const std::string & hashPart)
+{
+    auto pseudoPath = StorePath(hashPart + "-" + MissingName);
+    try {
+        auto info = queryPathInfo(pseudoPath);
+        return info->path;
+    } catch (InvalidPath &) {
+        return std::nullopt;
+    }
+}
+
 void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();

src/libstore/binary-cache-store.hh

@@ -95,8 +95,7 @@ public:
     void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { unsupported("queryPathFromHashPart"); }
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs) override;

src/libstore/build/hook-instance.cc

@@ -16,11 +16,11 @@ HookInstance::HookInstance()
     buildHookArgs.pop_front();
 
     Strings args;
+    args.push_back(std::string(baseNameOf(buildHook)));
 
     for (auto & arg : buildHookArgs)
         args.push_back(arg);
 
-    args.push_back(std::string(baseNameOf(settings.buildHook.get())));
     args.push_back(std::to_string(verbosity));
 
     /* Create a pipe to get the output of the child. */

src/libstore/build/local-derivation-goal.cc

@@ -1594,6 +1594,8 @@ void LocalDerivationGoal::runChild()
     /* Warning: in the child we should absolutely not make any SQLite
        calls! */
 
+    bool sendException = true;
+
     try { /* child */
 
         commonChildInit(builderOut);
@@ -2050,6 +2052,8 @@ void LocalDerivationGoal::runChild()
         /* Indicate that we managed to set up the build environment. */
         writeFull(STDERR_FILENO, std::string("\2\n"));
 
+        sendException = false;
+
         /* Execute the program.  This should not return. */
         if (drv->isBuiltin()) {
             try {
@@ -2103,10 +2107,13 @@ void LocalDerivationGoal::runChild()
         throw SysError("executing '%1%'", drv->builder);
 
     } catch (Error & e) {
-        writeFull(STDERR_FILENO, "\1\n");
-        FdSink sink(STDERR_FILENO);
-        sink << e;
-        sink.flush();
+        if (sendException) {
+            writeFull(STDERR_FILENO, "\1\n");
+            FdSink sink(STDERR_FILENO);
+            sink << e;
+            sink.flush();
+        } else
+            std::cerr << e.msg();
         _exit(1);
     }
 }

src/libstore/globals.cc

@@ -154,13 +154,9 @@ StringSet Settings::getDefaultExtraPlatforms()
     // machines. Note that we can’t force processes from executing
     // x86_64 in aarch64 environments or vice versa since they can
     // always exec with their own binary preferences.
-    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") ||
-        pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
-        if (std::string{SYSTEM} == "x86_64-darwin")
-            extraPlatforms.insert("aarch64-darwin");
-        else if (std::string{SYSTEM} == "aarch64-darwin")
-            extraPlatforms.insert("x86_64-darwin");
-    }
+    if (std::string{SYSTEM} == "aarch64-darwin" &&
+        runProgram(RunOptions {.program = "arch", .args = {"-arch", "x86_64", "/usr/bin/true"}, .mergeStderrToStdout = true}).first == 0)
+        extraPlatforms.insert("x86_64-darwin");
 #endif
 
     return extraPlatforms;

src/libstore/ssh.cc

@@ -67,7 +67,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         if (fakeSSH) {
             args = { "bash", "-c" };
         } else {
-            args = { "ssh", host.c_str(), "-x", "-a" };
+            args = { "ssh", host.c_str(), "-x" };
             addCommonSSHOpts(args);
             if (socketPath != "")
                 args.insert(args.end(), {"-S", socketPath});

src/libstore/store-api.cc

@@ -1363,9 +1363,9 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
                 } catch (Error & e) {
                     return std::make_shared<LocalStore>(params);
                 }
-                warn("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore);
+                warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
             } else
-                debug("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore);
+                debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
             Store::Params params2;
             params2["root"] = chrootStore;
             return std::make_shared<LocalStore>(params2);

src/libutil/tarfile.cc

@@ -77,9 +77,7 @@ TarArchive::~TarArchive()
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
-    int flags = ARCHIVE_EXTRACT_FFLAGS
-        | ARCHIVE_EXTRACT_PERM
-        | ARCHIVE_EXTRACT_TIME
+    int flags = ARCHIVE_EXTRACT_TIME
         | ARCHIVE_EXTRACT_SECURE_SYMLINKS
         | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
 
@@ -98,6 +96,10 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         archive_entry_copy_pathname(entry,
             (destDir + "/" + name).c_str());
 
+        // sources can and do contain dirs with no rx bits
+        if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500)
+            archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500);
+
         // Patch hardlink path
         const char *original_hardlink = archive_entry_hardlink(entry);
         if (original_hardlink) {

src/nix/app.cc

@@ -66,7 +66,9 @@ UnresolvedApp Installable::toApp(EvalState & state)
 
     auto type = cursor->getAttr("type")->getString();
 
-    std::string expected = !attrPath.empty() && state.symbols[attrPath[0]] == "apps" ? "app" : "derivation";
+    std::string expected = !attrPath.empty() &&
+        (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp")
+        ? "app" : "derivation";
     if (type != expected)
         throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected);
 

src/nix/daemon.md

@@ -11,7 +11,7 @@ R""(
 # Description
 
 This command runs the Nix daemon, which is a required component in
-multi-user Nix installations. It performs build actions and other
+multi-user Nix installations. It runs build tasks and other
 operations on the Nix store on behalf of non-root users. Usually you
 don't run the daemon directly; instead it's managed by a service
 management framework such as `systemd`.

src/nix/main.cc

@@ -53,7 +53,6 @@ static bool haveInternet()
 }
 
 std::string programPath;
-char * * savedArgv;
 
 struct HelpRequested { };
 
@@ -74,6 +73,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         addFlag({
             .longName = "help",
             .description = "Show usage information.",
+            .category = miscCategory,
             .handler = {[&]() { throw HelpRequested(); }},
         });
 
@@ -88,6 +88,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         addFlag({
             .longName = "version",
             .description = "Show version information.",
+            .category = miscCategory,
             .handler = {[&]() { showVersion = true; }},
         });
 
@@ -95,12 +96,14 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .longName = "offline",
             .aliases = {"no-net"}, // FIXME: remove
             .description = "Disable substituters and consider all previously downloaded files up-to-date.",
+            .category = miscCategory,
             .handler = {[&]() { useNet = false; }},
         });
 
         addFlag({
             .longName = "refresh",
             .description = "Consider all previously downloaded files out-of-date.",
+            .category = miscCategory,
             .handler = {[&]() { refresh = true; }},
         });
     }
@@ -187,7 +190,7 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
         *vUtils);
 
     auto attrs = state.buildBindings(16);
-    attrs.alloc("command").mkString(toplevel.toJSON().dump());
+    attrs.alloc("toplevel").mkString(toplevel.toJSON().dump());
 
     auto vRes = state.allocValue();
     state.callFunction(*vGenerateManpage, state.allocValue()->mkAttrs(attrs), *vRes, noPos);
@@ -266,7 +269,7 @@ void mainWrapped(int argc, char * * argv)
     programPath = argv[0];
     auto programName = std::string(baseNameOf(programPath));
 
-    if (argc > 0 && std::string_view(argv[0]) == "__build-remote") {
+    if (argc > 1 && std::string_view(argv[1]) == "__build-remote") {
         programName = "build-remote";
         argv++; argc--;
     }

src/nix/path-from-hash-part.cc

@@ -0,0 +1,39 @@
+#include "command.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdPathFromHashPart : StoreCommand
+{
+    std::string hashPart;
+
+    CmdPathFromHashPart()
+    {
+        expectArgs({
+            .label = "hash-part",
+            .handler = {&hashPart},
+        });
+    }
+
+    std::string description() override
+    {
+        return "get a store path from its hash part";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "path-from-hash-part.md"
+          ;
+    }
+
+    void run(ref<Store> store) override
+    {
+        if (auto storePath = store->queryPathFromHashPart(hashPart))
+            logger->cout(store->printStorePath(*storePath));
+        else
+            throw Error("there is no store path corresponding to '%s'", hashPart);
+    }
+};
+
+static auto rCmdPathFromHashPart = registerCommand2<CmdPathFromHashPart>({"store", "path-from-hash-part"});

src/nix/path-from-hash-part.md

@@ -0,0 +1,20 @@
+R""(
+
+# Examples
+
+* Return the full store path with the given hash part:
+
+  ```console
+  # nix store path-from-hash-part --store https://cache.nixos.org/ 0i2jd68mp5g6h2sa5k9c85rb80sn8hi9
+  /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
+  ```
+
+# Description
+
+Given the hash part of a store path (that is, the 32 characters
+following `/nix/store/`), return the full store path. This is
+primarily useful in the implementation of binary caches, where a
+request for a `.narinfo` file only supplies the hash part
+(e.g. `https://cache.nixos.org/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9.narinfo`).
+
+)""

src/nix/repl.md

@@ -36,7 +36,7 @@ R""(
   Loading Installable ''...
   Added 1 variables.
 
-  # nix repl --extra_experimental_features 'flakes repl-flake' nixpkgs
+  # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs
   Loading Installable 'flake:nixpkgs#'...
   Added 5 variables.