Merge remote-tracking branch 'origin/master' into flakes

src/libexpr/eval.cc

@@ -10,6 +10,7 @@
 #include "flake/flake.hh"
 
 #include <algorithm>
+#include <chrono>
 #include <cstring>
 #include <unistd.h>
 #include <sys/time.h>
@@ -17,7 +18,6 @@
 #include <iostream>
 #include <fstream>
 
-#include <sys/time.h>
 #include <sys/resource.h>
 
 #if HAVE_BOEHMGC
@@ -1103,9 +1103,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
     }
 }
 
-
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
+    std::optional<FunctionCallTrace> trace;
+    if (evalSettings.traceFunctionCalls) {
+        trace.emplace(pos);
+    }
+
     forceValue(fun, pos);
 
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {

src/libexpr/eval.hh

@@ -6,6 +6,7 @@
 #include "symbol-table.hh"
 #include "hash.hh"
 #include "config.hh"
+#include "function-trace.hh"
 
 #include <map>
 #include <unordered_map>
@@ -373,6 +374,9 @@ struct EvalSettings : Config
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 
+    Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
+
     Setting<std::string> flakeRegistry{this, "https://raw.githubusercontent.com/NixOS/flake-registry/master/flake-registry.json", "flake-registry",
         "Path or URI of the global flake registry."};
 };

src/libexpr/function-trace.hh

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "eval.hh"
+#include <sys/time.h>
+
+namespace nix {
+
+struct FunctionCallTrace
+{
+    const Pos & pos;
+
+    FunctionCallTrace(const Pos & pos) : pos(pos) {
+        auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+        auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+        vomit("function-trace entered %1% at %2%", pos, ns.count());
+    }
+
+    ~FunctionCallTrace() {
+        auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+        auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+        vomit("function-trace exited %1% at %2%", pos, ns.count());
+    }
+};
+}

src/libmain/shared.cc

@@ -125,6 +125,15 @@ void initNix()
     act.sa_handler = sigHandler;
     if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1");
 
+#if __APPLE__
+    /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH.
+     * Instead, add a dummy sigaction handler, and signalHandlerThread
+     * can handle the rest. */
+    struct sigaction sa;
+    sa.sa_handler = sigHandler;
+    if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH");
+#endif
+
     /* Register a SIGSEGV handler to detect stack overflows. */
     detectStackOverflow();
 

src/libstore/build.cc

@@ -2357,17 +2357,37 @@ void DerivationGoal::startBuilder()
                 flags |= CLONE_NEWNET;
 
             pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL)
+            if (child == -1 && errno == EINVAL) {
                 /* Fallback for Linux < 2.13 where CLONE_NEWPID and
                    CLONE_PARENT are not allowed together. */
-                child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this);
+                flags &= ~CLONE_NEWPID;
+                child = clone(childEntry, stack + stackSize, flags, this);
+            }
+            if (child == -1 && (errno == EPERM || errno == EINVAL)) {
+                /* Some distros patch Linux to not allow unpriveleged
+                 * user namespaces. If we get EPERM or EINVAL, try
+                 * without CLONE_NEWUSER and see if that works.
+                 */
+                flags &= ~CLONE_NEWUSER;
+                child = clone(childEntry, stack + stackSize, flags, this);
+            }
+            /* Otherwise exit with EPERM so we can handle this in the
+               parent. This is only done when sandbox-fallback is set
+               to true (the default). */
+            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
+                _exit(1);
             if (child == -1) throw SysError("cloning builder process");
 
             writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n");
             _exit(0);
         }, options);
 
-        if (helper.wait() != 0)
+        int res = helper.wait();
+        if (res != 0 && settings.sandboxFallback) {
+            useChroot = false;
+            tmpDirInSandbox = tmpDir;
+            goto fallback;
+        } else if (res != 0)
             throw Error("unable to start build process");
 
         userNamespaceSync.readSide = -1;
@@ -2398,6 +2418,7 @@ void DerivationGoal::startBuilder()
     } else
 #endif
     {
+    fallback:
         options.allowVfork = !buildUser && !drv->isBuiltin();
         pid = startProcess([&]() {
             runChild();

src/libstore/download.cc

@@ -236,8 +236,6 @@ struct CurlDownloader : public Downloader
             return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
         }
 
-        long lowSpeedTimeout = 300;
-
         void init()
         {
             if (!req) req = curl_easy_init();
@@ -297,7 +295,7 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
-            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout);
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
 
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */

src/libstore/download.hh

@@ -24,6 +24,9 @@ struct DownloadSettings : Config
     Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
         "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
 
+    Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
+        "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
+
     Setting<unsigned int> tries{this, 5, "download-attempts",
         "How often Nix will attempt to download a file before giving up."};
 };

src/libstore/gc.cc

@@ -690,9 +690,8 @@ void LocalStore::removeUnusedLinks(const GCState & state)
             throw SysError(format("statting '%1%'") % path);
 
         if (st.st_nlink != 1) {
-            unsigned long long size = st.st_blocks * 512ULL;
-            actualSize += size;
-            unsharedSize += (st.st_nlink - 1) * size;
+            actualSize += st.st_size;
+            unsharedSize += (st.st_nlink - 1) * st.st_size;
             continue;
         }
 
@@ -701,7 +700,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         if (unlink(path.c_str()) == -1)
             throw SysError(format("deleting '%1%'") % path);
 
-        state.results.bytesFreed += st.st_blocks * 512ULL;
+        state.results.bytesFreed += st.st_size;
     }
 
     struct stat st;
@@ -920,11 +919,11 @@ void LocalStore::autoGC(bool sync)
                     promise.set_value();
                 });
 
-                printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
-
                 GCOptions options;
                 options.maxFreed = settings.maxFree - avail;
 
+                printInfo("running auto-GC to free %d bytes", options.maxFreed);
+
                 GCResults results;
 
                 collectGarbage(options, results);

src/libstore/globals.hh

@@ -209,6 +209,9 @@ public:
         "The paths to make available inside the build sandbox.",
         {"build-chroot-dirs", "build-sandbox-paths"}};
 
+    Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
+        "Whether to disable sandboxing when the kernel doesn't allow it."};
+
     Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
         "Additional paths to make available inside the build sandbox.",
         {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};

src/libstore/remote-store.cc

@@ -191,6 +191,13 @@ void RemoteStore::setOptions(Connection & conn)
     if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
         std::map<std::string, Config::SettingInfo> overrides;
         globalConfig.getSettings(overrides, true);
+        overrides.erase(settings.keepFailed.name);
+        overrides.erase(settings.keepGoing.name);
+        overrides.erase(settings.tryFallback.name);
+        overrides.erase(settings.maxBuildJobs.name);
+        overrides.erase(settings.maxSilentTime.name);
+        overrides.erase(settings.buildCores.name);
+        overrides.erase(settings.useSubstitutes.name);
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second.value;

src/libstore/store-api.cc

@@ -97,6 +97,10 @@ void checkStoreName(const string & name)
        reasons (e.g., "." and ".."). */
     if (string(name, 0, 1) == ".")
         throw Error(baseError % "it is illegal to start the name with a period");
+    /* Disallow names longer than 211 characters. ext4’s max is 256,
+       but we need extra space for the hash and .chroot extensions. */
+    if (name.length() > 211)
+        throw Error(baseError % "name must be less than 212 characters");
     for (auto & i : name)
         if (!((i >= 'A' && i <= 'Z') ||
               (i >= 'a' && i <= 'z') ||

src/libutil/serialise.hh

@@ -179,6 +179,36 @@ struct TeeSource : Source
     }
 };
 
+/* A reader that consumes the original Source until 'size'. */
+struct SizedSource : Source
+{
+    Source & orig;
+    size_t remain;
+    SizedSource(Source & orig, size_t size)
+        : orig(orig), remain(size) { }
+    size_t read(unsigned char * data, size_t len)
+    {
+        if (this->remain <= 0) {
+            throw EndOfFile("sized: unexpected end-of-file");
+        }
+        len = std::min(len, this->remain);
+        size_t n = this->orig.read(data, len);
+        this->remain -= n;
+        return n;
+    }
+
+    /* Consume the original source until no remain data is left to consume. */
+    size_t drainAll()
+    {
+        std::vector<unsigned char> buf(8192);
+        size_t sum = 0;
+        while (this->remain > 0) {
+            size_t n = read(buf.data(), buf.size());
+            sum += n;
+        }
+        return sum;
+    }
+};
 
 /* Convert a function into a sink. */
 struct LambdaSink : Sink

src/libutil/util.cc

@@ -397,7 +397,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
     }
 
     if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
-        bytesFreed += st.st_blocks * 512;
+        bytesFreed += st.st_size;
 
     if (S_ISDIR(st.st_mode)) {
         /* Make the directory accessible. */

src/nix-store/nix-store.cc

@@ -950,8 +950,16 @@ static void opServe(Strings opFlags, Strings opArgs)
                 info.sigs = readStrings<StringSet>(in);
                 in >> info.ca;
 
-                // FIXME: race if addToStore doesn't read source?
-                store->addToStore(info, in, NoRepair, NoCheckSigs);
+                if (info.narSize == 0) {
+                    throw Error("narInfo is too old and missing the narSize field");
+                }
+
+                SizedSource sizedSource(in, info.narSize);
+
+                store->addToStore(info, sizedSource, NoRepair, NoCheckSigs);
+
+                // consume all the data that has been sent before continuing.
+                sizedSource.drainAll();
 
                 out << 1; // indicate success
 

src/nix/run.cc

@@ -255,7 +255,10 @@ void chrootHelper(int argc, char * * argv)
     uid_t gid = getgid();
 
     if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1)
-        throw SysError("setting up a private mount namespace");
+        /* Try with just CLONE_NEWNS in case user namespaces are
+           specifically disabled. */
+        if (unshare(CLONE_NEWNS) == -1)
+            throw SysError("setting up a private mount namespace");
 
     /* Bind-mount realStoreDir on /nix/store. If the latter mount
        point doesn't already exists, we have to create a chroot