Merge remote-tracking branch 'upstream/master' into overlayfs-store

src/libutil/args.cc

@@ -285,7 +285,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
 
                 std::string line;
                 std::getline(stream,line);
-                static const std::string commentChars("#/\\%@*-");
+                static const std::string commentChars("#/\\%@*-(");
                 std::string shebangContent;
                 while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){
                     line = chomp(line);

src/libutil/compression.cc

@@ -12,8 +12,6 @@
 #include <brotli/decode.h>
 #include <brotli/encode.h>
 
-#include <iostream>
-
 namespace nix {
 
 static const int COMPRESSION_LEVEL_DEFAULT = -1;
@@ -40,20 +38,26 @@ struct ArchiveDecompressionSource : Source
 {
     std::unique_ptr<TarArchive> archive = 0;
     Source & src;
-    ArchiveDecompressionSource(Source & src) : src(src) {}
+    std::optional<std::string> compressionMethod;
+    ArchiveDecompressionSource(Source & src, std::optional<std::string> compressionMethod = std::nullopt)
+        : src(src)
+        , compressionMethod(std::move(compressionMethod))
+    {
+    }
     ~ArchiveDecompressionSource() override {}
-    size_t read(char * data, size_t len) override {
+    size_t read(char * data, size_t len) override
+    {
         struct archive_entry * ae;
         if (!archive) {
-            archive = std::make_unique<TarArchive>(src, true);
-            this->archive->check(archive_read_next_header(this->archive->archive, &ae),
-                "failed to read header (%s)");
+            archive = std::make_unique<TarArchive>(src, /*raw*/ true, compressionMethod);
+            this->archive->check(archive_read_next_header(this->archive->archive, &ae), "failed to read header (%s)");
             if (archive_filter_count(this->archive->archive) < 2) {
                 throw CompressionError("input compression not recognized");
             }
         }
         ssize_t result = archive_read_data(this->archive->archive, data, len);
-        if (result > 0) return result;
+        if (result > 0)
+            return result;
         if (result == 0) {
             throw EndOfFile("reached end of compressed file");
         }
@@ -67,16 +71,19 @@ struct ArchiveCompressionSink : CompressionSink
     Sink & nextSink;
     struct archive * archive;
 
-    ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink)
+    ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
+        : nextSink(nextSink)
     {
         archive = archive_write_new();
-        if (!archive) throw Error("failed to initialize libarchive");
+        if (!archive)
+            throw Error("failed to initialize libarchive");
         check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
         check(archive_write_set_format_raw(archive));
         if (parallel)
             check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
         if (level != COMPRESSION_LEVEL_DEFAULT)
-            check(archive_write_set_filter_option(archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
+            check(archive_write_set_filter_option(
+                archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
         // disable internal buffering
         check(archive_write_set_bytes_per_block(archive, 0));
         // disable output padding
@@ -86,7 +93,8 @@ struct ArchiveCompressionSink : CompressionSink
 
     ~ArchiveCompressionSink() override
     {
-        if (archive) archive_write_free(archive);
+        if (archive)
+            archive_write_free(archive);
     }
 
     void finish() override
@@ -106,7 +114,8 @@ struct ArchiveCompressionSink : CompressionSink
     void writeUnbuffered(std::string_view data) override
     {
         ssize_t result = archive_write_data(archive, data.data(), data.length());
-        if (result <= 0) check(result);
+        if (result <= 0)
+            check(result);
     }
 
 private:
@@ -130,13 +139,20 @@ private:
 struct NoneSink : CompressionSink
 {
     Sink & nextSink;
-    NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink)
+    NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT)
+        : nextSink(nextSink)
     {
         if (level != COMPRESSION_LEVEL_DEFAULT)
             warn("requested compression level '%d' not supported by compression method 'none'", level);
     }
-    void finish() override { flush(); }
-    void writeUnbuffered(std::string_view data) override { nextSink(data); }
+    void finish() override
+    {
+        flush();
+    }
+    void writeUnbuffered(std::string_view data) override
+    {
+        nextSink(data);
+    }
 };
 
 struct BrotliDecompressionSink : ChunkedCompressionSink
@@ -145,7 +161,8 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
     BrotliDecoderState * state;
     bool finished = false;
 
-    BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    BrotliDecompressionSink(Sink & nextSink)
+        : nextSink(nextSink)
     {
         state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
         if (!state)
@@ -173,10 +190,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
         while (!finished && (!data.data() || avail_in)) {
             checkInterrupt();
 
-            if (!BrotliDecoderDecompressStream(state,
-                    &avail_in, &next_in,
-                    &avail_out, &next_out,
-                    nullptr))
+            if (!BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, &next_out, nullptr))
                 throw CompressionError("error while decompressing brotli file");
 
             if (avail_out < sizeof(outbuf) || avail_in == 0) {
@@ -206,8 +220,8 @@ std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Si
     else if (method == "br")
         return std::make_unique<BrotliDecompressionSink>(nextSink);
     else
-        return sourceToSink([&](Source & source) {
-            auto decompressionSource = std::make_unique<ArchiveDecompressionSource>(source);
+        return sourceToSink([method, &nextSink](Source & source) {
+            auto decompressionSource = std::make_unique<ArchiveDecompressionSource>(source, method);
             decompressionSource->drainInto(nextSink);
         });
 }
@@ -219,7 +233,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink
     BrotliEncoderState * state;
     bool finished = false;
 
-    BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
+    BrotliCompressionSink(Sink & nextSink)
+        : nextSink(nextSink)
     {
         state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
         if (!state)
@@ -247,11 +262,9 @@ struct BrotliCompressionSink : ChunkedCompressionSink
         while (!finished && (!data.data() || avail_in)) {
             checkInterrupt();
 
-            if (!BrotliEncoderCompressStream(state,
-                    data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
-                    &avail_in, &next_in,
-                    &avail_out, &next_out,
-                    nullptr))
+            if (!BrotliEncoderCompressStream(
+                    state, data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in,
+                    &avail_out, &next_out, nullptr))
                 throw CompressionError("error while compressing brotli compression");
 
             if (avail_out < sizeof(outbuf) || avail_in == 0) {
@@ -267,9 +280,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
 {
-    std::vector<std::string> la_supports = {
-        "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"
-    };
+    std::vector<std::string> la_supports = {"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4",
+                                            "lzip",  "lzma",     "lzop",  "xz",   "zstd"};
     if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
         return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
     }

src/libutil/compression.hh

@@ -11,7 +11,7 @@ namespace nix {
 
 struct CompressionSink : BufferedSink, FinishSink
 {
-    using BufferedSink::operator ();
+    using BufferedSink::operator();
     using BufferedSink::writeUnbuffered;
     using FinishSink::finish;
 };
@@ -22,7 +22,8 @@ std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Si
 
 std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1);
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
+ref<CompressionSink>
+makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
 
 MakeError(UnknownCompressionMethod, Error);
 

src/libutil/config.cc

@@ -124,7 +124,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p
             auto p = absPath(tokens[1], dirOf(path));
             if (pathExists(p)) {
                 try {
-                    std::string includedContents = readFile(path);
+                    std::string includedContents = readFile(p);
                     applyConfigInner(includedContents, p, parsedContents);
                 } catch (SystemError &) {
                     // TODO: Do we actually want to ignore this? Or is it better to fail?

src/libutil/current-process.cc

@@ -2,7 +2,6 @@
 #include <cstring>
 
 #include "current-process.hh"
-#include "namespaces.hh"
 #include "util.hh"
 #include "finally.hh"
 #include "file-system.hh"
@@ -17,6 +16,7 @@
 # include <mutex>
 # include <sys/resource.h>
 # include "cgroup.hh"
+# include "namespaces.hh"
 #endif
 
 #include <sys/mount.h>
@@ -82,9 +82,11 @@ void setStackSize(rlim_t stackSize)
 
 void restoreProcessContext(bool restoreMounts)
 {
-    restoreSignals();
+    unix::restoreSignals();
     if (restoreMounts) {
+        #if __linux__
         restoreMountNamespace();
+        #endif
     }
 
     if (savedStackSize) {

src/libutil/environment-variables.cc

@@ -32,7 +32,6 @@ std::map<std::string, std::string> getEnv()
     return env;
 }
 
-
 void clearEnv()
 {
     for (auto & name : getEnv())
@@ -43,7 +42,7 @@ void replaceEnv(const std::map<std::string, std::string> & newEnv)
 {
     clearEnv();
     for (auto & newEnvVar : newEnv)
-        setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
+        setEnv(newEnvVar.first.c_str(), newEnvVar.second.c_str());
 }
 
 }

src/libutil/environment-variables.hh

@@ -28,6 +28,14 @@ std::optional<std::string> getEnvNonEmpty(const std::string & key);
  */
 std::map<std::string, std::string> getEnv();
 
+/**
+ * Like POSIX `setenv`, but always overrides.
+ *
+ * We don't need the non-overriding version, and this is easier to
+ * reimplement on Windows.
+ */
+int setEnv(const char * name, const char * value);
+
 /**
  * Clear the environment.
  */

src/libutil/error.cc

@@ -11,14 +11,15 @@
 
 namespace nix {
 
-void BaseError::addTrace(std::shared_ptr<Pos> && e, HintFmt hint)
+void BaseError::addTrace(std::shared_ptr<Pos> && e, HintFmt hint, TracePrint print)
 {
-    err.traces.push_front(Trace { .pos = std::move(e), .hint = hint });
+    err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .print = print });
 }
 
-void throwExceptionSelfCheck(){
+void throwExceptionSelfCheck()
+{
     // This is meant to be caught in initLibUtil()
-    throw SysError("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded.");
+    throw Error("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded.");
 }
 
 // c++ std::exception descendants must have a 'const char* what()' function.
@@ -163,7 +164,7 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std
     return hasPos;
 }
 
-void printTrace(
+static void printTrace(
     std::ostream & output,
     const std::string_view & indent,
     size_t & count,
@@ -379,29 +380,39 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
         // A consecutive sequence of stack traces that are all in `tracesSeen`.
         std::vector<Trace> skippedTraces;
         size_t count = 0;
+        bool truncate = false;
 
         for (const auto & trace : einfo.traces) {
             if (trace.hint.str().empty()) continue;
 
             if (!showTrace && count > 3) {
-                oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n";
-                break;
+                truncate = true;
             }
 
-            if (tracesSeen.count(trace)) {
-                skippedTraces.push_back(trace);
-                continue;
+            if (!truncate || trace.print == TracePrint::Always) {
+
+                if (tracesSeen.count(trace)) {
+                    skippedTraces.push_back(trace);
+                    continue;
+                }
+
+                tracesSeen.insert(trace);
+
+                printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
+
+                count++;
+
+                printTrace(oss, ellipsisIndent, count, trace);
             }
-            tracesSeen.insert(trace);
-
-            printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
-
-            count++;
-
-            printTrace(oss, ellipsisIndent, count, trace);
         }
 
+
         printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
+
+        if (truncate) {
+            oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full, detailed trace)" ANSI_NORMAL << "\n";
+        }
+
         oss << "\n" << prefix;
     }
 

src/libutil/error.hh

@@ -61,9 +61,22 @@ void printCodeLines(std::ostream & out,
     const Pos & errPos,
     const LinesOfCode & loc);
 
+/**
+ * When a stack frame is printed.
+ */
+enum struct TracePrint {
+    /**
+     * The default behavior; always printed when `--show-trace` is set.
+     */
+    Default,
+    /** Always printed. Produced by `builtins.addErrorContext`. */
+    Always,
+};
+
 struct Trace {
     std::shared_ptr<Pos> pos;
     HintFmt hint;
+    TracePrint print = TracePrint::Default;
 };
 
 inline bool operator<(const Trace& lhs, const Trace& rhs);
@@ -137,6 +150,10 @@ public:
         : err(e)
     { }
 
+    std::string message() {
+        return err.msg.str();
+    }
+
     const char * what() const noexcept override { return calcWhat().c_str(); }
     const std::string & msg() const { return calcWhat(); }
     const ErrorInfo & info() const { calcWhat(); return err; }
@@ -161,7 +178,7 @@ public:
         addTrace(std::move(e), HintFmt(std::string(fs), args...));
     }
 
-    void addTrace(std::shared_ptr<Pos> && e, HintFmt hint);
+    void addTrace(std::shared_ptr<Pos> && e, HintFmt hint, TracePrint print = TracePrint::Default);
 
     bool hasTrace() const { return !err.traces.empty(); }
 

src/libutil/experimental-features.cc

@@ -203,16 +203,6 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         )",
         .trackingUrl = "https://github.com/NixOS/nix/milestone/40",
     },
-    {
-        .tag = Xp::ReplFlake,
-        .name = "repl-flake",
-        .description = R"(
-            *Enabled with [`flakes`](#xp-feature-flakes) since 2.19*
-
-            Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands.
-        )",
-        .trackingUrl = "https://github.com/NixOS/nix/milestone/32",
-    },
     {
         .tag = Xp::AutoAllocateUids,
         .name = "auto-allocate-uids",

src/libutil/experimental-features.hh

@@ -26,7 +26,6 @@ enum struct ExperimentalFeature
     RecursiveNix,
     NoUrlLiterals,
     FetchClosure,
-    ReplFlake,
     AutoAllocateUids,
     Cgroups,
     DaemonTrustOverride,

src/libutil/file-descriptor.cc

@@ -8,74 +8,14 @@
 
 namespace nix {
 
-std::string readFile(int fd)
-{
-    struct stat st;
-    if (fstat(fd, &st) == -1)
-        throw SysError("statting file");
-
-    return drainFD(fd, true, st.st_size);
-}
-
-
-void readFull(int fd, char * buf, size_t count)
-{
-    while (count) {
-        checkInterrupt();
-        ssize_t res = read(fd, buf, count);
-        if (res == -1) {
-            if (errno == EINTR) continue;
-            throw SysError("reading from file");
-        }
-        if (res == 0) throw EndOfFile("unexpected end-of-file");
-        count -= res;
-        buf += res;
-    }
-}
-
-
-void writeFull(int fd, std::string_view s, bool allowInterrupts)
-{
-    while (!s.empty()) {
-        if (allowInterrupts) checkInterrupt();
-        ssize_t res = write(fd, s.data(), s.size());
-        if (res == -1 && errno != EINTR)
-            throw SysError("writing to file");
-        if (res > 0)
-            s.remove_prefix(res);
-    }
-}
-
-
-std::string readLine(int fd)
-{
-    std::string s;
-    while (1) {
-        checkInterrupt();
-        char ch;
-        // FIXME: inefficient
-        ssize_t rd = read(fd, &ch, 1);
-        if (rd == -1) {
-            if (errno != EINTR)
-                throw SysError("reading a line");
-        } else if (rd == 0)
-            throw EndOfFile("unexpected EOF reading a line");
-        else {
-            if (ch == '\n') return s;
-            s += ch;
-        }
-    }
-}
-
-
-void writeLine(int fd, std::string s)
+void writeLine(Descriptor fd, std::string s)
 {
     s += '\n';
     writeFull(fd, s);
 }
 
 
-std::string drainFD(int fd, bool block, const size_t reserveSize)
+std::string drainFD(Descriptor fd, bool block, const size_t reserveSize)
 {
     // the parser needs two extra bytes to append terminating characters, other users will
     // not care very much about the extra memory.
@@ -85,50 +25,18 @@ std::string drainFD(int fd, bool block, const size_t reserveSize)
 }
 
 
-void drainFD(int fd, Sink & sink, bool block)
-{
-    // silence GCC maybe-uninitialized warning in finally
-    int saved = 0;
-
-    if (!block) {
-        saved = fcntl(fd, F_GETFL);
-        if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
-            throw SysError("making file descriptor non-blocking");
-    }
-
-    Finally finally([&] {
-        if (!block) {
-            if (fcntl(fd, F_SETFL, saved) == -1)
-                throw SysError("making file descriptor blocking");
-        }
-    });
-
-    std::vector<unsigned char> buf(64 * 1024);
-    while (1) {
-        checkInterrupt();
-        ssize_t rd = read(fd, buf.data(), buf.size());
-        if (rd == -1) {
-            if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
-                break;
-            if (errno != EINTR)
-                throw SysError("reading from file");
-        }
-        else if (rd == 0) break;
-        else sink({reinterpret_cast<char *>(buf.data()), size_t(rd)});
-    }
-}
-
 //////////////////////////////////////////////////////////////////////
 
-AutoCloseFD::AutoCloseFD() : fd{-1} {}
+
+AutoCloseFD::AutoCloseFD() : fd{INVALID_DESCRIPTOR} {}
 
 
-AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
+AutoCloseFD::AutoCloseFD(Descriptor fd) : fd{fd} {}
 
 
 AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
 {
-    that.fd = -1;
+    that.fd = INVALID_DESCRIPTOR;
 }
 
 
@@ -136,7 +44,7 @@ AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
 {
     close();
     fd = that.fd;
-    that.fd = -1;
+    that.fd = INVALID_DESCRIPTOR;
     return *this;
 }
 
@@ -151,7 +59,7 @@ AutoCloseFD::~AutoCloseFD()
 }
 
 
-int AutoCloseFD::get() const
+Descriptor AutoCloseFD::get() const
 {
     return fd;
 }
@@ -159,56 +67,46 @@ int AutoCloseFD::get() const
 
 void AutoCloseFD::close()
 {
-    if (fd != -1) {
-        if (::close(fd) == -1)
+    if (fd != INVALID_DESCRIPTOR) {
+        if(::close(fd) == -1)
             /* This should never happen. */
             throw SysError("closing file descriptor %1%", fd);
-        fd = -1;
+        fd = INVALID_DESCRIPTOR;
     }
 }
 
 void AutoCloseFD::fsync()
 {
-  if (fd != -1) {
-      int result;
+    if (fd != INVALID_DESCRIPTOR) {
+        int result;
+        result =
 #if __APPLE__
-      result = ::fcntl(fd, F_FULLFSYNC);
+            ::fcntl(fd, F_FULLFSYNC)
 #else
-      result = ::fsync(fd);
+            ::fsync(fd)
 #endif
-      if (result == -1)
-          throw SysError("fsync file descriptor %1%", fd);
-  }
+            ;
+        if (result == -1)
+            throw SysError("fsync file descriptor %1%", fd);
+    }
 }
 
 
 AutoCloseFD::operator bool() const
 {
-    return fd != -1;
+    return fd != INVALID_DESCRIPTOR;
 }
 
 
-int AutoCloseFD::release()
+Descriptor AutoCloseFD::release()
 {
-    int oldFD = fd;
-    fd = -1;
+    Descriptor oldFD = fd;
+    fd = INVALID_DESCRIPTOR;
     return oldFD;
 }
 
 
-void Pipe::create()
-{
-    int fds[2];
-#if HAVE_PIPE2
-    if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
-#else
-    if (pipe(fds) != 0) throw SysError("creating pipe");
-    closeOnExec(fds[0]);
-    closeOnExec(fds[1]);
-#endif
-    readSide = fds[0];
-    writeSide = fds[1];
-}
+//////////////////////////////////////////////////////////////////////
 
 
 void Pipe::close()
@@ -217,38 +115,4 @@ void Pipe::close()
     writeSide.close();
 }
 
-//////////////////////////////////////////////////////////////////////
-
-void closeMostFDs(const std::set<int> & exceptions)
-{
-#if __linux__
-    try {
-        for (auto & s : readDirectory("/proc/self/fd")) {
-            auto fd = std::stoi(s.name);
-            if (!exceptions.count(fd)) {
-                debug("closing leaked FD %d", fd);
-                close(fd);
-            }
-        }
-        return;
-    } catch (SystemError &) {
-    }
-#endif
-
-    int maxFD = 0;
-    maxFD = sysconf(_SC_OPEN_MAX);
-    for (int fd = 0; fd < maxFD; ++fd)
-        if (!exceptions.count(fd))
-            close(fd); /* ignore result */
-}
-
-
-void closeOnExec(int fd)
-{
-    int prev;
-    if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
-        fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
-        throw SysError("setting close-on-exec flag");
-}
-
 }

src/libutil/file-descriptor.hh

@@ -9,53 +9,85 @@ namespace nix {
 struct Sink;
 struct Source;
 
+/**
+ * Operating System capability
+ */
+typedef int Descriptor;
+
+const Descriptor INVALID_DESCRIPTOR = -1;
+
+/**
+ * Convert a native `Descriptor` to a POSIX file descriptor
+ *
+ * This is a no-op except on Windows.
+ */
+static inline Descriptor toDescriptor(int fd)
+{
+    return fd;
+}
+
+/**
+ * Convert a POSIX file descriptor to a native `Descriptor`
+ *
+ * This is a no-op except on Windows.
+ */
+static inline int fromDescriptor(Descriptor fd, int flags)
+{
+    return fd;
+}
+
 /**
  * Read the contents of a resource into a string.
  */
-std::string readFile(int fd);
+std::string readFile(Descriptor fd);
 
 /**
  * Wrappers arount read()/write() that read/write exactly the
  * requested number of bytes.
  */
-void readFull(int fd, char * buf, size_t count);
+void readFull(Descriptor fd, char * buf, size_t count);
 
-void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
+void writeFull(Descriptor fd, std::string_view s, bool allowInterrupts = true);
 
 /**
  * Read a line from a file descriptor.
  */
-std::string readLine(int fd);
+std::string readLine(Descriptor fd);
 
 /**
  * Write a line to a file descriptor.
  */
-void writeLine(int fd, std::string s);
+void writeLine(Descriptor fd, std::string s);
 
 /**
  * Read a file descriptor until EOF occurs.
  */
-std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
+std::string drainFD(Descriptor fd, bool block = true, const size_t reserveSize=0);
 
-void drainFD(int fd, Sink & sink, bool block = true);
+void drainFD(Descriptor fd, Sink & sink, bool block = true);
+
+[[gnu::always_inline]]
+inline Descriptor getStandardOut() {
+    return STDOUT_FILENO;
+}
 
 /**
  * Automatic cleanup of resources.
  */
 class AutoCloseFD
 {
-    int fd;
+    Descriptor fd;
 public:
     AutoCloseFD();
-    AutoCloseFD(int fd);
+    AutoCloseFD(Descriptor fd);
     AutoCloseFD(const AutoCloseFD & fd) = delete;
     AutoCloseFD(AutoCloseFD&& fd);
     ~AutoCloseFD();
     AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
     AutoCloseFD& operator =(AutoCloseFD&& fd);
-    int get() const;
+    Descriptor get() const;
     explicit operator bool() const;
-    int release();
+    Descriptor release();
     void close();
     void fsync();
 };
@@ -72,12 +104,12 @@ public:
  * Close all file descriptors except those listed in the given set.
  * Good practice in child processes.
  */
-void closeMostFDs(const std::set<int> & exceptions);
+void closeMostFDs(const std::set<Descriptor> & exceptions);
 
 /**
  * Set the close-on-exec flag for the given file descriptor.
  */
-void closeOnExec(int fd);
+void closeOnExec(Descriptor fd);
 
 MakeError(EndOfFile, Error);
 

src/libutil/file-system.cc

@@ -128,7 +128,7 @@ std::string_view baseNameOf(std::string_view path)
         return "";
 
     auto last = path.size() - 1;
-    if (path[last] == '/' && last > 0)
+    while (last > 0 && path[last] == '/')
         last -= 1;
 
     auto pos = path.rfind('/', last);
@@ -174,15 +174,23 @@ struct stat lstat(const Path & path)
 }
 
 
+std::optional<struct stat> maybeLstat(const Path & path)
+{
+    std::optional<struct stat> st{std::in_place};
+    if (lstat(path.c_str(), &*st))
+    {
+        if (errno == ENOENT || errno == ENOTDIR)
+            st.reset();
+        else
+            throw SysError("getting status of '%s'", path);
+    }
+    return st;
+}
+
+
 bool pathExists(const Path & path)
 {
-    int res;
-    struct stat st;
-    res = lstat(path.c_str(), &st);
-    if (!res) return true;
-    if (errno != ENOENT && errno != ENOTDIR)
-        throw SysError("getting status of %1%", path);
-    return false;
+    return maybeLstat(path).has_value();
 }
 
 bool pathAccessible(const Path & path)
@@ -494,10 +502,14 @@ void AutoDelete::reset(const Path & p, bool recursive) {
 
 //////////////////////////////////////////////////////////////////////
 
+std::string defaultTempDir() {
+    return getEnvNonEmpty("TMPDIR").value_or("/tmp");
+}
+
 static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
     std::atomic<unsigned int> & counter)
 {
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    tmpRoot = canonPath(tmpRoot.empty() ? defaultTempDir() : tmpRoot, true);
     if (includePid)
         return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
     else
@@ -537,7 +549,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 
 std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
 {
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    Path tmpl(defaultTempDir() + "/" + prefix + ".XXXXXX");
     // Strictly speaking, this is UB, but who cares...
     // FIXME: use O_TMPFILE.
     AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));

src/libutil/file-system.hh

@@ -84,6 +84,11 @@ bool isDirOrInDir(std::string_view path, std::string_view dir);
  */
 struct stat stat(const Path & path);
 struct stat lstat(const Path & path);
+/**
+ * `lstat` the given path if it exists.
+ * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise
+ */
+std::optional<struct stat> maybeLstat(const Path & path);
 
 /**
  * @return true iff the given path exists.
@@ -234,6 +239,10 @@ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
  */
 std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
 
+/**
+ * Return `TMPDIR`, or the default temporary directory if unset or empty.
+ */
+Path defaultTempDir();
 
 /**
  * Used in various places.

src/libutil/finally.hh

@@ -1,16 +1,25 @@
 #pragma once
 ///@file
 
+#include <utility>
+
 /**
  * A trivial class to run a function at the end of a scope.
  */
 template<typename Fn>
-class Finally
+class [[nodiscard("Finally values must be used")]] Finally
 {
 private:
     Fn fun;
+    bool movedFrom = false;
 
 public:
     Finally(Fn fun) : fun(std::move(fun)) { }
-    ~Finally() { fun(); }
+    // Copying Finallys is definitely not a good idea and will cause them to be
+    // called twice.
+    Finally(Finally &other) = delete;
+    Finally(Finally &&other) : fun(std::move(other.fun)) {
+        other.movedFrom = true;
+    }
+    ~Finally() { if (!movedFrom) fun(); }
 };

src/libutil/fmt.hh

@@ -144,6 +144,10 @@ public:
         : HintFmt("%s", Uncolored(literal))
     { }
 
+    static HintFmt fromFormatString(const std::string & format) {
+        return HintFmt(boost::format(format));
+    }
+
     /**
      * Interpolate the given arguments into the format string.
      */

src/libutil/git.cc

@@ -251,6 +251,7 @@ void dumpTree(const Tree & entries, Sink & sink,
     for (auto & [name, entry] : entries) {
         auto name2 = name;
         if (entry.mode == Mode::Directory) {
+            assert(!name2.empty());
             assert(name2.back() == '/');
             name2.pop_back();
         }

src/libutil/json-utils.cc

@@ -1,5 +1,8 @@
 #include "json-utils.hh"
 #include "error.hh"
+#include "types.hh"
+#include <nlohmann/json_fwd.hpp>
+#include <iostream>
 
 namespace nix {
 
@@ -18,26 +21,115 @@ nlohmann::json * get(nlohmann::json & map, const std::string & key)
 }
 
 const nlohmann::json & valueAt(
-    const nlohmann::json & map,
+    const nlohmann::json::object_t & map,
     const std::string & key)
 {
     if (!map.contains(key))
-        throw Error("Expected JSON object to contain key '%s' but it doesn't", key);
+        throw Error("Expected JSON object to contain key '%s' but it doesn't: %s", key, nlohmann::json(map).dump());
 
-    return map[key];
+    return map.at(key);
 }
 
-const nlohmann::json & ensureType(
+std::optional<nlohmann::json> optionalValueAt(const nlohmann::json & value, const std::string & key)
+{
+    try {
+        auto & v = valueAt(value, key);
+        return v.get<nlohmann::json>();
+    } catch (...) {
+        return std::nullopt;
+    }
+}
+
+
+std::optional<nlohmann::json> getNullable(const nlohmann::json & value)
+{
+    if (value.is_null())
+        return std::nullopt;
+
+    return value.get<nlohmann::json>();
+}
+
+/**
+ * Ensure the type of a JSON object is what you expect, failing with a
+ * ensure type if it isn't.
+ *
+ * Use before type conversions and element access to avoid ugly
+ * exceptions, but only part of this module to define the other `get*`
+ * functions. It is too cumbersome and easy to forget to expect regular
+ * JSON code to use it directly.
+ */
+static const nlohmann::json & ensureType(
     const nlohmann::json & value,
     nlohmann::json::value_type expectedType
     )
 {
     if (value.type() != expectedType)
         throw Error(
-            "Expected JSON value to be of type '%s' but it is of type '%s'",
+            "Expected JSON value to be of type '%s' but it is of type '%s': %s",
             nlohmann::json(expectedType).type_name(),
-            value.type_name());
+            value.type_name(), value.dump());
 
     return value;
 }
+
+const nlohmann::json::object_t & getObject(const nlohmann::json & value)
+{
+    return ensureType(value, nlohmann::json::value_t::object).get_ref<const nlohmann::json::object_t &>();
+}
+
+const nlohmann::json::array_t & getArray(const nlohmann::json & value)
+{
+    return ensureType(value, nlohmann::json::value_t::array).get_ref<const nlohmann::json::array_t &>();
+}
+
+const nlohmann::json::string_t & getString(const nlohmann::json & value)
+{
+    return ensureType(value, nlohmann::json::value_t::string).get_ref<const nlohmann::json::string_t &>();
+}
+
+const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value)
+{
+    return ensureType(value, nlohmann::json::value_t::number_integer).get_ref<const nlohmann::json::number_integer_t &>();
+}
+
+const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value)
+{
+    return ensureType(value, nlohmann::json::value_t::boolean).get_ref<const nlohmann::json::boolean_t &>();
+}
+
+Strings getStringList(const nlohmann::json & value)
+{
+    auto & jsonArray = getArray(value);
+
+    Strings stringList;
+
+    for (const auto & elem : jsonArray)
+        stringList.push_back(getString(elem));
+
+    return stringList;
+}
+
+StringMap getStringMap(const nlohmann::json & value)
+{
+    auto & jsonObject = getObject(value);
+
+    StringMap stringMap;
+
+    for (const auto & [key, value] : jsonObject)
+        stringMap[getString(key)] = getString(value);
+
+    return stringMap;
+}
+
+StringSet getStringSet(const nlohmann::json & value)
+{
+    auto & jsonArray = getArray(value);
+
+    StringSet stringSet;
+
+    for (const auto & elem : jsonArray)
+        stringSet.insert(getString(elem));
+
+    return stringSet;
+}
 }

src/libutil/json-utils.hh

@@ -3,6 +3,9 @@
 
 #include <nlohmann/json.hpp>
 #include <list>
+#include <nlohmann/json_fwd.hpp>
+
+#include "types.hh"
 
 namespace nix {
 
@@ -11,26 +14,30 @@ const nlohmann::json * get(const nlohmann::json & map, const std::string & key);
 nlohmann::json * get(nlohmann::json & map, const std::string & key);
 
 /**
- * Get the value of a json object at a key safely, failing
- * with a Nix Error if the key does not exist.
+ * Get the value of a json object at a key safely, failing with a nice
+ * error if the key does not exist.
  *
  * Use instead of nlohmann::json::at() to avoid ugly exceptions.
- *
- * _Does not check whether `map` is an object_, use `ensureType` for that.
  */
 const nlohmann::json & valueAt(
-    const nlohmann::json & map,
+    const nlohmann::json::object_t & map,
     const std::string & key);
 
+std::optional<nlohmann::json> optionalValueAt(const nlohmann::json & value, const std::string & key);
+
 /**
- * Ensure the type of a json object is what you expect, failing
- * with a Nix Error if it isn't.
- *
- * Use before type conversions and element access to avoid ugly exceptions.
+ * Downcast the json object, failing with a nice error if the conversion fails.
+ * See https://json.nlohmann.me/features/types/
  */
-const nlohmann::json & ensureType(
-    const nlohmann::json & value,
-    nlohmann::json::value_type expectedType);
+std::optional<nlohmann::json> getNullable(const nlohmann::json & value);
+const nlohmann::json::object_t & getObject(const nlohmann::json & value);
+const nlohmann::json::array_t & getArray(const nlohmann::json & value);
+const nlohmann::json::string_t & getString(const nlohmann::json & value);
+const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value);
+const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value);
+Strings getStringList(const nlohmann::json & value);
+StringMap getStringMap(const nlohmann::json & value);
+StringSet getStringSet(const nlohmann::json & value);
 
 /**
  * For `adl_serializer<std::optional<T>>` below, we need to track what

src/libutil/linux/namespaces.cc

@@ -5,18 +5,14 @@
 #include "processes.hh"
 #include "signals.hh"
 
-#if __linux__
-# include <mutex>
-# include <sys/resource.h>
-# include "cgroup.hh"
-#endif
+#include <mutex>
+#include <sys/resource.h>
+#include "cgroup.hh"
 
 #include <sys/mount.h>
 
 namespace nix {
 
-#if __linux__
-
 bool userNamespacesSupported()
 {
     static auto res = [&]() -> bool
@@ -101,19 +97,14 @@ bool mountAndPidNamespacesSupported()
     return res;
 }
 
-#endif
-
 
 //////////////////////////////////////////////////////////////////////
 
-#if __linux__
 static AutoCloseFD fdSavedMountNamespace;
 static AutoCloseFD fdSavedRoot;
-#endif
 
 void saveMountNamespace()
 {
-#if __linux__
     static std::once_flag done;
     std::call_once(done, []() {
         fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
@@ -122,12 +113,10 @@ void saveMountNamespace()
 
         fdSavedRoot = open("/proc/self/root", O_RDONLY);
     });
-#endif
 }
 
 void restoreMountNamespace()
 {
-#if __linux__
     try {
         auto savedCwd = absPath(".");
 
@@ -146,15 +135,12 @@ void restoreMountNamespace()
     } catch (Error & e) {
         debug(e.msg());
     }
-#endif
 }
 
 void unshareFilesystem()
 {
-#ifdef __linux__
     if (unshare(CLONE_FS) != 0 && errno != EPERM)
         throw SysError("unsharing filesystem state in download thread");
-#endif
 }
 
 }

src/libutil/linux/namespaces.hh

@@ -26,12 +26,8 @@ void restoreMountNamespace();
  */
 void unshareFilesystem();
 
-#if __linux__
-
 bool userNamespacesSupported();
 
 bool mountAndPidNamespacesSupported();
 
-#endif
-
 }

src/libutil/local.mk

@@ -5,8 +5,23 @@ libutil_NAME = libnixutil
 libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc)
+ifdef HOST_UNIX
+  libutil_SOURCES += $(wildcard $(d)/unix/*.cc)
+endif
+ifdef HOST_LINUX
+    libutil_SOURCES += $(wildcard $(d)/linux/*.cc)
+endif
 
-libutil_CXXFLAGS += -I src/libutil
+# Not just for this library itself, but also for downstream libraries using this library
+
+INCLUDE_libutil := -I $(d)
+ifdef HOST_UNIX
+  INCLUDE_libutil += -I $(d)/unix
+endif
+ifdef HOST_LINUX
+  INCLUDE_libutil += -I $(d)/linux
+endif
+libutil_CXXFLAGS += $(INCLUDE_libutil)
 
 libutil_LDFLAGS += $(THREAD_LDFLAGS) $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 

src/libutil/logging.cc

@@ -37,8 +37,9 @@ void Logger::warn(const std::string & msg)
 
 void Logger::writeToStdout(std::string_view s)
 {
-    writeFull(STDOUT_FILENO, s);
-    writeFull(STDOUT_FILENO, "\n");
+    Descriptor standard_out = getStandardOut();
+    writeFull(standard_out, s);
+    writeFull(standard_out, "\n");
 }
 
 class SimpleLogger : public Logger
@@ -52,7 +53,7 @@ public:
         : printBuildLogs(printBuildLogs)
     {
         systemd = getEnv("IN_SYSTEMD") == "1";
-        tty = shouldANSI();
+        tty = isTTY();
     }
 
     bool isVerbose() override {

src/libutil/posix-source-accessor.cc

@@ -97,13 +97,7 @@ std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & pa
         if (i != cache->end()) return i->second;
     }
 
-    std::optional<struct stat> st{std::in_place};
-    if (::lstat(absPath.c_str(), &*st)) {
-        if (errno == ENOENT || errno == ENOTDIR)
-            st.reset();
-        else
-            throw SysError("getting status of '%s'", showPath(path));
-    }
+    auto st = nix::maybeLstat(absPath.c_str());
 
     auto cache(_cache.lock());
     if (cache->size() >= 16384) cache->clear();

src/libutil/serialise.hh

@@ -119,18 +119,18 @@ protected:
  */
 struct FdSink : BufferedSink
 {
-    int fd;
+    Descriptor fd;
     size_t written = 0;
 
-    FdSink() : fd(-1) { }
-    FdSink(int fd) : fd(fd) { }
+    FdSink() : fd(INVALID_DESCRIPTOR) { }
+    FdSink(Descriptor fd) : fd(fd) { }
     FdSink(FdSink&&) = default;
 
     FdSink & operator=(FdSink && s)
     {
         flush();
         fd = s.fd;
-        s.fd = -1;
+        s.fd = INVALID_DESCRIPTOR;
         written = s.written;
         return *this;
     }
@@ -151,18 +151,18 @@ private:
  */
 struct FdSource : BufferedSource
 {
-    int fd;
+    Descriptor fd;
     size_t read = 0;
     BackedStringView endOfFileError{"unexpected end-of-file"};
 
-    FdSource() : fd(-1) { }
-    FdSource(int fd) : fd(fd) { }
+    FdSource() : fd(INVALID_DESCRIPTOR) { }
+    FdSource(Descriptor fd) : fd(fd) { }
     FdSource(FdSource &&) = default;
 
     FdSource & operator=(FdSource && s)
     {
         fd = s.fd;
-        s.fd = -1;
+        s.fd = INVALID_DESCRIPTOR;
         read = s.read;
         return *this;
     }

src/libutil/signals.hh

@@ -4,72 +4,39 @@
 #include "types.hh"
 #include "error.hh"
 #include "logging.hh"
-#include "ansicolor.hh"
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <signal.h>
-
-#include <boost/lexical_cast.hpp>
-
-#include <atomic>
 #include <functional>
-#include <map>
-#include <sstream>
-#include <optional>
 
 namespace nix {
 
 /* User interruption. */
 
-extern std::atomic<bool> _isInterrupted;
+/**
+ * @note Does nothing on Windows
+ */
+static inline void setInterrupted(bool isInterrupted);
 
-extern thread_local std::function<bool()> interruptCheck;
+/**
+ * @note Does nothing on Windows
+ */
+static inline bool getInterrupted();
 
+/**
+ * @note Does nothing on Windows
+ */
 void setInterruptThrown();
 
-void _interrupted();
-
-void inline checkInterrupt()
-{
-    if (_isInterrupted || (interruptCheck && interruptCheck()))
-        _interrupted();
-}
+/**
+ * @note Does nothing on Windows
+ */
+inline void checkInterrupt();
 
+/**
+ * @note Never will happen on Windows
+ */
 MakeError(Interrupted, BaseError);
 
 
-/**
- * Start a thread that handles various signals. Also block those signals
- * on the current thread (and thus any threads created by it).
- * Saves the signal mask before changing the mask to block those signals.
- * See saveSignalMask().
- */
-void startSignalHandlerThread();
-
-/**
- * Saves the signal mask, which is the signal mask that nix will restore
- * before creating child processes.
- * See setChildSignalMask() to set an arbitrary signal mask instead of the
- * current mask.
- */
-void saveSignalMask();
-
-/**
- * To use in a process that already called `startSignalHandlerThread()`
- * or `saveSignalMask()` first.
- */
-void restoreSignals();
-
-/**
- * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
- * necessarily match the current thread's mask.
- * See saveSignalMask() to set the saved mask to the current mask.
- */
-void setChildSignalMask(sigset_t *sigs);
-
 struct InterruptCallback
 {
     virtual ~InterruptCallback() { };
@@ -78,27 +45,21 @@ struct InterruptCallback
 /**
  * Register a function that gets called on SIGINT (in a non-signal
  * context).
+ *
+ * @note Does nothing on Windows
  */
 std::unique_ptr<InterruptCallback> createInterruptCallback(
     std::function<void()> callback);
 
-void triggerInterrupt();
-
 /**
  * A RAII class that causes the current thread to receive SIGUSR1 when
  * the signal handler thread receives SIGINT. That is, this allows
  * SIGINT to be multiplexed to multiple threads.
+ *
+ * @note Does nothing on Windows
  */
-struct ReceiveInterrupts
-{
-    pthread_t target;
-    std::unique_ptr<InterruptCallback> callback;
-
-    ReceiveInterrupts()
-        : target(pthread_self())
-        , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); }))
-    { }
-};
-
+struct ReceiveInterrupts;
 
 }
+
+#include "signals-impl.hh"

src/libutil/tarfile.cc

@@ -1,18 +1,21 @@
 #include <archive.h>
 #include <archive_entry.h>
 
+#include "finally.hh"
 #include "serialise.hh"
 #include "tarfile.hh"
 #include "file-system.hh"
 
 namespace nix {
 
-static int callback_open(struct archive *, void * self)
+namespace {
+
+int callback_open(struct archive *, void * self)
 {
     return ARCHIVE_OK;
 }
 
-static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
+ssize_t callback_read(struct archive * archive, void * _self, const void ** buffer)
 {
     auto self = (TarArchive *) _self;
     *buffer = self->buffer.data();
@@ -27,41 +30,71 @@ static ssize_t callback_read(struct archive * archive, void * _self, const void
     }
 }
 
-static int callback_close(struct archive *, void * self)
+int callback_close(struct archive *, void * self)
 {
     return ARCHIVE_OK;
 }
 
-void TarArchive::check(int err, const std::string & reason)
+void checkLibArchive(archive * archive, int err, const std::string & reason)
 {
     if (err == ARCHIVE_EOF)
         throw EndOfFile("reached end of archive");
     else if (err != ARCHIVE_OK)
-        throw Error(reason, archive_error_string(this->archive));
+        throw Error(reason, archive_error_string(archive));
 }
 
-TarArchive::TarArchive(Source & source, bool raw) : buffer(65536)
+constexpr auto defaultBufferSize = std::size_t{65536};
+}
+
+void TarArchive::check(int err, const std::string & reason)
 {
-    this->archive = archive_read_new();
-    this->source = &source;
+    checkLibArchive(archive, err, reason);
+}
+
+/// @brief Get filter_code from its name.
+///
+/// libarchive does not provide a convenience function like archive_write_add_filter_by_name but for reading.
+/// Instead it's necessary to use this kludge to convert method -> code and
+/// then use archive_read_support_filter_by_code. Arguably this is better than
+/// hand-rolling the equivalent function that is better implemented in libarchive.
+int getArchiveFilterCodeByName(const std::string & method)
+{
+    auto * ar = archive_write_new();
+    auto cleanup = Finally{[&ar]() { checkLibArchive(ar, archive_write_close(ar), "failed to close archive: %s"); }};
+    auto err = archive_write_add_filter_by_name(ar, method.c_str());
+    checkLibArchive(ar, err, "failed to get libarchive filter by name: %s");
+    auto code = archive_filter_code(ar, 0);
+    return code;
+}
+
+TarArchive::TarArchive(Source & source, bool raw, std::optional<std::string> compression_method)
+    : archive{archive_read_new()}
+    , source{&source}
+    , buffer(defaultBufferSize)
+{
+    if (!compression_method) {
+        archive_read_support_filter_all(archive);
+    } else {
+        archive_read_support_filter_by_code(archive, getArchiveFilterCodeByName(*compression_method));
+    }
 
     if (!raw) {
-        archive_read_support_filter_all(archive);
         archive_read_support_format_all(archive);
     } else {
-        archive_read_support_filter_all(archive);
         archive_read_support_format_raw(archive);
         archive_read_support_format_empty(archive);
     }
+
     archive_read_set_option(archive, NULL, "mac-ext", NULL);
-    check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
+    check(
+        archive_read_open(archive, (void *) this, callback_open, callback_read, callback_close),
+        "Failed to open archive (%s)");
 }
 
-
 TarArchive::TarArchive(const Path & path)
+    : archive{archive_read_new()}
+    , buffer(defaultBufferSize)
 {
-    this->archive = archive_read_new();
-
     archive_read_support_filter_all(archive);
     archive_read_support_format_all(archive);
     archive_read_set_option(archive, NULL, "mac-ext", NULL);
@@ -75,19 +108,19 @@ void TarArchive::close()
 
 TarArchive::~TarArchive()
 {
-    if (this->archive) archive_read_free(this->archive);
+    if (this->archive)
+        archive_read_free(this->archive);
 }
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
-    int flags = ARCHIVE_EXTRACT_TIME
-        | ARCHIVE_EXTRACT_SECURE_SYMLINKS
-        | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
+    int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
 
     for (;;) {
         struct archive_entry * entry;
         int r = archive_read_next_header(archive.archive, &entry);
-        if (r == ARCHIVE_EOF) break;
+        if (r == ARCHIVE_EOF)
+            break;
         auto name = archive_entry_pathname(entry);
         if (!name)
             throw Error("cannot get archive member name: %s", archive_error_string(archive.archive));
@@ -96,18 +129,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         else
             archive.check(r);
 
-        archive_entry_copy_pathname(entry,
-            (destDir + "/" + name).c_str());
+        archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str());
 
         // sources can and do contain dirs with no rx bits
         if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500)
             archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500);
 
         // Patch hardlink path
-        const char *original_hardlink = archive_entry_hardlink(entry);
+        const char * original_hardlink = archive_entry_hardlink(entry);
         if (original_hardlink) {
-            archive_entry_copy_hardlink(entry,
-                (destDir + "/" + original_hardlink).c_str());
+            archive_entry_copy_hardlink(entry, (destDir + "/" + original_hardlink).c_str());
         }
 
         archive.check(archive_read_extract(archive.archive, entry, flags));
@@ -140,7 +171,8 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin
         // FIXME: merge with extract_archive
         struct archive_entry * entry;
         int r = archive_read_next_header(archive.archive, &entry);
-        if (r == ARCHIVE_EOF) break;
+        if (r == ARCHIVE_EOF)
+            break;
         auto path = archive_entry_pathname(entry);
         if (!path)
             throw Error("cannot get archive member name: %s", archive_error_string(archive.archive));
@@ -167,8 +199,9 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin
                     auto n = archive_read_data(archive.archive, buf.data(), buf.size());
                     if (n < 0)
                         throw Error("cannot read file '%s' from tarball", path);
-                    if (n == 0) break;
-                    crf(std::string_view {
+                    if (n == 0)
+                        break;
+                    crf(std::string_view{
                         (const char *) buf.data(),
                         (size_t) n,
                     });

src/libutil/tarfile.hh

@@ -7,25 +7,36 @@
 
 namespace nix {
 
-struct TarArchive {
+struct TarArchive
+{
     struct archive * archive;
     Source * source;
     std::vector<unsigned char> buffer;
 
     void check(int err, const std::string & reason = "failed to extract archive (%s)");
 
-    TarArchive(Source & source, bool raw = false);
+    explicit TarArchive(const Path & path);
 
-    TarArchive(const Path & path);
+    /// @brief Create a generic archive from source.
+    /// @param source - Input byte stream.
+    /// @param raw - Whether to enable raw file support. For more info look in docs:
+    /// https://manpages.debian.org/stretch/libarchive-dev/archive_read_format.3.en.html
+    /// @param compression_method - Primary compression method to use. std::nullopt means 'all'.
+    TarArchive(Source & source, bool raw = false, std::optional<std::string> compression_method = std::nullopt);
 
-    /// disable copy constructor
+    /// Disable copy constructor. Explicitly default move assignment/constructor.
     TarArchive(const TarArchive &) = delete;
+    TarArchive & operator=(const TarArchive &) = delete;
+    TarArchive(TarArchive &&) = default;
+    TarArchive & operator=(TarArchive &&) = default;
 
     void close();
 
     ~TarArchive();
 };
 
+int getArchiveFilterCodeByName(const std::string & method);
+
 void unpackTarfile(Source & source, const Path & destDir);
 
 void unpackTarfile(const Path & tarFile, const Path & destDir);

src/libutil/terminal.cc

@@ -7,11 +7,14 @@
 
 namespace nix {
 
-bool shouldANSI()
+bool isTTY()
 {
-    return isatty(STDERR_FILENO)
+    static const bool tty =
+        isatty(STDERR_FILENO)
         && getEnv("TERM").value_or("dumb") != "dumb"
         && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value());
+
+    return tty;
 }
 
 std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width)

src/libutil/terminal.hh

@@ -8,7 +8,7 @@ namespace nix {
  * Determine whether ANSI escape sequences are appropriate for the
  * present output.
  */
-bool shouldANSI();
+bool isTTY();
 
 /**
  * Truncate a string to 'width' printable characters. If 'filterAll'

src/libutil/thread-pool.cc

@@ -82,7 +82,7 @@ void ThreadPool::doWork(bool mainThread)
     ReceiveInterrupts receiveInterrupts;
 
     if (!mainThread)
-        interruptCheck = [&]() { return (bool) quit; };
+        unix::interruptCheck = [&]() { return (bool) quit; };
 
     bool didWork = false;
     std::exception_ptr exc;

src/libutil/unix/environment-variables.cc

@@ -0,0 +1,12 @@
+#include <cstdlib>
+
+#include "environment-variables.hh"
+
+namespace nix {
+
+int setEnv(const char * name, const char * value)
+{
+    return ::setenv(name, value, 1);
+}
+
+}

src/libutil/unix/file-descriptor.cc

@@ -0,0 +1,155 @@
+#include "file-system.hh"
+#include "signals.hh"
+#include "finally.hh"
+#include "serialise.hh"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace nix {
+
+std::string readFile(int fd)
+{
+    struct stat st;
+    if (fstat(fd, &st) == -1)
+        throw SysError("statting file");
+
+    return drainFD(fd, true, st.st_size);
+}
+
+
+void readFull(int fd, char * buf, size_t count)
+{
+    while (count) {
+        checkInterrupt();
+        ssize_t res = read(fd, buf, count);
+        if (res == -1) {
+            if (errno == EINTR) continue;
+            throw SysError("reading from file");
+        }
+        if (res == 0) throw EndOfFile("unexpected end-of-file");
+        count -= res;
+        buf += res;
+    }
+}
+
+
+void writeFull(int fd, std::string_view s, bool allowInterrupts)
+{
+    while (!s.empty()) {
+        if (allowInterrupts) checkInterrupt();
+        ssize_t res = write(fd, s.data(), s.size());
+        if (res == -1 && errno != EINTR)
+            throw SysError("writing to file");
+        if (res > 0)
+            s.remove_prefix(res);
+    }
+}
+
+
+std::string readLine(int fd)
+{
+    std::string s;
+    while (1) {
+        checkInterrupt();
+        char ch;
+        // FIXME: inefficient
+        ssize_t rd = read(fd, &ch, 1);
+        if (rd == -1) {
+            if (errno != EINTR)
+                throw SysError("reading a line");
+        } else if (rd == 0)
+            throw EndOfFile("unexpected EOF reading a line");
+        else {
+            if (ch == '\n') return s;
+            s += ch;
+        }
+    }
+}
+
+
+void drainFD(int fd, Sink & sink, bool block)
+{
+    // silence GCC maybe-uninitialized warning in finally
+    int saved = 0;
+
+    if (!block) {
+        saved = fcntl(fd, F_GETFL);
+        if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
+            throw SysError("making file descriptor non-blocking");
+    }
+
+    Finally finally([&]() {
+        if (!block) {
+            if (fcntl(fd, F_SETFL, saved) == -1)
+                throw SysError("making file descriptor blocking");
+        }
+    });
+
+    std::vector<unsigned char> buf(64 * 1024);
+    while (1) {
+        checkInterrupt();
+        ssize_t rd = read(fd, buf.data(), buf.size());
+        if (rd == -1) {
+            if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
+                break;
+            if (errno != EINTR)
+                throw SysError("reading from file");
+        }
+        else if (rd == 0) break;
+        else sink({reinterpret_cast<char *>(buf.data()), (size_t) rd});
+    }
+}
+
+//////////////////////////////////////////////////////////////////////
+
+void Pipe::create()
+{
+    int fds[2];
+#if HAVE_PIPE2
+    if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
+#else
+    if (pipe(fds) != 0) throw SysError("creating pipe");
+    closeOnExec(fds[0]);
+    closeOnExec(fds[1]);
+#endif
+    readSide = fds[0];
+    writeSide = fds[1];
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+void closeMostFDs(const std::set<int> & exceptions)
+{
+#if __linux__
+    try {
+        for (auto & s : readDirectory("/proc/self/fd")) {
+            auto fd = std::stoi(s.name);
+            if (!exceptions.count(fd)) {
+                debug("closing leaked FD %d", fd);
+                close(fd);
+            }
+        }
+        return;
+    } catch (SysError &) {
+    }
+#endif
+
+    int maxFD = 0;
+    maxFD = sysconf(_SC_OPEN_MAX);
+    for (int fd = 0; fd < maxFD; ++fd)
+        if (!exceptions.count(fd))
+            close(fd); /* ignore result */
+}
+
+
+void closeOnExec(int fd)
+{
+    int prev;
+    if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
+        fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
+        throw SysError("setting close-on-exec flag");
+}
+
+}

src/libutil/unix/monitor-fd.hh

@@ -50,7 +50,7 @@ public:
                 */
                 if (count == 0) continue;
                 if (fds[0].revents & POLLHUP) {
-                    triggerInterrupt();
+                    unix::triggerInterrupt();
                     break;
                 }
                 /* This will only happen on macOS. We sleep a bit to

src/libutil/unix/signals-impl.hh

@@ -0,0 +1,111 @@
+#pragma once
+/**
+ * @file
+ *
+ * Implementation of some inline definitions for Unix signals, and also
+ * some extra Unix-only interfaces.
+ *
+ * (The only reason everything about signals isn't Unix-only is some
+ * no-op definitions are provided on Windows to avoid excess CPP in
+ * downstream code.)
+ */
+
+#include "types.hh"
+#include "error.hh"
+#include "logging.hh"
+#include "ansicolor.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <sstream>
+#include <optional>
+
+namespace nix {
+
+/* User interruption. */
+
+namespace unix {
+
+extern std::atomic<bool> _isInterrupted;
+
+extern thread_local std::function<bool()> interruptCheck;
+
+void _interrupted();
+
+/**
+ * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
+ * necessarily match the current thread's mask.
+ * See saveSignalMask() to set the saved mask to the current mask.
+ */
+void setChildSignalMask(sigset_t *sigs);
+
+/**
+ * Start a thread that handles various signals. Also block those signals
+ * on the current thread (and thus any threads created by it).
+ * Saves the signal mask before changing the mask to block those signals.
+ * See saveSignalMask().
+ */
+void startSignalHandlerThread();
+
+/**
+ * Saves the signal mask, which is the signal mask that nix will restore
+ * before creating child processes.
+ * See setChildSignalMask() to set an arbitrary signal mask instead of the
+ * current mask.
+ */
+void saveSignalMask();
+
+/**
+ * To use in a process that already called `startSignalHandlerThread()`
+ * or `saveSignalMask()` first.
+ */
+void restoreSignals();
+
+void triggerInterrupt();
+
+}
+
+static inline void setInterrupted(bool isInterrupted)
+{
+    unix::_isInterrupted = isInterrupted;
+}
+
+static inline bool getInterrupted()
+{
+    return unix::_isInterrupted;
+}
+
+void inline checkInterrupt()
+{
+    using namespace unix;
+    if (_isInterrupted || (interruptCheck && interruptCheck()))
+        _interrupted();
+}
+
+/**
+ * A RAII class that causes the current thread to receive SIGUSR1 when
+ * the signal handler thread receives SIGINT. That is, this allows
+ * SIGINT to be multiplexed to multiple threads.
+ */
+struct ReceiveInterrupts
+{
+    pthread_t target;
+    std::unique_ptr<InterruptCallback> callback;
+
+    ReceiveInterrupts()
+        : target(pthread_self())
+        , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); }))
+    { }
+};
+
+
+}

src/libutil/unix/signals.cc

@@ -8,17 +8,22 @@
 
 namespace nix {
 
-std::atomic<bool> _isInterrupted = false;
+using namespace unix;
 
+std::atomic<bool> unix::_isInterrupted = false;
+
+namespace unix {
 static thread_local bool interruptThrown = false;
-thread_local std::function<bool()> interruptCheck;
+}
+
+thread_local std::function<bool()> unix::interruptCheck;
 
 void setInterruptThrown()
 {
-    interruptThrown = true;
+    unix::interruptThrown = true;
 }
 
-void _interrupted()
+void unix::_interrupted()
 {
     /* Block user interrupts while an exception is being handled.
        Throwing an exception while another exception is being handled
@@ -65,7 +70,7 @@ static void signalHandlerThread(sigset_t set)
     }
 }
 
-void triggerInterrupt()
+void unix::triggerInterrupt()
 {
     _isInterrupted = true;
 
@@ -96,7 +101,7 @@ void triggerInterrupt()
 static sigset_t savedSignalMask;
 static bool savedSignalMaskIsSet = false;
 
-void setChildSignalMask(sigset_t * sigs)
+void unix::setChildSignalMask(sigset_t * sigs)
 {
     assert(sigs); // C style function, but think of sigs as a reference
 
@@ -115,14 +120,14 @@ void setChildSignalMask(sigset_t * sigs)
     savedSignalMaskIsSet = true;
 }
 
-void saveSignalMask() {
+void unix::saveSignalMask() {
     if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
         throw SysError("querying signal mask");
 
     savedSignalMaskIsSet = true;
 }
 
-void startSignalHandlerThread()
+void unix::startSignalHandlerThread()
 {
     updateWindowSize();
 
@@ -141,7 +146,7 @@ void startSignalHandlerThread()
     std::thread(signalHandlerThread, set).detach();
 }
 
-void restoreSignals()
+void unix::restoreSignals()
 {
     // If startSignalHandlerThread wasn't called, that means we're not running
     // in a proper libmain process, but a process that presumably manages its

src/libutil/unix/users.cc

@@ -0,0 +1,66 @@
+#include "util.hh"
+#include "users.hh"
+#include "environment-variables.hh"
+#include "file-system.hh"
+
+#include <pwd.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+namespace nix {
+
+std::string getUserName()
+{
+    auto pw = getpwuid(geteuid());
+    std::string name = pw ? pw->pw_name : getEnv("USER").value_or("");
+    if (name.empty())
+        throw Error("cannot figure out user name");
+    return name;
+}
+
+Path getHomeOf(uid_t userId)
+{
+    std::vector<char> buf(16384);
+    struct passwd pwbuf;
+    struct passwd * pw;
+    if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0
+        || !pw || !pw->pw_dir || !pw->pw_dir[0])
+        throw Error("cannot determine user's home directory");
+    return pw->pw_dir;
+}
+
+Path getHome()
+{
+    static Path homeDir = []()
+    {
+        std::optional<std::string> unownedUserHomeDir = {};
+        auto homeDir = getEnv("HOME");
+        if (homeDir) {
+            // Only use $HOME if doesn't exist or is owned by the current user.
+            struct stat st;
+            int result = stat(homeDir->c_str(), &st);
+            if (result != 0) {
+                if (errno != ENOENT) {
+                    warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
+                    homeDir.reset();
+                }
+            } else if (st.st_uid != geteuid()) {
+                unownedUserHomeDir.swap(homeDir);
+            }
+        }
+        if (!homeDir) {
+            homeDir = getHomeOf(geteuid());
+            if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
+                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
+            }
+        }
+        return *homeDir;
+    }();
+    return homeDir;
+}
+
+bool isRootUser() {
+    return getuid() == 0;
+}
+
+}

src/libutil/users.cc

@@ -3,63 +3,8 @@
 #include "environment-variables.hh"
 #include "file-system.hh"
 
-#include <pwd.h>
-#include <sys/types.h>
-#include <unistd.h>
-
 namespace nix {
 
-std::string getUserName()
-{
-    auto pw = getpwuid(geteuid());
-    std::string name = pw ? pw->pw_name : getEnv("USER").value_or("");
-    if (name.empty())
-        throw Error("cannot figure out user name");
-    return name;
-}
-
-Path getHomeOf(uid_t userId)
-{
-    std::vector<char> buf(16384);
-    struct passwd pwbuf;
-    struct passwd * pw;
-    if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0
-        || !pw || !pw->pw_dir || !pw->pw_dir[0])
-        throw Error("cannot determine user's home directory");
-    return pw->pw_dir;
-}
-
-Path getHome()
-{
-    static Path homeDir = []()
-    {
-        std::optional<std::string> unownedUserHomeDir = {};
-        auto homeDir = getEnv("HOME");
-        if (homeDir) {
-            // Only use $HOME if doesn't exist or is owned by the current user.
-            struct stat st;
-            int result = stat(homeDir->c_str(), &st);
-            if (result != 0) {
-                if (errno != ENOENT) {
-                    warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
-                    homeDir.reset();
-                }
-            } else if (st.st_uid != geteuid()) {
-                unownedUserHomeDir.swap(homeDir);
-            }
-        }
-        if (!homeDir) {
-            homeDir = getHomeOf(geteuid());
-            if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
-                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
-            }
-        }
-        return *homeDir;
-    }();
-    return homeDir;
-}
-
-
 Path getCacheDir()
 {
     auto cacheDir = getEnv("XDG_CACHE_HOME");

src/libutil/users.hh

@@ -55,4 +55,10 @@ Path createNixStateDir();
  */
 std::string expandTilde(std::string_view path);
 
+
+/**
+ * Is the current user UID 0 on Unix?
+ */
+bool isRootUser();
+
 }

src/libutil/util.cc

@@ -4,7 +4,6 @@
 #include <array>
 #include <cctype>
 #include <iostream>
-#include <grp.h>
 #include <regex>
 
 #include <sodium.h>