Merge remote-tracking branch 'origin/master' into large-path-warning

src/libutil/archive.cc

@@ -315,13 +315,4 @@ void copyNAR(Source & source, Sink & sink)
 }
 
 
-void copyPath(const Path & from, const Path & to)
-{
-    auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(from, sink);
-    });
-    restorePath(to, *source);
-}
-
-
 }

src/libutil/archive.hh

@@ -82,8 +82,6 @@ void restorePath(const Path & path, Source & source);
  */
 void copyNAR(Source & source, Sink & sink);
 
-void copyPath(const Path & from, const Path & to);
-
 
 inline constexpr std::string_view narVersionMagic1 = "nix-archive-1";
 

src/libutil/args.cc

@@ -268,8 +268,6 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
         verbosity = lvlError;
     }
 
-    bool argsSeen = false;
-
     // Heuristic to see if we're invoked as a shebang script, namely,
     // if we have at least one argument, it's the name of an
     // executable file, and it starts with "#!".
@@ -336,10 +334,6 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
                 throw UsageError("unrecognised flag '%1%'", arg);
         }
         else {
-            if (!argsSeen) {
-                argsSeen = true;
-                initialFlagsProcessed();
-            }
             pos = rewriteArgs(cmdline, pos);
             pendingArgs.push_back(*pos++);
             if (processArgs(pendingArgs, false))
@@ -349,8 +343,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
 
     processArgs(pendingArgs, true);
 
-    if (!argsSeen)
-        initialFlagsProcessed();
+    initialFlagsProcessed();
 
     /* Now that we are done parsing, make sure that any experimental
      * feature required by the flags is enabled */

src/libutil/compression.cc

@@ -263,8 +263,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink
             checkInterrupt();
 
             if (!BrotliEncoderCompressStream(
-                    state, data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in,
-                    &avail_out, &next_out, nullptr))
+                    state,
+                    data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
+                    &avail_in,
+                    &next_in,
+                    &avail_out,
+                    &next_out,
+                    nullptr))
                 throw CompressionError("error while compressing brotli compression");
 
             if (avail_out < sizeof(outbuf) || avail_in == 0) {
@@ -280,8 +285,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
 {
-    std::vector<std::string> la_supports = {"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4",
-                                            "lzip",  "lzma",     "lzop",  "xz",   "zstd"};
+    std::vector<std::string> la_supports = {
+        "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"};
     if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
         return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
     }

src/libutil/current-process.cc

@@ -7,6 +7,7 @@
 #include "file-system.hh"
 #include "processes.hh"
 #include "signals.hh"
+#include <math.h>
 
 #ifdef __APPLE__
 # include <mach-o/dyld.h>
@@ -59,15 +60,15 @@ unsigned int getMaxCPU()
 //////////////////////////////////////////////////////////////////////
 
 
-#ifndef _WIN32
-rlim_t savedStackSize = 0;
+size_t savedStackSize = 0;
 
-void setStackSize(rlim_t stackSize)
+void setStackSize(size_t stackSize)
 {
+    #ifndef _WIN32
     struct rlimit limit;
     if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
         savedStackSize = limit.rlim_cur;
-        limit.rlim_cur = std::min(stackSize, limit.rlim_max);
+        limit.rlim_cur = std::min(static_cast<rlim_t>(stackSize), limit.rlim_max);
         if (setrlimit(RLIMIT_STACK, &limit) != 0) {
             logger->log(
                 lvlError,
@@ -81,8 +82,31 @@ void setStackSize(rlim_t stackSize)
             );
         }
     }
+    #else
+    ULONG_PTR stackLow, stackHigh;
+    GetCurrentThreadStackLimits(&stackLow, &stackHigh);
+    ULONG maxStackSize = stackHigh - stackLow;
+    ULONG currStackSize = 0;
+    // This retrieves the current promised stack size
+    SetThreadStackGuarantee(&currStackSize);
+    if (currStackSize < stackSize) {
+        savedStackSize = currStackSize;
+        ULONG newStackSize = std::min(static_cast<ULONG>(stackSize), maxStackSize);
+        if (SetThreadStackGuarantee(&newStackSize) == 0) {
+            logger->log(
+                lvlError,
+                HintFmt(
+                    "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%",
+                    savedStackSize,
+                    stackSize,
+                    maxStackSize,
+                    std::to_string(GetLastError())
+                ).str()
+            );
+        }
+    }
+    #endif
 }
-#endif
 
 void restoreProcessContext(bool restoreMounts)
 {

src/libutil/current-process.hh

@@ -17,12 +17,10 @@ namespace nix {
  */
 unsigned int getMaxCPU();
 
-#ifndef _WIN32 // TODO implement on Windows, if needed.
 /**
  * Change the stack size.
  */
-void setStackSize(rlim_t stackSize);
-#endif
+void setStackSize(size_t stackSize);
 
 /**
  * Restore the original inherited Unix process context (such as signal

src/libutil/error.hh

@@ -206,11 +206,11 @@ MakeError(SystemError, Error);
  *
  * Throw this, but prefer not to catch this, and catch `SystemError`
  * instead. This allows implementations to freely switch between this
- * and `WinError` without breaking catch blocks.
+ * and `windows::WinError` without breaking catch blocks.
  *
  * However, it is permissible to catch this and rethrow so long as
  * certain conditions are not met (e.g. to catch only if `errNo =
- * EFooBar`). In that case, try to also catch the equivalent `WinError`
+ * EFooBar`). In that case, try to also catch the equivalent `windows::WinError`
  * code.
  *
  * @todo Rename this to `PosixError` or similar. At this point Windows
@@ -248,7 +248,9 @@ public:
 };
 
 #ifdef _WIN32
-class WinError;
+namespace windows {
+    class WinError;
+}
 #endif
 
 /**
@@ -258,7 +260,7 @@ class WinError;
  */
 using NativeSysError =
 #ifdef _WIN32
-    WinError
+    windows::WinError
 #else
     SysError
 #endif

src/libutil/file-content-address.hh

@@ -12,16 +12,28 @@ struct SourcePath;
 /**
  * An enumeration of the ways we can serialize file system
  * objects.
+ *
+ * See `file-system-object/content-address.md#serial` in the manual for
+ * a user-facing description of this concept, but note that this type is also
+ * used for storing or sending copies; not just for addressing.
+ * Note also that there are other content addressing methods that don't
+ * correspond to a serialisation method.
  */
 enum struct FileSerialisationMethod : uint8_t {
     /**
      * Flat-file. The contents of a single file exactly.
+     *
+     * See `file-system-object/content-address.md#serial-flat` in the
+     * manual.
      */
     Flat,
 
     /**
      * Nix Archive. Serializes the file-system object in
      * Nix Archive format.
+     *
+     * See `file-system-object/content-address.md#serial-nix-archive` in
+     * the manual.
      */
     Recursive,
 };
@@ -81,33 +93,32 @@ HashResult hashPath(
 /**
  * An enumeration of the ways we can ingest file system
  * objects, producing a hash or digest.
+ *
+ * See `file-system-object/content-address.md` in the manual for a
+ * user-facing description of this concept.
  */
 enum struct FileIngestionMethod : uint8_t {
     /**
      * Hash `FileSerialisationMethod::Flat` serialisation.
+     *
+     * See `file-system-object/content-address.md#serial-flat` in the
+     * manual.
      */
     Flat,
 
     /**
-     * Hash `FileSerialisationMethod::Git` serialisation.
+     * Hash `FileSerialisationMethod::Recursive` serialisation.
+     *
+     * See `file-system-object/content-address.md#serial-flat` in the
+     * manual.
      */
     Recursive,
 
     /**
-     * Git hashing. In particular files are hashed as git "blobs", and
-     * directories are hashed as git "trees".
+     * Git hashing.
      *
-     * Unlike `Flat` and `Recursive`, this is not a hash of a single
-     * serialisation but a [Merkle
-     * DAG](https://en.wikipedia.org/wiki/Merkle_tree) of multiple
-     * rounds of serialisation and hashing.
-     *
-     * @note Git's data model is slightly different, in that a plain
-     * file doesn't have an executable bit, directory entries do
-     * instead. We decide treat a bare file as non-executable by fiat,
-     * as we do with `FileIngestionMethod::Flat` which also lacks this
-     * information. Thus, Git can encode some but all of Nix's "File
-     * System Objects", and this sort of hashing is likewise partial.
+     * See `file-system-object/content-address.md#serial-git` in the
+     * manual.
      */
     Git,
 };

src/libutil/file-system.cc

@@ -222,26 +222,6 @@ Path readLink(const Path & path)
 }
 
 
-std::vector<fs::directory_entry> readDirectory(const Path & path)
-{
-    std::vector<fs::directory_entry> entries;
-    entries.reserve(64);
-
-    for (auto & entry : fs::directory_iterator{path}) {
-        checkInterrupt();
-        entries.push_back(std::move(entry));
-    }
-
-    return entries;
-}
-
-
-fs::file_type getFileType(const Path & path)
-{
-    return fs::symlink_status(path).type();
-}
-
-
 std::string readFile(const Path & path)
 {
     AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY
@@ -588,7 +568,7 @@ void replaceSymlink(const Path & target, const Path & link)
             throw;
         }
 
-        renameFile(tmp, link);
+        std::filesystem::rename(tmp, link);
 
         break;
     }
@@ -611,29 +591,29 @@ static void setWriteTime(const fs::path & p, const struct stat & st)
 }
 #endif
 
-void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
+void copyFile(const fs::path & from, const fs::path & to, bool andDelete)
 {
 #ifndef _WIN32
     // TODO: Rewrite the `is_*` to use `symlink_status()`
-    auto statOfFrom = lstat(from.path().c_str());
+    auto statOfFrom = lstat(from.c_str());
 #endif
-    auto fromStatus = from.symlink_status();
+    auto fromStatus = fs::symlink_status(from);
 
     // Mark the directory as writable so that we can delete its children
     if (andDelete && fs::is_directory(fromStatus)) {
-        fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+        fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
     }
 
 
     if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
-        fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
+        fs::copy(from, to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
     } else if (fs::is_directory(fromStatus)) {
         fs::create_directory(to);
-        for (auto & entry : fs::directory_iterator(from.path())) {
-            copy(entry, to / entry.path().filename(), andDelete);
+        for (auto & entry : fs::directory_iterator(from)) {
+            copyFile(entry, to / entry.path().filename(), andDelete);
         }
     } else {
-        throw Error("file '%s' has an unsupported type", from.path());
+        throw Error("file '%s' has an unsupported type", from);
     }
 
 #ifndef _WIN32
@@ -641,25 +621,15 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
 #endif
     if (andDelete) {
         if (!fs::is_symlink(fromStatus))
-            fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
-        fs::remove(from.path());
+            fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+        fs::remove(from);
     }
 }
 
-void copyFile(const Path & oldPath, const Path & newPath, bool andDelete)
-{
-    return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), andDelete);
-}
-
-void renameFile(const Path & oldName, const Path & newName)
-{
-    fs::rename(oldName, newName);
-}
-
 void moveFile(const Path & oldName, const Path & newName)
 {
     try {
-        renameFile(oldName, newName);
+        std::filesystem::rename(oldName, newName);
     } catch (fs::filesystem_error & e) {
         auto oldPath = fs::path(oldName);
         auto newPath = fs::path(newName);
@@ -673,8 +643,8 @@ void moveFile(const Path & oldName, const Path & newName)
         if (e.code().value() == EXDEV) {
             fs::remove(newPath);
             warn("Can’t rename %s as %s, copying instead", oldName, newName);
-            copy(fs::directory_entry(oldPath), tempCopyTarget, true);
-            renameFile(
+            copyFile(oldPath, tempCopyTarget, true);
+            std::filesystem::rename(
                 os_string_to_string(PathViewNG { tempCopyTarget }),
                 os_string_to_string(PathViewNG { newPath }));
         }

src/libutil/file-system.hh

@@ -20,8 +20,6 @@
 #endif
 #include <signal.h>
 
-#include <boost/lexical_cast.hpp>
-
 #include <atomic>
 #include <functional>
 #include <map>
@@ -122,14 +120,6 @@ Path readLink(const Path & path);
  */
 Descriptor openDirectory(const std::filesystem::path & path);
 
-/**
- * Read the contents of a directory.  The entries `.` and `..` are
- * removed.
- */
-std::vector<std::filesystem::directory_entry> readDirectory(const Path & path);
-
-std::filesystem::file_type getFileType(const Path & path);
-
 /**
  * Read the contents of a file into a string.
  */
@@ -177,8 +167,6 @@ void createSymlink(const Path & target, const Path & link);
  */
 void replaceSymlink(const Path & target, const Path & link);
 
-void renameFile(const Path & src, const Path & dst);
-
 /**
  * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
  * are on a different filesystem.
@@ -194,7 +182,7 @@ void moveFile(const Path & src, const Path & dst);
  * with the guaranty that the destination will be “fresh”, with no stale inode
  * or file descriptor pointing to it).
  */
-void copyFile(const Path & oldPath, const Path & newPath, bool andDelete);
+void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete);
 
 /**
  * Automatic cleanup of resources.

src/libutil/fmt.hh

@@ -182,6 +182,8 @@ public:
         return *this;
     }
 
+    HintFmt & operator=(HintFmt const & rhs) = default;
+
     std::string str() const
     {
         return fmt.str();

src/libutil/json-utils.cc

@@ -39,12 +39,9 @@ std::optional<nlohmann::json> optionalValueAt(const nlohmann::json::object_t & m
 }
 
 
-std::optional<nlohmann::json> getNullable(const nlohmann::json & value)
+const nlohmann::json * getNullable(const nlohmann::json & value)
 {
-    if (value.is_null())
-        return std::nullopt;
-
-    return value.get<nlohmann::json>();
+    return value.is_null() ? nullptr : &value;
 }
 
 /**

src/libutil/json-utils.hh

@@ -29,7 +29,7 @@ std::optional<nlohmann::json> optionalValueAt(const nlohmann::json::object_t & v
  * Downcast the json object, failing with a nice error if the conversion fails.
  * See https://json.nlohmann.me/features/types/
  */
-std::optional<nlohmann::json> getNullable(const nlohmann::json & value);
+const nlohmann::json * getNullable(const nlohmann::json & value);
 const nlohmann::json::object_t & getObject(const nlohmann::json & value);
 const nlohmann::json::array_t & getArray(const nlohmann::json & value);
 const nlohmann::json::string_t & getString(const nlohmann::json & value);

src/libutil/linux/cgroup.cc

@@ -64,7 +64,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu
 
     /* Otherwise, manually kill every process in the subcgroups and
        this cgroup. */
-    for (auto & entry : readDirectory(cgroup)) {
+    for (auto & entry : std::filesystem::directory_iterator{cgroup}) {
         if (entry.symlink_status().type() != std::filesystem::file_type::directory) continue;
         destroyCgroup(cgroup / entry.path().filename(), false);
     }

src/libutil/linux/namespaces.cc

@@ -137,10 +137,10 @@ void restoreMountNamespace()
     }
 }
 
-void unshareFilesystem()
+void tryUnshareFilesystem()
 {
-    if (unshare(CLONE_FS) != 0 && errno != EPERM)
-        throw SysError("unsharing filesystem state in download thread");
+    if (unshare(CLONE_FS) != 0 && errno != EPERM && errno != ENOSYS)
+        throw SysError("unsharing filesystem state");
 }
 
 }

src/libutil/linux/namespaces.hh

@@ -20,11 +20,13 @@ void saveMountNamespace();
 void restoreMountNamespace();
 
 /**
- * Cause this thread to not share any FS attributes with the main
+ * Cause this thread to try to not share any FS attributes with the main
  * thread, because this causes setns() in restoreMountNamespace() to
  * fail.
+ *
+ * This is best effort -- EPERM and ENOSYS failures are just ignored.
  */
-void unshareFilesystem();
+void tryUnshareFilesystem();
 
 bool userNamespacesSupported();
 

src/libutil/logging.cc

@@ -8,6 +8,7 @@
 #include "position.hh"
 
 #include <atomic>
+#include <sstream>
 #include <nlohmann/json.hpp>
 #include <iostream>
 

src/libutil/muxable-pipe.hh

@@ -0,0 +1,82 @@
+#pragma once
+///@file
+
+#include "file-descriptor.hh"
+#ifdef _WIN32
+#  include "windows-async-pipe.hh"
+#endif
+
+#ifndef _WIN32
+#  include <poll.h>
+#else
+#  include <ioapiset.h>
+#  include "windows-error.hh"
+#endif
+
+namespace nix {
+
+/**
+ * An "muxable pipe" is a type of pipe supporting endpoints that wait
+ * for events on multiple pipes at once.
+ *
+ * On Unix, this is just a regular anonymous pipe. On Windows, this has
+ * to be a named pipe because we need I/O Completion Ports to wait on
+ * multiple pipes.
+ */
+using MuxablePipe =
+#ifndef _WIN32
+    Pipe
+#else
+    windows::AsyncPipe
+#endif
+    ;
+
+/**
+ * Use pool() (Unix) / I/O Completion Ports (Windows) to wait for the
+ * input side of any logger pipe to become `available'.  Note that
+ * `available' (i.e., non-blocking) includes EOF.
+ */
+struct MuxablePipePollState
+{
+#ifndef _WIN32
+    std::vector<struct pollfd> pollStatus;
+    std::map<int, size_t> fdToPollStatus;
+#else
+    OVERLAPPED_ENTRY oentries[0x20] = {0};
+    ULONG removed;
+    bool gotEOF = false;
+
+#endif
+
+    /**
+     * Check for ready (Unix) / completed (Windows) operations
+     */
+    void poll(
+#ifdef _WIN32
+        HANDLE ioport,
+#endif
+        std::optional<unsigned int> timeout);
+
+    using CommChannel =
+#ifndef _WIN32
+        Descriptor
+#else
+        windows::AsyncPipe *
+#endif
+        ;
+
+    /**
+     * Process for ready (Unix) / completed (Windows) operations,
+     * calling the callbacks as needed.
+     *
+     * @param handleRead callback to be passed read data.
+     *
+     * @param handleEOF callback for when the `MuxablePipe` has closed.
+     */
+    void iterate(
+        std::set<CommChannel> & channels,
+        std::function<void(Descriptor fd, std::string_view data)> handleRead,
+        std::function<void(Descriptor fd)> handleEOF);
+};
+
+}

src/libutil/posix-source-accessor.cc

@@ -132,7 +132,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath &
 {
     assertNoSymlinks(path);
     DirEntries res;
-    for (auto & entry : nix::readDirectory(makeAbsPath(path).string())) {
+    for (auto & entry : std::filesystem::directory_iterator{makeAbsPath(path)}) {
         auto type = [&]() -> std::optional<Type> {
             std::filesystem::file_type nativeType;
             try {

src/libutil/processes.hh

@@ -12,8 +12,6 @@
 #include <unistd.h>
 #include <signal.h>
 
-#include <boost/lexical_cast.hpp>
-
 #include <atomic>
 #include <functional>
 #include <map>
@@ -118,8 +116,6 @@ public:
     { }
 };
 
-#ifndef _WIN32
-
 /**
  * Convert the exit status of a child as returned by wait() into an
  * error string.
@@ -128,6 +124,4 @@ std::string statusToString(int status);
 
 bool statusOk(int status);
 
-#endif
-
 }

src/libutil/ref.hh

@@ -77,6 +77,8 @@ public:
         return ref<T2>((std::shared_ptr<T2>) p);
     }
 
+    ref<T> & operator=(ref<T> const & rhs) = default;
+
     bool operator == (const ref<T> & other) const
     {
         return p == other.p;

src/libutil/serialise.cc

@@ -136,7 +136,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len)
     checkInterrupt();
     if (!::ReadFile(fd, data, len, &n, NULL)) {
         _good = false;
-        throw WinError("ReadFile when FdSource::readUnbuffered");
+        throw windows::WinError("ReadFile when FdSource::readUnbuffered");
     }
 #else
     ssize_t n;

src/libutil/source-accessor.cc

@@ -53,7 +53,7 @@ SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
     if (auto st = maybeLstat(path))
         return *st;
     else
-        throw Error("path '%s' does not exist", showPath(path));
+        throw FileNotFound("path '%s' does not exist", showPath(path));
 }
 
 void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix)

src/libutil/source-accessor.hh

@@ -29,6 +29,8 @@ enum class SymlinkResolution {
     Full,
 };
 
+MakeError(FileNotFound, Error);
+
 /**
  * A read-only filesystem abstraction. This is used by the Nix
  * evaluator and elsewhere for accessing sources in various

src/libutil/terminal.cc

@@ -4,6 +4,8 @@
 
 #if _WIN32
 # include <io.h>
+# define WIN32_LEAN_AND_MEAN
+# include <windows.h>
 # define isatty _isatty
 #else
 # include <sys/ioctl.h>
@@ -97,17 +99,26 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 
 
-#ifndef _WIN32
 void updateWindowSize()
 {
+    #ifndef _WIN32
     struct winsize ws;
     if (ioctl(2, TIOCGWINSZ, &ws) == 0) {
         auto windowSize_(windowSize.lock());
         windowSize_->first = ws.ws_row;
         windowSize_->second = ws.ws_col;
     }
+    #else
+    CONSOLE_SCREEN_BUFFER_INFO info;
+    // From https://stackoverflow.com/a/12642749
+    if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info) != 0) {
+        auto windowSize_(windowSize.lock());
+        // From https://github.com/libuv/libuv/blob/v1.48.0/src/win/tty.c#L1130
+        windowSize_->first = info.srWindow.Bottom - info.srWindow.Top + 1;
+        windowSize_->second = info.dwSize.X;
+    }
+    #endif
 }
-#endif
 
 
 std::pair<unsigned short, unsigned short> getWindowSize()

src/libutil/terminal.hh

@@ -21,16 +21,13 @@ std::string filterANSIEscapes(std::string_view s,
     bool filterAll = false,
     unsigned int width = std::numeric_limits<unsigned int>::max());
 
-#ifndef _WIN32
-
 /**
- * Recalculate the window size, updating a global variable. Used in the
- * `SIGWINCH` signal handler.
+ * Recalculate the window size, updating a global variable.
+ *
+ * Used in the `SIGWINCH` signal handler on Unix, for example.
  */
 void updateWindowSize();
 
-#endif
-
 /**
  * @return the number of rows and columns of the terminal.
  *

src/libutil/unix/file-descriptor.cc

@@ -124,7 +124,7 @@ void closeMostFDs(const std::set<int> & exceptions)
 {
 #if __linux__
     try {
-        for (auto & s : readDirectory("/proc/self/fd")) {
+        for (auto & s : std::filesystem::directory_iterator{"/proc/self/fd"}) {
             auto fd = std::stoi(s.path().filename());
             if (!exceptions.count(fd)) {
                 debug("closing leaked FD %d", fd);

src/libutil/unix/muxable-pipe.cc

@@ -0,0 +1,47 @@
+#include <poll.h>
+
+#include "logging.hh"
+#include "util.hh"
+#include "muxable-pipe.hh"
+
+namespace nix {
+
+void MuxablePipePollState::poll(std::optional<unsigned int> timeout)
+{
+    if (::poll(pollStatus.data(), pollStatus.size(), timeout ? *timeout : -1) == -1) {
+        if (errno == EINTR)
+            return;
+        throw SysError("waiting for input");
+    }
+}
+
+void MuxablePipePollState::iterate(
+    std::set<MuxablePipePollState::CommChannel> & channels,
+    std::function<void(Descriptor fd, std::string_view data)> handleRead,
+    std::function<void(Descriptor fd)> handleEOF)
+{
+    std::set<Descriptor> fds2(channels);
+    std::vector<unsigned char> buffer(4096);
+    for (auto & k : fds2) {
+        const auto fdPollStatusId = get(fdToPollStatus, k);
+        assert(fdPollStatusId);
+        assert(*fdPollStatusId < pollStatus.size());
+        if (pollStatus.at(*fdPollStatusId).revents) {
+            ssize_t rd = ::read(fromDescriptorReadOnly(k), buffer.data(), buffer.size());
+            // FIXME: is there a cleaner way to handle pt close
+            // than EIO? Is this even standard?
+            if (rd == 0 || (rd == -1 && errno == EIO)) {
+                handleEOF(k);
+                channels.erase(k);
+            } else if (rd == -1) {
+                if (errno != EINTR)
+                    throw SysError("read failed");
+            } else {
+                std::string_view data((char *) buffer.data(), rd);
+                handleRead(k, data);
+            }
+        }
+    }
+}
+
+}

src/libutil/url.hh

@@ -33,6 +33,8 @@ std::string percentEncode(std::string_view s, std::string_view keep="");
 
 std::map<std::string, std::string> decodeQuery(const std::string & query);
 
+std::string encodeQuery(const std::map<std::string, std::string> & query);
+
 ParsedURL parseURL(const std::string & url);
 
 /**

src/libutil/util.cc

@@ -7,6 +7,8 @@
 #include <regex>
 
 #include <sodium.h>
+#include <boost/lexical_cast.hpp>
+#include <stdint.h>
 
 #ifdef NDEBUG
 #error "Nix may not be built with assertions disabled (i.e. with -DNDEBUG)."
@@ -111,6 +113,43 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites)
     return s;
 }
 
+template<class N>
+std::optional<N> string2Int(const std::string_view s)
+{
+    if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
+        return std::nullopt;
+    try {
+        return boost::lexical_cast<N>(s.data(), s.size());
+    } catch (const boost::bad_lexical_cast &) {
+        return std::nullopt;
+    }
+}
+
+// Explicitly instantiated in one place for faster compilation
+template std::optional<unsigned char>  string2Int<unsigned char>(const std::string_view s);
+template std::optional<unsigned short> string2Int<unsigned short>(const std::string_view s);
+template std::optional<unsigned int> string2Int<unsigned int>(const std::string_view s);
+template std::optional<unsigned long> string2Int<unsigned long>(const std::string_view s);
+template std::optional<unsigned long long> string2Int<unsigned long long>(const std::string_view s);
+template std::optional<signed char> string2Int<signed char>(const std::string_view s);
+template std::optional<signed short> string2Int<signed short>(const std::string_view s);
+template std::optional<signed int> string2Int<signed int>(const std::string_view s);
+template std::optional<signed long> string2Int<signed long>(const std::string_view s);
+template std::optional<signed long long> string2Int<signed long long>(const std::string_view s);
+
+template<class N>
+std::optional<N> string2Float(const std::string_view s)
+{
+    try {
+        return boost::lexical_cast<N>(s.data(), s.size());
+    } catch (const boost::bad_lexical_cast &) {
+        return std::nullopt;
+    }
+}
+
+template std::optional<double> string2Float<double>(const std::string_view s);
+template std::optional<float> string2Float<float>(const std::string_view s);
+
 
 std::string renderSize(uint64_t value, bool align)
 {

src/libutil/util.hh

@@ -5,7 +5,6 @@
 #include "error.hh"
 #include "logging.hh"
 
-#include <boost/lexical_cast.hpp>
 
 #include <functional>
 #include <map>
@@ -102,16 +101,7 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites);
  * Parse a string into an integer.
  */
 template<class N>
-std::optional<N> string2Int(const std::string_view s)
-{
-    if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
-        return std::nullopt;
-    try {
-        return boost::lexical_cast<N>(s.data(), s.size());
-    } catch (const boost::bad_lexical_cast &) {
-        return std::nullopt;
-    }
-}
+std::optional<N> string2Int(const std::string_view s);
 
 /**
  * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
@@ -148,14 +138,7 @@ std::string renderSize(uint64_t value, bool align = false);
  * Parse a string into a float.
  */
 template<class N>
-std::optional<N> string2Float(const std::string_view s)
-{
-    try {
-        return boost::lexical_cast<N>(s.data(), s.size());
-    } catch (const boost::bad_lexical_cast &) {
-        return std::nullopt;
-    }
-}
+std::optional<N> string2Float(const std::string_view s);
 
 
 /**

src/libutil/windows/file-descriptor.cc

@@ -14,6 +14,8 @@
 
 namespace nix {
 
+using namespace nix::windows;
+
 std::string readFile(HANDLE handle)
 {
     LARGE_INTEGER li;

src/libutil/windows/file-system.cc

@@ -5,8 +5,13 @@ namespace nix {
 Descriptor openDirectory(const std::filesystem::path & path)
 {
     return CreateFileW(
-        path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING,
-        FILE_FLAG_BACKUP_SEMANTICS, NULL);
+        path.c_str(),
+        GENERIC_READ,
+        FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+        NULL,
+        OPEN_EXISTING,
+        FILE_FLAG_BACKUP_SEMANTICS,
+        NULL);
 }
 
 }

src/libutil/windows/muxable-pipe.cc

@@ -0,0 +1,70 @@
+#include <ioapiset.h>
+#include "windows-error.hh"
+
+#include "logging.hh"
+#include "util.hh"
+#include "muxable-pipe.hh"
+
+namespace nix {
+
+void MuxablePipePollState::poll(HANDLE ioport, std::optional<unsigned int> timeout)
+{
+    /* We are on at least Windows Vista / Server 2008 and can get many
+       (countof(oentries)) statuses in one API call. */
+    if (!GetQueuedCompletionStatusEx(
+            ioport, oentries, sizeof(oentries) / sizeof(*oentries), &removed, timeout ? *timeout : INFINITE, false)) {
+        windows::WinError winError("GetQueuedCompletionStatusEx");
+        if (winError.lastError != WAIT_TIMEOUT)
+            throw winError;
+        assert(removed == 0);
+    } else {
+        assert(0 < removed && removed <= sizeof(oentries) / sizeof(*oentries));
+    }
+}
+
+void MuxablePipePollState::iterate(
+    std::set<MuxablePipePollState::CommChannel> & channels,
+    std::function<void(Descriptor fd, std::string_view data)> handleRead,
+    std::function<void(Descriptor fd)> handleEOF)
+{
+    auto p = channels.begin();
+    while (p != channels.end()) {
+        decltype(p) nextp = p;
+        ++nextp;
+        for (ULONG i = 0; i < removed; i++) {
+            if (oentries[i].lpCompletionKey == ((ULONG_PTR) ((*p)->readSide.get()) ^ 0x5555)) {
+                printMsg(lvlVomit, "read %s bytes", oentries[i].dwNumberOfBytesTransferred);
+                if (oentries[i].dwNumberOfBytesTransferred > 0) {
+                    std::string data{
+                        (char *) (*p)->buffer.data(),
+                        oentries[i].dwNumberOfBytesTransferred,
+                    };
+                    handleRead((*p)->readSide.get(), data);
+                }
+
+                if (gotEOF) {
+                    handleEOF((*p)->readSide.get());
+                    nextp = channels.erase(p); // no need to maintain `channels`?
+                } else {
+                    BOOL rc = ReadFile(
+                        (*p)->readSide.get(), (*p)->buffer.data(), (*p)->buffer.size(), &(*p)->got, &(*p)->overlapped);
+                    if (rc) {
+                        // here is possible (but not obligatory) to call
+                        // `handleRead` and repeat ReadFile immediately
+                    } else {
+                        windows::WinError winError("ReadFile(%s, ..)", (*p)->readSide.get());
+                        if (winError.lastError == ERROR_BROKEN_PIPE) {
+                            handleEOF((*p)->readSide.get());
+                            nextp = channels.erase(p); // no need to maintain `channels` ?
+                        } else if (winError.lastError != ERROR_IO_PENDING)
+                            throw winError;
+                    }
+                }
+                break;
+            }
+        }
+        p = nextp;
+    }
+}
+
+}

src/libutil/windows/processes.cc

@@ -16,16 +16,6 @@
 #include <sys/types.h>
 #include <unistd.h>
 
-#ifdef __APPLE__
-# include <sys/syscall.h>
-#endif
-
-#ifdef __linux__
-# include <sys/prctl.h>
-# include <sys/mman.h>
-#endif
-
-
 namespace nix {
 
 std::string runProgram(Path program, bool lookupPath, const Strings & args,
@@ -34,15 +24,31 @@ std::string runProgram(Path program, bool lookupPath, const Strings & args,
     throw UnimplementedError("Cannot shell out to git on Windows yet");
 }
 
+
 // Output = error code + "standard out" output stream
 std::pair<int, std::string> runProgram(RunOptions && options)
 {
     throw UnimplementedError("Cannot shell out to git on Windows yet");
 }
 
+
 void runProgram2(const RunOptions & options)
 {
     throw UnimplementedError("Cannot shell out to git on Windows yet");
 }
 
+std::string statusToString(int status)
+{
+    if (status != 0)
+        return fmt("with exit code %d", status);
+    else
+        return "succeeded";
+}
+
+
+bool statusOk(int status)
+{
+    return status == 0;
+}
+
 }

src/libutil/windows/users.cc

@@ -9,6 +9,8 @@
 
 namespace nix {
 
+using namespace nix::windows;
+
 std::string getUserName()
 {
     // Get the required buffer size

src/libutil/windows/windows-async-pipe.cc

@@ -0,0 +1,49 @@
+#include "windows-async-pipe.hh"
+#include "windows-error.hh"
+
+namespace nix::windows {
+
+void AsyncPipe::createAsyncPipe(HANDLE iocp)
+{
+    // std::cerr << (format("-----AsyncPipe::createAsyncPipe(%x)") % iocp) << std::endl;
+
+    buffer.resize(0x1000);
+    memset(&overlapped, 0, sizeof(overlapped));
+
+    std::string pipeName = fmt("\\\\.\\pipe\\nix-%d-%p", GetCurrentProcessId(), (void *) this);
+
+    readSide = CreateNamedPipeA(
+        pipeName.c_str(),
+        PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED,
+        PIPE_TYPE_BYTE,
+        PIPE_UNLIMITED_INSTANCES,
+        0,
+        0,
+        INFINITE,
+        NULL);
+    if (!readSide)
+        throw WinError("CreateNamedPipeA(%s)", pipeName);
+
+    HANDLE hIocp = CreateIoCompletionPort(readSide.get(), iocp, (ULONG_PTR) (readSide.get()) ^ 0x5555, 0);
+    if (hIocp != iocp)
+        throw WinError("CreateIoCompletionPort(%x[%s], %x, ...) returned %x", readSide.get(), pipeName, iocp, hIocp);
+
+    if (!ConnectNamedPipe(readSide.get(), &overlapped) && GetLastError() != ERROR_IO_PENDING)
+        throw WinError("ConnectNamedPipe(%s)", pipeName);
+
+    SECURITY_ATTRIBUTES psa2 = {0};
+    psa2.nLength = sizeof(SECURITY_ATTRIBUTES);
+    psa2.bInheritHandle = TRUE;
+
+    writeSide = CreateFileA(pipeName.c_str(), GENERIC_WRITE, 0, &psa2, OPEN_EXISTING, 0, NULL);
+    if (!readSide)
+        throw WinError("CreateFileA(%s)", pipeName);
+}
+
+void AsyncPipe::close()
+{
+    readSide.close();
+    writeSide.close();
+}
+
+}

src/libutil/windows/windows-async-pipe.hh

@@ -0,0 +1,27 @@
+#pragma once
+///@file
+
+#include "file-descriptor.hh"
+
+namespace nix::windows {
+
+/***
+ * An "async pipe" is a pipe that supports I/O Completion Ports so
+ * multiple pipes can be listened too.
+ *
+ * Unfortunately, only named pipes support that on windows, so we use
+ * those with randomized temp file names.
+ */
+class AsyncPipe
+{
+public:
+    AutoCloseFD writeSide, readSide;
+    OVERLAPPED overlapped;
+    DWORD got;
+    std::vector<unsigned char> buffer;
+
+    void createAsyncPipe(HANDLE iocp);
+    void close();
+};
+
+}

src/libutil/windows/windows-error.cc

@@ -4,7 +4,7 @@
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 
-namespace nix {
+namespace nix::windows {
 
 std::string WinError::renderError(DWORD lastError)
 {

src/libutil/windows/windows-error.hh

@@ -5,7 +5,7 @@
 
 #include "error.hh"
 
-namespace nix {
+namespace nix::windows {
 
 /**
  * Windows Error type.