Merge remote-tracking branch 'upstream/master' into overlayfs-store

src/libutil/abstract-setting-to-json.hh

@@ -7,7 +7,7 @@
 
 namespace nix {
 template<typename T>
-std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject()
+std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject() const
 {
     auto obj = AbstractSetting::toJSONObject();
     obj.emplace("value", value);

src/libutil/archive.cc

@@ -5,16 +5,11 @@
 
 #include <strings.h> // for strcasecmp
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <dirent.h>
-#include <fcntl.h>
-
 #include "archive.hh"
-#include "util.hh"
 #include "config.hh"
 #include "posix-source-accessor.hh"
+#include "file-system.hh"
+#include "signals.hh"
 
 namespace nix {
 
@@ -299,7 +294,7 @@ void copyNAR(Source & source, Sink & sink)
     // FIXME: if 'source' is the output of dumpPath() followed by EOF,
     // we should just forward all data directly without parsing.
 
-    ParseSink parseSink; /* null sink; just parse the NAR */
+    NullParseSink parseSink; /* just parse the NAR */
 
     TeeSource wrapper { source, sink };
 

src/libutil/archive.hh

@@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
  */
 void dumpString(std::string_view s, Sink & sink);
 
-/**
- * If the NAR archive contains a single file at top-level, then save
- * the contents of the file to `s`.  Otherwise barf.
- */
-struct RetrieveRegularNARSink : ParseSink
-{
-    bool regular = true;
-    Sink & sink;
-
-    RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
-
-    void createDirectory(const Path & path) override
-    {
-        regular = false;
-    }
-
-    void receiveContents(std::string_view data) override
-    {
-        sink(data);
-    }
-
-    void createSymlink(const Path & path, const std::string & target) override
-    {
-        regular = false;
-    }
-};
-
 void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);

src/libutil/args.cc

@@ -1,8 +1,14 @@
 #include "args.hh"
 #include "args/root.hh"
 #include "hash.hh"
+#include "environment-variables.hh"
+#include "signals.hh"
+#include "users.hh"
 #include "json-utils.hh"
 
+#include <fstream>
+#include <string>
+#include <regex>
 #include <glob.h>
 
 namespace nix {
@@ -74,7 +80,178 @@ std::optional<std::string> RootArgs::needsCompletion(std::string_view s)
     return {};
 }
 
-void RootArgs::parseCmdline(const Strings & _cmdline)
+/**
+ * Basically this is `typedef std::optional<Parser> Parser(std::string_view s, Strings & r);`
+ *
+ * Except we can't recursively reference the Parser typedef, so we have to write a class.
+ */
+struct Parser {
+    std::string_view remaining;
+
+    /**
+     * @brief Parse the next character(s)
+     *
+     * @param r
+     * @return std::shared_ptr<Parser>
+     */
+    virtual void operator()(std::shared_ptr<Parser> & state, Strings & r) = 0;
+
+    Parser(std::string_view s) : remaining(s) {};
+
+    virtual ~Parser() { };
+};
+
+struct ParseQuoted : public Parser {
+    /**
+     * @brief Accumulated string
+     *
+     * Parsed argument up to this point.
+     */
+    std::string acc;
+
+    ParseQuoted(std::string_view s) : Parser(s) {};
+
+    virtual void operator()(std::shared_ptr<Parser> & state, Strings & r) override;
+};
+
+
+struct ParseUnquoted : public Parser {
+    /**
+     * @brief Accumulated string
+     *
+     * Parsed argument up to this point. Empty string is not representable in
+     * unquoted syntax, so we use it for the initial state.
+     */
+    std::string acc;
+
+    ParseUnquoted(std::string_view s) : Parser(s) {};
+
+    virtual void operator()(std::shared_ptr<Parser> & state, Strings & r) override {
+        if (remaining.empty()) {
+            if (!acc.empty())
+                r.push_back(acc);
+            state = nullptr; // done
+            return;
+        }
+        switch (remaining[0]) {
+            case ' ': case '\t': case '\n': case '\r':
+                if (!acc.empty())
+                    r.push_back(acc);
+                state = std::make_shared<ParseUnquoted>(ParseUnquoted(remaining.substr(1)));
+                return;
+            case '`':
+                if (remaining.size() > 1 && remaining[1] == '`') {
+                    state = std::make_shared<ParseQuoted>(ParseQuoted(remaining.substr(2)));
+                    return;
+                }
+                else
+                    throw Error("single backtick is not a supported syntax in the nix shebang.");
+
+            // reserved characters
+            // meaning to be determined, or may be reserved indefinitely so that
+            // #!nix syntax looks unambiguous
+            case '$':
+            case '*':
+            case '~':
+            case '<':
+            case '>':
+            case '|':
+            case ';':
+            case '(':
+            case ')':
+            case '[':
+            case ']':
+            case '{':
+            case '}':
+            case '\'':
+            case '"':
+            case '\\':
+                throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?");
+
+            case '#':
+                if (acc.empty()) {
+                    throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?");
+                } else {
+                    acc += remaining[0];
+                    remaining = remaining.substr(1);
+                    return;
+                }
+
+            default:
+                acc += remaining[0];
+                remaining = remaining.substr(1);
+                return;
+        }
+        assert(false);
+    }
+};
+
+void ParseQuoted::operator()(std::shared_ptr<Parser> &state, Strings & r) {
+    if (remaining.empty()) {
+        throw Error("unterminated quoted string in nix shebang");
+    }
+    switch (remaining[0]) {
+        case ' ':
+            if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`')
+                || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) {
+                // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present.
+                state = std::make_shared<ParseUnquoted>(ParseUnquoted(remaining.substr(3)));
+                r.push_back(acc);
+                return;
+            }
+            else {
+                // just a normal space
+                acc += remaining[0];
+                remaining = remaining.substr(1);
+                return;
+            }
+        case '`':
+            // exactly two backticks mark the end of a quoted string
+            if ((remaining.size() == 2 && remaining[1] == '`')
+                || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) {
+                state = std::make_shared<ParseUnquoted>(ParseUnquoted(remaining.substr(2)));
+                r.push_back(acc);
+                return;
+            }
+
+            // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim
+            else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') {
+                // ignore "escape" backtick
+                remaining = remaining.substr(1);
+                // add the rest
+                while (remaining.size() > 0 && remaining[0] == '`') {
+                    acc += '`';
+                    remaining = remaining.substr(1);
+                }
+                return;
+            }
+            else {
+                acc += remaining[0];
+                remaining = remaining.substr(1);
+                return;
+            }
+        default:
+            acc += remaining[0];
+            remaining = remaining.substr(1);
+            return;
+    }
+    assert(false);
+}
+
+Strings parseShebangContent(std::string_view s) {
+    Strings result;
+    std::shared_ptr<Parser> parserState(std::make_shared<ParseUnquoted>(ParseUnquoted(s)));
+
+    // trampoline == iterated strategy pattern
+    while (parserState) {
+        auto currentState = parserState;
+        (*currentState)(parserState, result);
+    }
+
+    return result;
+}
+
+void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
 {
     Strings pendingArgs;
     bool dashDash = false;
@@ -90,6 +267,45 @@ void RootArgs::parseCmdline(const Strings & _cmdline)
     }
 
     bool argsSeen = false;
+
+    // Heuristic to see if we're invoked as a shebang script, namely,
+    // if we have at least one argument, it's the name of an
+    // executable file, and it starts with "#!".
+    Strings savedArgs;
+    if (allowShebang){
+        auto script = *cmdline.begin();
+        try {
+            std::ifstream stream(script);
+            char shebang[3]={0,0,0};
+            stream.get(shebang,3);
+            if (strncmp(shebang,"#!",2) == 0){
+                for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++)
+                    savedArgs.push_back(*pos);
+                cmdline.clear();
+
+                std::string line;
+                std::getline(stream,line);
+                static const std::string commentChars("#/\\%@*-");
+                std::string shebangContent;
+                while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){
+                    line = chomp(line);
+
+                    std::smatch match;
+                    // We match one space after `nix` so that we preserve indentation.
+                    // No space is necessary for an empty line. An empty line has basically no effect.
+                    if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$")))
+                        shebangContent += match[2].str() + "\n";
+                }
+                for (const auto & word : parseShebangContent(shebangContent)) {
+                    cmdline.push_back(word);
+                }
+                cmdline.push_back(script);
+                commandBaseDir = dirOf(script);
+                for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++)
+                    cmdline.push_back(*pos);
+            }
+        } catch (SysError &) { }
+    }
     for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
 
         auto arg = *pos;
@@ -145,6 +361,17 @@ void RootArgs::parseCmdline(const Strings & _cmdline)
         d.completer(*completions, d.n, d.prefix);
 }
 
+Path Args::getCommandBaseDir() const
+{
+    assert(parent);
+    return parent->getCommandBaseDir();
+}
+
+Path RootArgs::getCommandBaseDir() const
+{
+    return commandBaseDir;
+}
+
 bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
 {
     assert(pos != end);
@@ -255,7 +482,18 @@ bool Args::processArgs(const Strings & args, bool finish)
         }
         if (!anyCompleted)
             exp.handler.fun(ss);
-        expectedArgs.pop_front();
+
+        /* Move the list element to the processedArgs. This is almost the same as
+           `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`,
+           except that it will only adjust the next and prev pointers of the list
+           elements, meaning the actual contents don't move in memory. This is
+           critical to prevent invalidating internal pointers! */
+        processedArgs.splice(
+            processedArgs.end(),
+            expectedArgs,
+            expectedArgs.begin(),
+            ++expectedArgs.begin());
+
         res = true;
     }
 
@@ -306,36 +544,70 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
-static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
-    for (auto & type : hashTypes)
-        if (hasPrefix(type, prefix))
-            completions.add(type);
+    for (auto & format : hashFormats) {
+        if (hasPrefix(format, prefix)) {
+            completions.add(format);
+        }
+    }
 }
 
-Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
-{
-    return Flag {
-        .longName = std::move(longName),
-        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
-        .labels = {"hash-algo"},
-        .handler = {[ht](std::string s) {
-            *ht = parseHashType(s);
-        }},
-        .completer = hashTypeCompleter,
+Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) {
+    assert(*hf == nix::HashFormat::SRI);
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'",
+            .labels = {"hash-format"},
+            .handler = {[hf](std::string s) {
+                *hf = parseHashFormat(s);
+            }},
+            .completer = hashFormatCompleter,
     };
 }
 
-Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht)
+Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf) {
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash format ('base16', 'nix32', 'base64', 'sri').",
+            .labels = {"hash-format"},
+            .handler = {[ohf](std::string s) {
+                *ohf = std::optional<HashFormat>{parseHashFormat(s)};
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
-    return Flag {
-        .longName = std::move(longName),
-        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
-        .labels = {"hash-algo"},
-        .handler = {[oht](std::string s) {
-            *oht = std::optional<HashType> { parseHashType(s) };
-        }},
-        .completer = hashTypeCompleter,
+    for (auto & algo : hashAlgorithms)
+        if (hasPrefix(algo, prefix))
+            completions.add(algo);
+}
+
+Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha)
+{
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
+            .labels = {"hash-algo"},
+            .handler = {[ha](std::string s) {
+                *ha = parseHashAlgo(s);
+            }},
+            .completer = hashAlgoCompleter,
+    };
+}
+
+Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
+{
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
+            .labels = {"hash-algo"},
+            .handler = {[oha](std::string s) {
+                *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
+            }},
+            .completer = hashAlgoCompleter,
     };
 }
 
@@ -384,8 +656,9 @@ std::optional<ExperimentalFeature> Command::experimentalFeature ()
     return { Xp::NixCommand };
 }
 
-MultiCommand::MultiCommand(const Commands & commands_)
+MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_)
     : commands(commands_)
+    , commandName(commandName)
 {
     expectArgs({
         .label = "subcommand",

src/libutil/args.hh

@@ -2,16 +2,20 @@
 ///@file
 
 #include <iostream>
+#include <functional>
 #include <map>
 #include <memory>
+#include <optional>
 
 #include <nlohmann/json_fwd.hpp>
 
-#include "util.hh"
+#include "types.hh"
+#include "experimental-features.hh"
 
 namespace nix {
 
-enum HashType : char;
+enum struct HashAlgorithm : char;
+enum struct HashFormat : int;
 
 class MultiCommand;
 
@@ -21,11 +25,12 @@ class AddCompletions;
 
 class Args
 {
+
 public:
 
     /**
      * Return a short one-line description of the command.
-     */
+    */
     virtual std::string description() { return ""; }
 
     virtual bool forceImpureByDefault() { return false; }
@@ -35,6 +40,16 @@ public:
      */
     virtual std::string doc() { return ""; }
 
+    /**
+     * @brief Get the base directory for the command.
+     *
+     * @return Generally the working directory, but in case of a shebang
+     *         interpreter, returns the directory of the script.
+     *
+     * This only returns the correct value after parseCmdline() has run.
+     */
+    virtual Path getCommandBaseDir() const;
+
 protected:
 
     /**
@@ -161,8 +176,10 @@ protected:
 
         std::optional<ExperimentalFeature> experimentalFeature;
 
-        static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
-        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
+        static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
+        static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
+        static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf);
+        static Flag mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf);
     };
 
     /**
@@ -200,13 +217,25 @@ protected:
     /**
      * Queue of expected positional argument forms.
      *
-     * Positional arugment descriptions are inserted on the back.
+     * Positional argument descriptions are inserted on the back.
      *
      * As positional arguments are passed, these are popped from the
      * front, until there are hopefully none left as all args that were
      * expected in fact were passed.
      */
     std::list<ExpectedArg> expectedArgs;
+    /**
+     * List of processed positional argument forms.
+     *
+     * All items removed from `expectedArgs` are added here. After all
+     * arguments were processed, this list should be exactly the same as
+     * `expectedArgs` was before.
+     *
+     * This list is used to extend the lifetime of the argument forms.
+     * If this is not done, some closures that reference the command
+     * itself will segfault.
+    */
+   std::list<ExpectedArg> processedArgs;
 
     /**
      * Process some positional arugments
@@ -330,13 +359,16 @@ public:
      */
     std::optional<std::pair<std::string, ref<Command>>> command;
 
-    MultiCommand(const Commands & commands);
+    MultiCommand(std::string_view commandName, const Commands & commands);
 
     bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
 
     bool processArgs(const Strings & args, bool finish) override;
 
     nlohmann::json toJSON() override;
+
+protected:
+    std::string commandName = "";
 };
 
 Strings argvToStrings(int argc, char * * argv);
@@ -383,4 +415,6 @@ public:
     virtual void add(std::string completion, std::string description = "") = 0;
 };
 
+Strings parseShebangContent(std::string_view s);
+
 }

src/libutil/args/root.hh

@@ -29,14 +29,26 @@ struct Completions final : AddCompletions
  */
 class RootArgs : virtual public Args
 {
+    /**
+     * @brief The command's "working directory", but only set when top level.
+     *
+     * Use getCommandBaseDir() to get the directory regardless of whether this
+     * is a top-level command or subcommand.
+     *
+     * @see getCommandBaseDir()
+     */
+    Path commandBaseDir = ".";
+
 public:
     /** Parse the command line, throwing a UsageError if something goes
      * wrong.
      */
-    void parseCmdline(const Strings & cmdline);
+    void parseCmdline(const Strings & cmdline, bool allowShebang = false);
 
     std::shared_ptr<Completions> completions;
 
+    Path getCommandBaseDir() const override;
+
 protected:
 
     friend class Args;

src/libutil/canon-path.cc

@@ -1,5 +1,5 @@
 #include "canon-path.hh"
-#include "util.hh"
+#include "file-system.hh"
 
 namespace nix {
 
@@ -13,6 +13,13 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
     : path(absPath((Path) raw, root.abs()))
 { }
 
+CanonPath::CanonPath(const std::vector<std::string> & elems)
+    : path("/")
+{
+    for (auto & s : elems)
+        push(s);
+}
+
 CanonPath CanonPath::fromCwd(std::string_view path)
 {
     return CanonPath(unchecked_t(), absPath((Path) path));

src/libutil/canon-path.hh

@@ -6,6 +6,7 @@
 #include <cassert>
 #include <iostream>
 #include <set>
+#include <vector>
 
 namespace nix {
 
@@ -46,6 +47,11 @@ public:
         : path(std::move(path))
     { }
 
+    /**
+     * Construct a canon path from a vector of elements.
+     */
+    CanonPath(const std::vector<std::string> & elems);
+
     static CanonPath fromCwd(std::string_view path = ".");
 
     static CanonPath root;
@@ -199,8 +205,19 @@ public:
      * `CanonPath(this.makeRelative(x), this) == path`.
      */
     std::string makeRelative(const CanonPath & path) const;
+
+    friend class std::hash<CanonPath>;
 };
 
 std::ostream & operator << (std::ostream & stream, const CanonPath & path);
 
 }
+
+template<>
+struct std::hash<nix::CanonPath>
+{
+    std::size_t operator ()(const nix::CanonPath & s) const noexcept
+    {
+        return std::hash<std::string>{}(s.path);
+    }
+};

src/libutil/cgroup.cc

@@ -2,6 +2,7 @@
 
 #include "cgroup.hh"
 #include "util.hh"
+#include "file-system.hh"
 #include "finally.hh"
 
 #include <chrono>

src/libutil/compression.cc

@@ -1,6 +1,6 @@
 #include "compression.hh"
+#include "signals.hh"
 #include "tarfile.hh"
-#include "util.hh"
 #include "finally.hh"
 #include "logging.hh"
 

src/libutil/config-impl.hh

@@ -45,13 +45,13 @@ bool BaseSetting<T>::isAppendable()
     return trait::appendable;
 }
 
-template<> void BaseSetting<Strings>::appendOrSet(Strings && newValue, bool append);
-template<> void BaseSetting<StringSet>::appendOrSet(StringSet && newValue, bool append);
-template<> void BaseSetting<StringMap>::appendOrSet(StringMap && newValue, bool append);
-template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> && newValue, bool append);
+template<> void BaseSetting<Strings>::appendOrSet(Strings newValue, bool append);
+template<> void BaseSetting<StringSet>::appendOrSet(StringSet newValue, bool append);
+template<> void BaseSetting<StringMap>::appendOrSet(StringMap newValue, bool append);
+template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> newValue, bool append);
 
 template<typename T>
-void BaseSetting<T>::appendOrSet(T && newValue, bool append)
+void BaseSetting<T>::appendOrSet(T newValue, bool append)
 {
     static_assert(
         !trait::appendable,

src/libutil/config.cc

@@ -2,6 +2,8 @@
 #include "args.hh"
 #include "abstract-setting-to-json.hh"
 #include "experimental-features.hh"
+#include "util.hh"
+#include "file-system.hh"
 
 #include "config-impl.hh"
 
@@ -34,27 +36,25 @@ bool Config::set(const std::string & name, const std::string & value)
 void Config::addSetting(AbstractSetting * setting)
 {
     _settings.emplace(setting->name, Config::SettingData{false, setting});
-    for (auto & alias : setting->aliases)
+    for (const auto & alias : setting->aliases)
         _settings.emplace(alias, Config::SettingData{true, setting});
 
     bool set = false;
 
-    auto i = unknownSettings.find(setting->name);
-    if (i != unknownSettings.end()) {
-        setting->set(i->second);
+    if (auto i = unknownSettings.find(setting->name); i != unknownSettings.end()) {
+        setting->set(std::move(i->second));
         setting->overridden = true;
         unknownSettings.erase(i);
         set = true;
     }
 
     for (auto & alias : setting->aliases) {
-        auto i = unknownSettings.find(alias);
-        if (i != unknownSettings.end()) {
+        if (auto i = unknownSettings.find(alias); i != unknownSettings.end()) {
             if (set)
                 warn("setting '%s' is set, but it's an alias of '%s' which is also set",
                     alias, setting->name);
             else {
-                setting->set(i->second);
+                setting->set(std::move(i->second));
                 setting->overridden = true;
                 unknownSettings.erase(i);
                 set = true;
@@ -69,7 +69,7 @@ AbstractConfig::AbstractConfig(StringMap initials)
 
 void AbstractConfig::warnUnknownSettings()
 {
-    for (auto & s : unknownSettings)
+    for (const auto & s : unknownSettings)
         warn("unknown setting '%s'", s.first);
 }
 
@@ -83,15 +83,14 @@ void AbstractConfig::reapplyUnknownSettings()
 
 void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
-    for (auto & opt : _settings)
+    for (const auto & opt : _settings)
         if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
-void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
-    unsigned int pos = 0;
 
-    std::vector<std::pair<std::string, std::string>> parsedContents;
+static void applyConfigInner(const std::string & contents, const std::string & path, std::vector<std::pair<std::string, std::string>> & parsedContents) {
+    unsigned int pos = 0;
 
     while (pos < contents.size()) {
         std::string line;
@@ -99,8 +98,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             line += contents[pos++];
         pos++;
 
-        auto hash = line.find('#');
-        if (hash != std::string::npos)
+        if (auto hash = line.find('#'); hash != line.npos)
             line = std::string(line, 0, hash);
 
         auto tokens = tokenizeString<std::vector<std::string>>(line);
@@ -123,7 +121,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
                 throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
             auto p = absPath(tokens[1], dirOf(path));
             if (pathExists(p)) {
-                applyConfigFile(p);
+                try {
+                    std::string includedContents = readFile(path);
+                    applyConfigInner(includedContents, p, parsedContents);
+                } catch (SysError &) {
+                    // TODO: Do we actually want to ignore this? Or is it better to fail?
+                }
             } else if (!ignoreMissing) {
                 throw Error("file '%1%' included from '%2%' not found", p, path);
             }
@@ -133,36 +136,34 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
         if (tokens[1] != "=")
             throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
 
-        std::string name = tokens[0];
+        std::string name = std::move(tokens[0]);
 
         auto i = tokens.begin();
         advance(i, 2);
 
         parsedContents.push_back({
-            name,
+            std::move(name),
             concatStringsSep(" ", Strings(i, tokens.end())),
         });
     };
+}
+
+void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
+    std::vector<std::pair<std::string, std::string>> parsedContents;
+
+    applyConfigInner(contents, path, parsedContents);
 
     // First apply experimental-feature related settings
-    for (auto & [name, value] : parsedContents)
+    for (const auto & [name, value] : parsedContents)
         if (name == "experimental-features" || name == "extra-experimental-features")
             set(name, value);
 
     // Then apply other settings
-    for (auto & [name, value] : parsedContents)
+    for (const auto & [name, value] : parsedContents)
         if (name != "experimental-features" && name != "extra-experimental-features")
             set(name, value);
 }
 
-void AbstractConfig::applyConfigFile(const Path & path)
-{
-    try {
-        std::string contents = readFile(path);
-        applyConfig(contents, path);
-    } catch (SysError &) { }
-}
-
 void Config::resetOverridden()
 {
     for (auto & s : _settings)
@@ -172,7 +173,7 @@ void Config::resetOverridden()
 nlohmann::json Config::toJSON()
 {
     auto res = nlohmann::json::object();
-    for (auto & s : _settings)
+    for (const auto & s : _settings)
         if (!s.second.isAlias)
             res.emplace(s.first, s.second.setting->toJSON());
     return res;
@@ -180,8 +181,8 @@ nlohmann::json Config::toJSON()
 
 std::string Config::toKeyValue()
 {
-    auto res = std::string();
-    for (auto & s : _settings)
+    std::string res;
+    for (const auto & s : _settings)
         if (s.second.isAlias)
             res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
     return res;
@@ -203,7 +204,7 @@ AbstractSetting::AbstractSetting(
     : name(name)
     , description(stripIndentation(description))
     , aliases(aliases)
-    , experimentalFeature(experimentalFeature)
+    , experimentalFeature(std::move(experimentalFeature))
 {
 }
 
@@ -219,7 +220,7 @@ nlohmann::json AbstractSetting::toJSON()
     return nlohmann::json(toJSONObject());
 }
 
-std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject()
+std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject() const
 {
     std::map<std::string, nlohmann::json> obj;
     obj.emplace("description", description);
@@ -282,14 +283,14 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
         .longName = name,
         .description = fmt("Enable the `%s` setting.", name),
         .category = category,
-        .handler = {[this]() { override(true); }},
+        .handler = {[this] { override(true); }},
         .experimentalFeature = experimentalFeature,
     });
     args.addFlag({
         .longName = "no-" + name,
         .description = fmt("Disable the `%s` setting.", name),
         .category = category,
-        .handler = {[this]() { override(false); }},
+        .handler = {[this] { override(false); }},
         .experimentalFeature = experimentalFeature,
     });
 }
@@ -299,10 +300,11 @@ template<> Strings BaseSetting<Strings>::parse(const std::string & str) const
     return tokenizeString<Strings>(str);
 }
 
-template<> void BaseSetting<Strings>::appendOrSet(Strings && newValue, bool append)
+template<> void BaseSetting<Strings>::appendOrSet(Strings newValue, bool append)
 {
     if (!append) value.clear();
-    for (auto && s : std::move(newValue)) value.push_back(std::move(s));
+    value.insert(value.end(), std::make_move_iterator(newValue.begin()),
+                              std::make_move_iterator(newValue.end()));
 }
 
 template<> std::string BaseSetting<Strings>::to_string() const
@@ -315,11 +317,10 @@ template<> StringSet BaseSetting<StringSet>::parse(const std::string & str) cons
     return tokenizeString<StringSet>(str);
 }
 
-template<> void BaseSetting<StringSet>::appendOrSet(StringSet && newValue, bool append)
+template<> void BaseSetting<StringSet>::appendOrSet(StringSet newValue, bool append)
 {
     if (!append) value.clear();
-    for (auto && s : std::move(newValue))
-        value.insert(s);
+    value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end()));
 }
 
 template<> std::string BaseSetting<StringSet>::to_string() const
@@ -331,26 +332,26 @@ template<> std::set<ExperimentalFeature> BaseSetting<std::set<ExperimentalFeatur
 {
     std::set<ExperimentalFeature> res;
     for (auto & s : tokenizeString<StringSet>(str)) {
-        auto thisXpFeature = parseExperimentalFeature(s);
-        if (thisXpFeature)
+        if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) {
             res.insert(thisXpFeature.value());
-        else
+            if (thisXpFeature.value() == Xp::Flakes)
+                res.insert(Xp::FetchTree);
+        } else
             warn("unknown experimental feature '%s'", s);
     }
     return res;
 }
 
-template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> && newValue, bool append)
+template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set<ExperimentalFeature> newValue, bool append)
 {
     if (!append) value.clear();
-    for (auto && s : std::move(newValue))
-        value.insert(s);
+    value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end()));
 }
 
 template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() const
 {
     StringSet stringifiedXpFeatures;
-    for (auto & feature : value)
+    for (const auto & feature : value)
         stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature)));
     return concatStringsSep(" ", stringifiedXpFeatures);
 }
@@ -358,28 +359,25 @@ template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() c
 template<> StringMap BaseSetting<StringMap>::parse(const std::string & str) const
 {
     StringMap res;
-    for (auto & s : tokenizeString<Strings>(str)) {
-        auto eq = s.find_first_of('=');
-        if (std::string::npos != eq)
+    for (const auto & s : tokenizeString<Strings>(str)) {
+        if (auto eq = s.find_first_of('='); s.npos != eq)
             res.emplace(std::string(s, 0, eq), std::string(s, eq + 1));
         // else ignored
     }
     return res;
 }
 
-template<> void BaseSetting<StringMap>::appendOrSet(StringMap && newValue, bool append)
+template<> void BaseSetting<StringMap>::appendOrSet(StringMap newValue, bool append)
 {
     if (!append) value.clear();
-    for (auto && [k, v] : std::move(newValue))
-        value.emplace(std::move(k), std::move(v));
+    value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end()));
 }
 
 template<> std::string BaseSetting<StringMap>::to_string() const
 {
-    Strings kvstrs;
-    std::transform(value.begin(), value.end(), back_inserter(kvstrs),
-        [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; });
-    return concatStringsSep(" ", kvstrs);
+    return std::transform_reduce(value.cbegin(), value.cend(), std::string{},
+        [](const auto & l, const auto  &r) { return l + " " + r; },
+        [](const auto & kvpair){ return kvpair.first + "=" + kvpair.second; });
 }
 
 template class BaseSetting<int>;
@@ -468,7 +466,7 @@ void GlobalConfig::resetOverridden()
 nlohmann::json GlobalConfig::toJSON()
 {
     auto res = nlohmann::json::object();
-    for (auto & config : *configRegistrations)
+    for (const auto & config : *configRegistrations)
         res.update(config->toJSON());
     return res;
 }
@@ -478,7 +476,7 @@ std::string GlobalConfig::toKeyValue()
     std::string res;
     std::map<std::string, Config::SettingInfo> settings;
     globalConfig.getSettings(settings);
-    for (auto & s : settings)
+    for (const auto & s : settings)
         res += fmt("%s = %s\n", s.first, s.second.value);
     return res;
 }

src/libutil/config.hh

@@ -82,12 +82,6 @@ public:
      */
     void applyConfig(const std::string & contents, const std::string & path = "<unknown>");
 
-    /**
-     * Applies a nix configuration file
-     * - path: the location of the config file to apply
-     */
-    void applyConfigFile(const Path & path);
-
     /**
      * Resets the `overridden` flag of all Settings
      */
@@ -150,7 +144,7 @@ public:
         AbstractSetting * setting;
     };
 
-    typedef std::map<std::string, SettingData> Settings;
+    using Settings = std::map<std::string, SettingData>;
 
 private:
 
@@ -213,7 +207,7 @@ protected:
 
     nlohmann::json toJSON();
 
-    virtual std::map<std::string, nlohmann::json> toJSONObject();
+    virtual std::map<std::string, nlohmann::json> toJSONObject() const;
 
     virtual void convertToArg(Args & args, const std::string & category);
 
@@ -247,7 +241,7 @@ protected:
      *
      * @param append Whether to append or overwrite.
      */
-    virtual void appendOrSet(T && newValue, bool append);
+    virtual void appendOrSet(T newValue, bool append);
 
 public:
 
@@ -306,7 +300,7 @@ public:
 
     void convertToArg(Args & args, const std::string & category) override;
 
-    std::map<std::string, nlohmann::json> toJSONObject() override;
+    std::map<std::string, nlohmann::json> toJSONObject() const override;
 };
 
 template<typename T>
@@ -316,7 +310,7 @@ std::ostream & operator <<(std::ostream & str, const BaseSetting<T> & opt)
 }
 
 template<typename T>
-bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == (const T &) v2; }
+bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == static_cast<const T &>(v2); }
 
 template<typename T>
 class Setting : public BaseSetting<T>
@@ -329,7 +323,7 @@ public:
         const std::set<std::string> & aliases = {},
         const bool documentDefault = true,
         std::optional<ExperimentalFeature> experimentalFeature = std::nullopt)
-        : BaseSetting<T>(def, documentDefault, name, description, aliases, experimentalFeature)
+        : BaseSetting<T>(def, documentDefault, name, description, aliases, std::move(experimentalFeature))
     {
         options->addSetting(this);
     }

src/libutil/current-process.cc

@@ -0,0 +1,110 @@
+#include "current-process.hh"
+#include "namespaces.hh"
+#include "util.hh"
+#include "finally.hh"
+#include "file-system.hh"
+#include "processes.hh"
+#include "signals.hh"
+
+#ifdef __APPLE__
+# include <mach-o/dyld.h>
+#endif
+
+#if __linux__
+# include <mutex>
+# include <sys/resource.h>
+# include "cgroup.hh"
+#endif
+
+#include <sys/mount.h>
+
+namespace nix {
+
+unsigned int getMaxCPU()
+{
+    #if __linux__
+    try {
+        auto cgroupFS = getCgroupFS();
+        if (!cgroupFS) return 0;
+
+        auto cgroups = getCgroups("/proc/self/cgroup");
+        auto cgroup = cgroups[""];
+        if (cgroup == "") return 0;
+
+        auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max";
+
+        auto cpuMax = readFile(cpuFile);
+        auto cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " \n");
+        auto quota = cpuMaxParts[0];
+        auto period = cpuMaxParts[1];
+        if (quota != "max")
+                return std::ceil(std::stoi(quota) / std::stof(period));
+    } catch (Error &) { ignoreException(lvlDebug); }
+    #endif
+
+    return 0;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+#if __linux__
+rlim_t savedStackSize = 0;
+#endif
+
+void setStackSize(size_t stackSize)
+{
+    #if __linux__
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
+        savedStackSize = limit.rlim_cur;
+        limit.rlim_cur = stackSize;
+        setrlimit(RLIMIT_STACK, &limit);
+    }
+    #endif
+}
+
+void restoreProcessContext(bool restoreMounts)
+{
+    restoreSignals();
+    if (restoreMounts) {
+        restoreMountNamespace();
+    }
+
+    #if __linux__
+    if (savedStackSize) {
+        struct rlimit limit;
+        if (getrlimit(RLIMIT_STACK, &limit) == 0) {
+            limit.rlim_cur = savedStackSize;
+            setrlimit(RLIMIT_STACK, &limit);
+        }
+    }
+    #endif
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+std::optional<Path> getSelfExe()
+{
+    static auto cached = []() -> std::optional<Path>
+    {
+        #if __linux__
+        return readLink("/proc/self/exe");
+        #elif __APPLE__
+        char buf[1024];
+        uint32_t size = sizeof(buf);
+        if (_NSGetExecutablePath(buf, &size) == 0)
+            return buf;
+        else
+            return std::nullopt;
+        #else
+        return std::nullopt;
+        #endif
+    }();
+    return cached;
+}
+
+}

src/libutil/current-process.hh

@@ -0,0 +1,34 @@
+#pragma once
+///@file
+
+#include <optional>
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * If cgroups are active, attempt to calculate the number of CPUs available.
+ * If cgroups are unavailable or if cpu.max is set to "max", return 0.
+ */
+unsigned int getMaxCPU();
+
+/**
+ * Change the stack size.
+ */
+void setStackSize(size_t stackSize);
+
+/**
+ * Restore the original inherited Unix process context (such as signal
+ * masks, stack size).
+
+ * See startSignalHandlerThread(), saveSignalMask().
+ */
+void restoreProcessContext(bool restoreMounts = true);
+
+/**
+ * @return the path of the current executable.
+ */
+std::optional<Path> getSelfExe();
+
+}

src/libutil/environment-variables.cc

@@ -0,0 +1,49 @@
+#include "util.hh"
+#include "environment-variables.hh"
+
+extern char * * environ __attribute__((weak));
+
+namespace nix {
+
+std::optional<std::string> getEnv(const std::string & key)
+{
+    char * value = getenv(key.c_str());
+    if (!value) return {};
+    return std::string(value);
+}
+
+std::optional<std::string> getEnvNonEmpty(const std::string & key) {
+    auto value = getEnv(key);
+    if (value == "") return {};
+    return value;
+}
+
+std::map<std::string, std::string> getEnv()
+{
+    std::map<std::string, std::string> env;
+    for (size_t i = 0; environ[i]; ++i) {
+        auto s = environ[i];
+        auto eq = strchr(s, '=');
+        if (!eq)
+            // invalid env, just keep going
+            continue;
+        env.emplace(std::string(s, eq), std::string(eq + 1));
+    }
+    return env;
+}
+
+
+void clearEnv()
+{
+    for (auto & name : getEnv())
+        unsetenv(name.first.c_str());
+}
+
+void replaceEnv(const std::map<std::string, std::string> & newEnv)
+{
+    clearEnv();
+    for (auto & newEnvVar : newEnv)
+        setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
+}
+
+}

src/libutil/environment-variables.hh

@@ -0,0 +1,41 @@
+#pragma once
+/**
+ * @file
+ *
+ * Utilities for working with the current process's environment
+ * variables.
+ */
+
+#include <optional>
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * @return an environment variable.
+ */
+std::optional<std::string> getEnv(const std::string & key);
+
+/**
+ * @return a non empty environment variable. Returns nullopt if the env
+ * variable is set to ""
+ */
+std::optional<std::string> getEnvNonEmpty(const std::string & key);
+
+/**
+ * Get the entire environment.
+ */
+std::map<std::string, std::string> getEnv();
+
+/**
+ * Clear the environment.
+ */
+void clearEnv();
+
+/**
+ * Replace the entire environment with the given one.
+ */
+void replaceEnv(const std::map<std::string, std::string> & newEnv);
+
+}

src/libutil/error.cc

@@ -1,4 +1,7 @@
 #include "error.hh"
+#include "environment-variables.hh"
+#include "signals.hh"
+#include "terminal.hh"
 
 #include <iostream>
 #include <optional>
@@ -7,8 +10,6 @@
 
 namespace nix {
 
-const std::string nativeSystem = SYSTEM;
-
 void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame)
 {
     err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
@@ -158,11 +159,11 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR
 /**
  * A development aid for finding missing positions, to improve error messages. Example use:
  *
- *     NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test
+ *     _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test
  *     git diff -U20 tests
  *
  */
-static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value();
+static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value();
 
 /**
  * Print a position, if it is known.

src/libutil/experimental-features.cc

@@ -12,7 +12,19 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 16> xpFeatureDetails = {{
+/**
+ * If two different PRs both add an experimental feature, and we just
+ * used a number for this, we *woudln't* get merge conflict and the
+ * counter will be incremented once instead of twice, causing a build
+ * failure.
+ *
+ * By instead defining this instead as 1 + the bottom experimental
+ * feature, we either have no issue at all if few features are not added
+ * at the end of the list, or a proper merge conflict if they are.
+ */
+constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::VerifiedFetches);
+
+constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",
@@ -62,6 +74,19 @@ constexpr std::array<ExperimentalFeatureDetails, 16> xpFeatureDetails = {{
             flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details.
         )",
     },
+    {
+        .tag = Xp::FetchTree,
+        .name = "fetch-tree",
+        .description = R"(
+            Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language.
+
+            `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources.
+            The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`.
+            This built-in was previously guarded by the `flakes` experimental feature because of that overlap.
+
+            Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation.
+        )",
+    },
     {
         .tag = Xp::NixCommand,
         .name = "nix-command",
@@ -70,6 +95,14 @@ constexpr std::array<ExperimentalFeatureDetails, 16> xpFeatureDetails = {{
             [`nix`](@docroot@/command-ref/new-cli/nix.md) for details.
         )",
     },
+    {
+        .tag = Xp::GitHashing,
+        .name = "git-hashing",
+        .description = R"(
+            Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm.
+            These store objects will not be understandable by older versions of Nix.
+        )",
+    },
     {
         .tag = Xp::RecursiveNix,
         .name = "recursive-nix",
@@ -218,7 +251,7 @@ constexpr std::array<ExperimentalFeatureDetails, 16> xpFeatureDetails = {{
         .tag = Xp::ReadOnlyLocalStore,
         .name = "read-only-local-store",
         .description = R"(
-            Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs.
+            Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs.
         )",
     },
     {
@@ -234,7 +267,21 @@ constexpr std::array<ExperimentalFeatureDetails, 16> xpFeatureDetails = {{
         .description = R"(
             Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting.
         )",
-    }
+    },
+    {
+        .tag = Xp::MountedSSHStore,
+        .name = "mounted-ssh-store",
+        .description = R"(
+            Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
+        )",
+    },
+    {
+        .tag = Xp::VerifiedFetches,
+        .name = "verified-fetches",
+        .description = R"(
+            Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in.
+        )",
+    },
 }};
 
 static_assert(

src/libutil/experimental-features.hh

@@ -20,7 +20,9 @@ enum struct ExperimentalFeature
     CaDerivations,
     ImpureDerivations,
     Flakes,
+    FetchTree,
     NixCommand,
+    GitHashing,
     RecursiveNix,
     NoUrlLiterals,
     FetchClosure,
@@ -33,6 +35,8 @@ enum struct ExperimentalFeature
     ReadOnlyLocalStore,
     LocalOverlayStore,
     ConfigurableImpureEnv,
+    MountedSSHStore,
+    VerifiedFetches,
 };
 
 /**

src/libutil/file-descriptor.cc

@@ -0,0 +1,254 @@
+#include "file-system.hh"
+#include "signals.hh"
+#include "finally.hh"
+#include "serialise.hh"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace nix {
+
+std::string readFile(int fd)
+{
+    struct stat st;
+    if (fstat(fd, &st) == -1)
+        throw SysError("statting file");
+
+    return drainFD(fd, true, st.st_size);
+}
+
+
+void readFull(int fd, char * buf, size_t count)
+{
+    while (count) {
+        checkInterrupt();
+        ssize_t res = read(fd, buf, count);
+        if (res == -1) {
+            if (errno == EINTR) continue;
+            throw SysError("reading from file");
+        }
+        if (res == 0) throw EndOfFile("unexpected end-of-file");
+        count -= res;
+        buf += res;
+    }
+}
+
+
+void writeFull(int fd, std::string_view s, bool allowInterrupts)
+{
+    while (!s.empty()) {
+        if (allowInterrupts) checkInterrupt();
+        ssize_t res = write(fd, s.data(), s.size());
+        if (res == -1 && errno != EINTR)
+            throw SysError("writing to file");
+        if (res > 0)
+            s.remove_prefix(res);
+    }
+}
+
+
+std::string readLine(int fd)
+{
+    std::string s;
+    while (1) {
+        checkInterrupt();
+        char ch;
+        // FIXME: inefficient
+        ssize_t rd = read(fd, &ch, 1);
+        if (rd == -1) {
+            if (errno != EINTR)
+                throw SysError("reading a line");
+        } else if (rd == 0)
+            throw EndOfFile("unexpected EOF reading a line");
+        else {
+            if (ch == '\n') return s;
+            s += ch;
+        }
+    }
+}
+
+
+void writeLine(int fd, std::string s)
+{
+    s += '\n';
+    writeFull(fd, s);
+}
+
+
+std::string drainFD(int fd, bool block, const size_t reserveSize)
+{
+    // the parser needs two extra bytes to append terminating characters, other users will
+    // not care very much about the extra memory.
+    StringSink sink(reserveSize + 2);
+    drainFD(fd, sink, block);
+    return std::move(sink.s);
+}
+
+
+void drainFD(int fd, Sink & sink, bool block)
+{
+    // silence GCC maybe-uninitialized warning in finally
+    int saved = 0;
+
+    if (!block) {
+        saved = fcntl(fd, F_GETFL);
+        if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
+            throw SysError("making file descriptor non-blocking");
+    }
+
+    Finally finally([&]() {
+        if (!block) {
+            if (fcntl(fd, F_SETFL, saved) == -1)
+                throw SysError("making file descriptor blocking");
+        }
+    });
+
+    std::vector<unsigned char> buf(64 * 1024);
+    while (1) {
+        checkInterrupt();
+        ssize_t rd = read(fd, buf.data(), buf.size());
+        if (rd == -1) {
+            if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
+                break;
+            if (errno != EINTR)
+                throw SysError("reading from file");
+        }
+        else if (rd == 0) break;
+        else sink({(char *) buf.data(), (size_t) rd});
+    }
+}
+
+//////////////////////////////////////////////////////////////////////
+
+AutoCloseFD::AutoCloseFD() : fd{-1} {}
+
+
+AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
+
+
+AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
+{
+    that.fd = -1;
+}
+
+
+AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
+{
+    close();
+    fd = that.fd;
+    that.fd = -1;
+    return *this;
+}
+
+
+AutoCloseFD::~AutoCloseFD()
+{
+    try {
+        close();
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+
+int AutoCloseFD::get() const
+{
+    return fd;
+}
+
+
+void AutoCloseFD::close()
+{
+    if (fd != -1) {
+        if (::close(fd) == -1)
+            /* This should never happen. */
+            throw SysError("closing file descriptor %1%", fd);
+        fd = -1;
+    }
+}
+
+void AutoCloseFD::fsync()
+{
+  if (fd != -1) {
+      int result;
+#if __APPLE__
+      result = ::fcntl(fd, F_FULLFSYNC);
+#else
+      result = ::fsync(fd);
+#endif
+      if (result == -1)
+          throw SysError("fsync file descriptor %1%", fd);
+  }
+}
+
+
+AutoCloseFD::operator bool() const
+{
+    return fd != -1;
+}
+
+
+int AutoCloseFD::release()
+{
+    int oldFD = fd;
+    fd = -1;
+    return oldFD;
+}
+
+
+void Pipe::create()
+{
+    int fds[2];
+#if HAVE_PIPE2
+    if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
+#else
+    if (pipe(fds) != 0) throw SysError("creating pipe");
+    closeOnExec(fds[0]);
+    closeOnExec(fds[1]);
+#endif
+    readSide = fds[0];
+    writeSide = fds[1];
+}
+
+
+void Pipe::close()
+{
+    readSide.close();
+    writeSide.close();
+}
+
+//////////////////////////////////////////////////////////////////////
+
+void closeMostFDs(const std::set<int> & exceptions)
+{
+#if __linux__
+    try {
+        for (auto & s : readDirectory("/proc/self/fd")) {
+            auto fd = std::stoi(s.name);
+            if (!exceptions.count(fd)) {
+                debug("closing leaked FD %d", fd);
+                close(fd);
+            }
+        }
+        return;
+    } catch (SysError &) {
+    }
+#endif
+
+    int maxFD = 0;
+    maxFD = sysconf(_SC_OPEN_MAX);
+    for (int fd = 0; fd < maxFD; ++fd)
+        if (!exceptions.count(fd))
+            close(fd); /* ignore result */
+}
+
+
+void closeOnExec(int fd)
+{
+    int prev;
+    if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
+        fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
+        throw SysError("setting close-on-exec flag");
+}
+
+}

src/libutil/file-descriptor.hh

@@ -0,0 +1,84 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "error.hh"
+
+namespace nix {
+
+struct Sink;
+struct Source;
+
+/**
+ * Read the contents of a resource into a string.
+ */
+std::string readFile(int fd);
+
+/**
+ * Wrappers arount read()/write() that read/write exactly the
+ * requested number of bytes.
+ */
+void readFull(int fd, char * buf, size_t count);
+
+void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
+
+/**
+ * Read a line from a file descriptor.
+ */
+std::string readLine(int fd);
+
+/**
+ * Write a line to a file descriptor.
+ */
+void writeLine(int fd, std::string s);
+
+/**
+ * Read a file descriptor until EOF occurs.
+ */
+std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
+
+void drainFD(int fd, Sink & sink, bool block = true);
+
+/**
+ * Automatic cleanup of resources.
+ */
+class AutoCloseFD
+{
+    int fd;
+public:
+    AutoCloseFD();
+    AutoCloseFD(int fd);
+    AutoCloseFD(const AutoCloseFD & fd) = delete;
+    AutoCloseFD(AutoCloseFD&& fd);
+    ~AutoCloseFD();
+    AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
+    AutoCloseFD& operator =(AutoCloseFD&& fd);
+    int get() const;
+    explicit operator bool() const;
+    int release();
+    void close();
+    void fsync();
+};
+
+class Pipe
+{
+public:
+    AutoCloseFD readSide, writeSide;
+    void create();
+    void close();
+};
+
+/**
+ * Close all file descriptors except those listed in the given set.
+ * Good practice in child processes.
+ */
+void closeMostFDs(const std::set<int> & exceptions);
+
+/**
+ * Set the close-on-exec flag for the given file descriptor.
+ */
+void closeOnExec(int fd);
+
+MakeError(EndOfFile, Error);
+
+}

src/libutil/file-system.cc

@@ -0,0 +1,647 @@
+#include "environment-variables.hh"
+#include "file-system.hh"
+#include "signals.hh"
+#include "finally.hh"
+#include "serialise.hh"
+
+#include <atomic>
+#include <cerrno>
+#include <climits>
+#include <cstdio>
+#include <cstdlib>
+#include <sstream>
+#include <filesystem>
+
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+namespace fs = std::filesystem;
+
+namespace nix {
+
+Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
+{
+    if (path[0] != '/') {
+        if (!dir) {
+#ifdef __GNU__
+            /* GNU (aka. GNU/Hurd) doesn't have any limitation on path
+               lengths and doesn't define `PATH_MAX'.  */
+            char *buf = getcwd(NULL, 0);
+            if (buf == NULL)
+#else
+            char buf[PATH_MAX];
+            if (!getcwd(buf, sizeof(buf)))
+#endif
+                throw SysError("cannot get cwd");
+            path = concatStrings(buf, "/", path);
+#ifdef __GNU__
+            free(buf);
+#endif
+        } else
+            path = concatStrings(*dir, "/", path);
+    }
+    return canonPath(path, resolveSymlinks);
+}
+
+
+Path canonPath(PathView path, bool resolveSymlinks)
+{
+    assert(path != "");
+
+    std::string s;
+    s.reserve(256);
+
+    if (path[0] != '/')
+        throw Error("not an absolute path: '%1%'", path);
+
+    std::string temp;
+
+    /* Count the number of times we follow a symlink and stop at some
+       arbitrary (but high) limit to prevent infinite loops. */
+    unsigned int followCount = 0, maxFollow = 1024;
+
+    while (1) {
+
+        /* Skip slashes. */
+        while (!path.empty() && path[0] == '/') path.remove_prefix(1);
+        if (path.empty()) break;
+
+        /* Ignore `.'. */
+        if (path == "." || path.substr(0, 2) == "./")
+            path.remove_prefix(1);
+
+        /* If `..', delete the last component. */
+        else if (path == ".." || path.substr(0, 3) == "../")
+        {
+            if (!s.empty()) s.erase(s.rfind('/'));
+            path.remove_prefix(2);
+        }
+
+        /* Normal component; copy it. */
+        else {
+            s += '/';
+            if (const auto slash = path.find('/'); slash == std::string::npos) {
+                s += path;
+                path = {};
+            } else {
+                s += path.substr(0, slash);
+                path = path.substr(slash);
+            }
+
+            /* If s points to a symlink, resolve it and continue from there */
+            if (resolveSymlinks && isLink(s)) {
+                if (++followCount >= maxFollow)
+                    throw Error("infinite symlink recursion in path '%1%'", path);
+                temp = concatStrings(readLink(s), path);
+                path = temp;
+                if (!temp.empty() && temp[0] == '/') {
+                    s.clear();  /* restart for symlinks pointing to absolute path */
+                } else {
+                    s = dirOf(s);
+                    if (s == "/") {  // we don’t want trailing slashes here, which dirOf only produces if s = /
+                        s.clear();
+                    }
+                }
+            }
+        }
+    }
+
+    return s.empty() ? "/" : std::move(s);
+}
+
+
+Path dirOf(const PathView path)
+{
+    Path::size_type pos = path.rfind('/');
+    if (pos == std::string::npos)
+        return ".";
+    return pos == 0 ? "/" : Path(path, 0, pos);
+}
+
+
+std::string_view baseNameOf(std::string_view path)
+{
+    if (path.empty())
+        return "";
+
+    auto last = path.size() - 1;
+    if (path[last] == '/' && last > 0)
+        last -= 1;
+
+    auto pos = path.rfind('/', last);
+    if (pos == std::string::npos)
+        pos = 0;
+    else
+        pos += 1;
+
+    return path.substr(pos, last - pos + 1);
+}
+
+
+bool isInDir(std::string_view path, std::string_view dir)
+{
+    return path.substr(0, 1) == "/"
+        && path.substr(0, dir.size()) == dir
+        && path.size() >= dir.size() + 2
+        && path[dir.size()] == '/';
+}
+
+
+bool isDirOrInDir(std::string_view path, std::string_view dir)
+{
+    return path == dir || isInDir(path, dir);
+}
+
+
+struct stat stat(const Path & path)
+{
+    struct stat st;
+    if (stat(path.c_str(), &st))
+        throw SysError("getting status of '%1%'", path);
+    return st;
+}
+
+
+struct stat lstat(const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+        throw SysError("getting status of '%1%'", path);
+    return st;
+}
+
+
+bool pathExists(const Path & path)
+{
+    int res;
+    struct stat st;
+    res = lstat(path.c_str(), &st);
+    if (!res) return true;
+    if (errno != ENOENT && errno != ENOTDIR)
+        throw SysError("getting status of %1%", path);
+    return false;
+}
+
+bool pathAccessible(const Path & path)
+{
+    try {
+        return pathExists(path);
+    } catch (SysError & e) {
+        // swallow EPERM
+        if (e.errNo == EPERM) return false;
+        throw;
+    }
+}
+
+
+Path readLink(const Path & path)
+{
+    checkInterrupt();
+    std::vector<char> buf;
+    for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) {
+        buf.resize(bufSize);
+        ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize);
+        if (rlSize == -1)
+            if (errno == EINVAL)
+                throw Error("'%1%' is not a symlink", path);
+            else
+                throw SysError("reading symbolic link '%1%'", path);
+        else if (rlSize < bufSize)
+            return std::string(buf.data(), rlSize);
+    }
+}
+
+
+bool isLink(const Path & path)
+{
+    struct stat st = lstat(path);
+    return S_ISLNK(st.st_mode);
+}
+
+
+DirEntries readDirectory(DIR *dir, const Path & path)
+{
+    DirEntries entries;
+    entries.reserve(64);
+
+    struct dirent * dirent;
+    while (errno = 0, dirent = readdir(dir)) { /* sic */
+        checkInterrupt();
+        std::string name = dirent->d_name;
+        if (name == "." || name == "..") continue;
+        entries.emplace_back(name, dirent->d_ino,
+#ifdef HAVE_STRUCT_DIRENT_D_TYPE
+            dirent->d_type
+#else
+            DT_UNKNOWN
+#endif
+        );
+    }
+    if (errno) throw SysError("reading directory '%1%'", path);
+
+    return entries;
+}
+
+DirEntries readDirectory(const Path & path)
+{
+    AutoCloseDir dir(opendir(path.c_str()));
+    if (!dir) throw SysError("opening directory '%1%'", path);
+
+    return readDirectory(dir.get(), path);
+}
+
+
+unsigned char getFileType(const Path & path)
+{
+    struct stat st = lstat(path);
+    if (S_ISDIR(st.st_mode)) return DT_DIR;
+    if (S_ISLNK(st.st_mode)) return DT_LNK;
+    if (S_ISREG(st.st_mode)) return DT_REG;
+    return DT_UNKNOWN;
+}
+
+
+std::string readFile(const Path & path)
+{
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    return readFile(fd.get());
+}
+
+
+void readFile(const Path & path, Sink & sink)
+{
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%s'", path);
+    drainFD(fd.get(), sink);
+}
+
+
+void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync)
+{
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    try {
+        writeFull(fd.get(), s);
+    } catch (Error & e) {
+        e.addTrace({}, "writing file '%1%'", path);
+        throw;
+    }
+    if (sync)
+        fd.fsync();
+    // Explicitly close to make sure exceptions are propagated.
+    fd.close();
+    if (sync)
+        syncParent(path);
+}
+
+
+void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
+{
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+
+    std::vector<char> buf(64 * 1024);
+
+    try {
+        while (true) {
+            try {
+                auto n = source.read(buf.data(), buf.size());
+                writeFull(fd.get(), {buf.data(), n});
+            } catch (EndOfFile &) { break; }
+        }
+    } catch (Error & e) {
+        e.addTrace({}, "writing file '%1%'", path);
+        throw;
+    }
+    if (sync)
+        fd.fsync();
+    // Explicitly close to make sure exceptions are propagated.
+    fd.close();
+    if (sync)
+        syncParent(path);
+}
+
+void syncParent(const Path & path)
+{
+    AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    fd.fsync();
+}
+
+
+static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
+{
+    checkInterrupt();
+
+    std::string name(baseNameOf(path));
+
+    struct stat st;
+    if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
+        if (errno == ENOENT) return;
+        throw SysError("getting status of '%1%'", path);
+    }
+
+    if (!S_ISDIR(st.st_mode)) {
+        /* We are about to delete a file. Will it likely free space? */
+
+        switch (st.st_nlink) {
+            /* Yes: last link. */
+            case 1:
+                bytesFreed += st.st_size;
+                break;
+            /* Maybe: yes, if 'auto-optimise-store' or manual optimisation
+               was performed. Instead of checking for real let's assume
+               it's an optimised file and space will be freed.
+
+               In worst case we will double count on freed space for files
+               with exactly two hardlinks for unoptimised packages.
+             */
+            case 2:
+                bytesFreed += st.st_size;
+                break;
+            /* No: 3+ links. */
+            default:
+                break;
+        }
+    }
+
+    if (S_ISDIR(st.st_mode)) {
+        /* Make the directory accessible. */
+        const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
+        if ((st.st_mode & PERM_MASK) != PERM_MASK) {
+            if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
+                throw SysError("chmod '%1%'", path);
+        }
+
+        int fd = openat(parentfd, path.c_str(), O_RDONLY);
+        if (fd == -1)
+            throw SysError("opening directory '%1%'", path);
+        AutoCloseDir dir(fdopendir(fd));
+        if (!dir)
+            throw SysError("opening directory '%1%'", path);
+        for (auto & i : readDirectory(dir.get(), path))
+            _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed);
+    }
+
+    int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
+    if (unlinkat(parentfd, name.c_str(), flags) == -1) {
+        if (errno == ENOENT) return;
+        throw SysError("cannot unlink '%1%'", path);
+    }
+}
+
+static void _deletePath(const Path & path, uint64_t & bytesFreed)
+{
+    Path dir = dirOf(path);
+    if (dir == "")
+        dir = "/";
+
+    AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
+    if (!dirfd) {
+        if (errno == ENOENT) return;
+        throw SysError("opening directory '%1%'", path);
+    }
+
+    _deletePath(dirfd.get(), path, bytesFreed);
+}
+
+
+void deletePath(const Path & path)
+{
+    uint64_t dummy;
+    deletePath(path, dummy);
+}
+
+
+Paths createDirs(const Path & path)
+{
+    Paths created;
+    if (path == "/") return created;
+
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1) {
+        created = createDirs(dirOf(path));
+        if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
+            throw SysError("creating directory '%1%'", path);
+        st = lstat(path);
+        created.push_back(path);
+    }
+
+    if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1)
+        throw SysError("statting symlink '%1%'", path);
+
+    if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path);
+
+    return created;
+}
+
+
+void deletePath(const Path & path, uint64_t & bytesFreed)
+{
+    //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
+    bytesFreed = 0;
+    _deletePath(path, bytesFreed);
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+AutoDelete::AutoDelete() : del{false} {}
+
+AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p)
+{
+    del = true;
+    this->recursive = recursive;
+}
+
+AutoDelete::~AutoDelete()
+{
+    try {
+        if (del) {
+            if (recursive)
+                deletePath(path);
+            else {
+                if (remove(path.c_str()) == -1)
+                    throw SysError("cannot unlink '%1%'", path);
+            }
+        }
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+void AutoDelete::cancel()
+{
+    del = false;
+}
+
+void AutoDelete::reset(const Path & p, bool recursive) {
+    path = p;
+    this->recursive = recursive;
+    del = true;
+}
+
+//////////////////////////////////////////////////////////////////////
+
+//////////////////////////////////////////////////////////////////////
+
+static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
+    std::atomic<unsigned int> & counter)
+{
+    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    if (includePid)
+        return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
+    else
+        return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++);
+}
+
+Path createTempDir(const Path & tmpRoot, const Path & prefix,
+    bool includePid, bool useGlobalCounter, mode_t mode)
+{
+    static std::atomic<unsigned int> globalCounter = 0;
+    std::atomic<unsigned int> localCounter = 0;
+    auto & counter(useGlobalCounter ? globalCounter : localCounter);
+
+    while (1) {
+        checkInterrupt();
+        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
+        if (mkdir(tmpDir.c_str(), mode) == 0) {
+#if __FreeBSD__
+            /* Explicitly set the group of the directory.  This is to
+               work around around problems caused by BSD's group
+               ownership semantics (directories inherit the group of
+               the parent).  For instance, the group of /tmp on
+               FreeBSD is "wheel", so all directories created in /tmp
+               will be owned by "wheel"; but if the user is not in
+               "wheel", then "tar" will fail to unpack archives that
+               have the setgid bit set on directories. */
+            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
+                throw SysError("setting group of directory '%1%'", tmpDir);
+#endif
+            return tmpDir;
+        }
+        if (errno != EEXIST)
+            throw SysError("creating directory '%1%'", tmpDir);
+    }
+}
+
+
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
+{
+    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    // Strictly speaking, this is UB, but who cares...
+    // FIXME: use O_TMPFILE.
+    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
+    if (!fd)
+        throw SysError("creating temporary file '%s'", tmpl);
+    closeOnExec(fd.get());
+    return {std::move(fd), tmpl};
+}
+
+void createSymlink(const Path & target, const Path & link)
+{
+    if (symlink(target.c_str(), link.c_str()))
+        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
+}
+
+void replaceSymlink(const Path & target, const Path & link)
+{
+    for (unsigned int n = 0; true; n++) {
+        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
+
+        try {
+            createSymlink(target, tmp);
+        } catch (SysError & e) {
+            if (e.errNo == EEXIST) continue;
+            throw;
+        }
+
+        renameFile(tmp, link);
+
+        break;
+    }
+}
+
+void setWriteTime(const fs::path & p, const struct stat & st)
+{
+    struct timeval times[2];
+    times[0] = {
+        .tv_sec = st.st_atime,
+        .tv_usec = 0,
+    };
+    times[1] = {
+        .tv_sec = st.st_mtime,
+        .tv_usec = 0,
+    };
+    if (lutimes(p.c_str(), times) != 0)
+        throw SysError("changing modification time of '%s'", p);
+}
+
+void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
+{
+    // TODO: Rewrite the `is_*` to use `symlink_status()`
+    auto statOfFrom = lstat(from.path().c_str());
+    auto fromStatus = from.symlink_status();
+
+    // Mark the directory as writable so that we can delete its children
+    if (andDelete && fs::is_directory(fromStatus)) {
+        fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+    }
+
+
+    if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
+        fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
+    } else if (fs::is_directory(fromStatus)) {
+        fs::create_directory(to);
+        for (auto & entry : fs::directory_iterator(from.path())) {
+            copy(entry, to / entry.path().filename(), andDelete);
+        }
+    } else {
+        throw Error("file '%s' has an unsupported type", from.path());
+    }
+
+    setWriteTime(to, statOfFrom);
+    if (andDelete) {
+        if (!fs::is_symlink(fromStatus))
+            fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+        fs::remove(from.path());
+    }
+}
+
+void renameFile(const Path & oldName, const Path & newName)
+{
+    fs::rename(oldName, newName);
+}
+
+void moveFile(const Path & oldName, const Path & newName)
+{
+    try {
+        renameFile(oldName, newName);
+    } catch (fs::filesystem_error & e) {
+        auto oldPath = fs::path(oldName);
+        auto newPath = fs::path(newName);
+        // For the move to be as atomic as possible, copy to a temporary
+        // directory
+        fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
+        Finally removeTemp = [&]() { fs::remove(temp); };
+        auto tempCopyTarget = temp / "copy-target";
+        if (e.code().value() == EXDEV) {
+            fs::remove(newPath);
+            warn("Can’t rename %s as %s, copying instead", oldName, newName);
+            copy(fs::directory_entry(oldPath), tempCopyTarget, true);
+            renameFile(tempCopyTarget, newPath);
+        }
+    }
+}
+
+//////////////////////////////////////////////////////////////////////
+
+}

src/libutil/file-system.hh

@@ -0,0 +1,238 @@
+#pragma once
+/**
+ * @file
+ *
+ * Utiltities for working with the file sytem and file paths.
+ */
+
+#include "types.hh"
+#include "error.hh"
+#include "logging.hh"
+#include "file-descriptor.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <sstream>
+#include <optional>
+
+#ifndef HAVE_STRUCT_DIRENT_D_TYPE
+#define DT_UNKNOWN 0
+#define DT_REG 1
+#define DT_LNK 2
+#define DT_DIR 3
+#endif
+
+namespace nix {
+
+struct Sink;
+struct Source;
+
+/**
+ * @return An absolutized path, resolving paths relative to the
+ * specified directory, or the current directory otherwise.  The path
+ * is also canonicalised.
+ */
+Path absPath(Path path,
+    std::optional<PathView> dir = {},
+    bool resolveSymlinks = false);
+
+/**
+ * Canonicalise a path by removing all `.` or `..` components and
+ * double or trailing slashes.  Optionally resolves all symlink
+ * components such that each component of the resulting path is *not*
+ * a symbolic link.
+ */
+Path canonPath(PathView path, bool resolveSymlinks = false);
+
+/**
+ * @return The directory part of the given canonical path, i.e.,
+ * everything before the final `/`.  If the path is the root or an
+ * immediate child thereof (e.g., `/foo`), this means `/`
+ * is returned.
+ */
+Path dirOf(const PathView path);
+
+/**
+ * @return the base name of the given canonical path, i.e., everything
+ * following the final `/` (trailing slashes are removed).
+ */
+std::string_view baseNameOf(std::string_view path);
+
+/**
+ * Check whether 'path' is a descendant of 'dir'. Both paths must be
+ * canonicalized.
+ */
+bool isInDir(std::string_view path, std::string_view dir);
+
+/**
+ * Check whether 'path' is equal to 'dir' or a descendant of
+ * 'dir'. Both paths must be canonicalized.
+ */
+bool isDirOrInDir(std::string_view path, std::string_view dir);
+
+/**
+ * Get status of `path`.
+ */
+struct stat stat(const Path & path);
+struct stat lstat(const Path & path);
+
+/**
+ * @return true iff the given path exists.
+ */
+bool pathExists(const Path & path);
+
+/**
+ * A version of pathExists that returns false on a permission error.
+ * Useful for inferring default paths across directories that might not
+ * be readable.
+ * @return true iff the given path can be accessed and exists
+ */
+bool pathAccessible(const Path & path);
+
+/**
+ * Read the contents (target) of a symbolic link.  The result is not
+ * in any way canonicalised.
+ */
+Path readLink(const Path & path);
+
+bool isLink(const Path & path);
+
+/**
+ * Read the contents of a directory.  The entries `.` and `..` are
+ * removed.
+ */
+struct DirEntry
+{
+    std::string name;
+    ino_t ino;
+    /**
+     * one of DT_*
+     */
+    unsigned char type;
+    DirEntry(std::string name, ino_t ino, unsigned char type)
+        : name(std::move(name)), ino(ino), type(type) { }
+};
+
+typedef std::vector<DirEntry> DirEntries;
+
+DirEntries readDirectory(const Path & path);
+
+unsigned char getFileType(const Path & path);
+
+/**
+ * Read the contents of a file into a string.
+ */
+std::string readFile(const Path & path);
+void readFile(const Path & path, Sink & sink);
+
+/**
+ * Write a string to a file.
+ */
+void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false);
+
+void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false);
+
+/**
+ * Flush a file's parent directory to disk
+ */
+void syncParent(const Path & path);
+
+/**
+ * Delete a path; i.e., in the case of a directory, it is deleted
+ * recursively. It's not an error if the path does not exist. The
+ * second variant returns the number of bytes and blocks freed.
+ */
+void deletePath(const Path & path);
+
+void deletePath(const Path & path, uint64_t & bytesFreed);
+
+/**
+ * Create a directory and all its parents, if necessary.  Returns the
+ * list of created directories, in order of creation.
+ */
+Paths createDirs(const Path & path);
+inline Paths createDirs(PathView path)
+{
+    return createDirs(Path(path));
+}
+
+/**
+ * Create a symlink.
+ */
+void createSymlink(const Path & target, const Path & link);
+
+/**
+ * Atomically create or replace a symlink.
+ */
+void replaceSymlink(const Path & target, const Path & link);
+
+void renameFile(const Path & src, const Path & dst);
+
+/**
+ * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
+ * are on a different filesystem.
+ *
+ * Beware that this might not be atomic because of the copy that happens behind
+ * the scenes
+ */
+void moveFile(const Path & src, const Path & dst);
+
+
+/**
+ * Automatic cleanup of resources.
+ */
+class AutoDelete
+{
+    Path path;
+    bool del;
+    bool recursive;
+public:
+    AutoDelete();
+    AutoDelete(const Path & p, bool recursive = true);
+    ~AutoDelete();
+    void cancel();
+    void reset(const Path & p, bool recursive = true);
+    operator Path() const { return path; }
+    operator PathView() const { return path; }
+};
+
+
+struct DIRDeleter
+{
+    void operator()(DIR * dir) const {
+        closedir(dir);
+    }
+};
+
+typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir;
+
+
+/**
+ * Create a temporary directory.
+ */
+Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
+    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
+
+/**
+ * Create a temporary file, returning a file handle and its path.
+ */
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
+
+
+/**
+ * Used in various places.
+ */
+typedef std::function<bool(const Path & path)> PathFilter;
+
+extern PathFilter defaultPathFilter;
+
+}

src/libutil/filesystem.cc

@@ -1,162 +0,0 @@
-#include <sys/time.h>
-#include <filesystem>
-#include <atomic>
-
-#include "finally.hh"
-#include "util.hh"
-#include "types.hh"
-
-namespace fs = std::filesystem;
-
-namespace nix {
-
-static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
-    std::atomic<unsigned int> & counter)
-{
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
-    if (includePid)
-        return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
-    else
-        return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++);
-}
-
-Path createTempDir(const Path & tmpRoot, const Path & prefix,
-    bool includePid, bool useGlobalCounter, mode_t mode)
-{
-    static std::atomic<unsigned int> globalCounter = 0;
-    std::atomic<unsigned int> localCounter = 0;
-    auto & counter(useGlobalCounter ? globalCounter : localCounter);
-
-    while (1) {
-        checkInterrupt();
-        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
-        if (mkdir(tmpDir.c_str(), mode) == 0) {
-#if __FreeBSD__
-            /* Explicitly set the group of the directory.  This is to
-               work around around problems caused by BSD's group
-               ownership semantics (directories inherit the group of
-               the parent).  For instance, the group of /tmp on
-               FreeBSD is "wheel", so all directories created in /tmp
-               will be owned by "wheel"; but if the user is not in
-               "wheel", then "tar" will fail to unpack archives that
-               have the setgid bit set on directories. */
-            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-                throw SysError("setting group of directory '%1%'", tmpDir);
-#endif
-            return tmpDir;
-        }
-        if (errno != EEXIST)
-            throw SysError("creating directory '%1%'", tmpDir);
-    }
-}
-
-
-std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
-{
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
-    // Strictly speaking, this is UB, but who cares...
-    // FIXME: use O_TMPFILE.
-    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
-    if (!fd)
-        throw SysError("creating temporary file '%s'", tmpl);
-    closeOnExec(fd.get());
-    return {std::move(fd), tmpl};
-}
-
-void createSymlink(const Path & target, const Path & link)
-{
-    if (symlink(target.c_str(), link.c_str()))
-        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
-}
-
-void replaceSymlink(const Path & target, const Path & link)
-{
-    for (unsigned int n = 0; true; n++) {
-        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
-
-        try {
-            createSymlink(target, tmp);
-        } catch (SysError & e) {
-            if (e.errNo == EEXIST) continue;
-            throw;
-        }
-
-        renameFile(tmp, link);
-
-        break;
-    }
-}
-
-void setWriteTime(const fs::path & p, const struct stat & st)
-{
-    struct timeval times[2];
-    times[0] = {
-        .tv_sec = st.st_atime,
-        .tv_usec = 0,
-    };
-    times[1] = {
-        .tv_sec = st.st_mtime,
-        .tv_usec = 0,
-    };
-    if (lutimes(p.c_str(), times) != 0)
-        throw SysError("changing modification time of '%s'", p);
-}
-
-void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
-{
-    // TODO: Rewrite the `is_*` to use `symlink_status()`
-    auto statOfFrom = lstat(from.path().c_str());
-    auto fromStatus = from.symlink_status();
-
-    // Mark the directory as writable so that we can delete its children
-    if (andDelete && fs::is_directory(fromStatus)) {
-        fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
-    }
-
-
-    if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
-        fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
-    } else if (fs::is_directory(fromStatus)) {
-        fs::create_directory(to);
-        for (auto & entry : fs::directory_iterator(from.path())) {
-            copy(entry, to / entry.path().filename(), andDelete);
-        }
-    } else {
-        throw Error("file '%s' has an unsupported type", from.path());
-    }
-
-    setWriteTime(to, statOfFrom);
-    if (andDelete) {
-        if (!fs::is_symlink(fromStatus))
-            fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
-        fs::remove(from.path());
-    }
-}
-
-void renameFile(const Path & oldName, const Path & newName)
-{
-    fs::rename(oldName, newName);
-}
-
-void moveFile(const Path & oldName, const Path & newName)
-{
-    try {
-        renameFile(oldName, newName);
-    } catch (fs::filesystem_error & e) {
-        auto oldPath = fs::path(oldName);
-        auto newPath = fs::path(newName);
-        // For the move to be as atomic as possible, copy to a temporary
-        // directory
-        fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
-        Finally removeTemp = [&]() { fs::remove(temp); };
-        auto tempCopyTarget = temp / "copy-target";
-        if (e.code().value() == EXDEV) {
-            fs::remove(newPath);
-            warn("Can’t rename %s as %s, copying instead", oldName, newName);
-            copy(fs::directory_entry(oldPath), tempCopyTarget, true);
-            renameFile(tempCopyTarget, newPath);
-        }
-    }
-}
-
-}

src/libutil/fmt.hh

@@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s)
     return s;
 }
 
+inline std::string fmt(std::string_view s)
+{
+    return std::string(s);
+}
+
 inline std::string fmt(const char * s)
 {
     return s;

src/libutil/fs-sink.cc

@@ -5,6 +5,54 @@
 
 namespace nix {
 
+void copyRecursive(
+    SourceAccessor & accessor, const CanonPath & from,
+    ParseSink & sink, const Path & to)
+{
+    auto stat = accessor.lstat(from);
+
+    switch (stat.type) {
+    case SourceAccessor::tSymlink:
+    {
+        sink.createSymlink(to, accessor.readLink(from));
+    }
+
+    case SourceAccessor::tRegular:
+    {
+        sink.createRegularFile(to);
+        if (stat.isExecutable)
+            sink.isExecutable();
+        LambdaSink sink2 {
+            [&](auto d) {
+                sink.receiveContents(d);
+            }
+        };
+        accessor.readFile(from, sink2, [&](uint64_t size) {
+            sink.preallocateContents(size);
+        });
+        break;
+    }
+
+    case SourceAccessor::tDirectory:
+    {
+        sink.createDirectory(to);
+        for (auto & [name, _] : accessor.readDirectory(from)) {
+            copyRecursive(
+                accessor, from + name,
+                sink, to + "/" + name);
+            break;
+        }
+    }
+
+    case SourceAccessor::tMisc:
+        throw Error("file '%1%' has an unsupported type", from);
+
+    default:
+        abort();
+    }
+}
+
+
 struct RestoreSinkSettings : Config
 {
     Setting<bool> preallocateContents{this, false, "preallocate-contents",

src/libutil/fs-sink.hh

@@ -3,6 +3,8 @@
 
 #include "types.hh"
 #include "serialise.hh"
+#include "source-accessor.hh"
+#include "file-system.hh"
 
 namespace nix {
 
@@ -11,32 +13,93 @@ namespace nix {
  */
 struct ParseSink
 {
-    virtual void createDirectory(const Path & path) { };
+    virtual void createDirectory(const Path & path) = 0;
 
-    virtual void createRegularFile(const Path & path) { };
-    virtual void closeRegularFile() { };
-    virtual void isExecutable() { };
+    virtual void createRegularFile(const Path & path) = 0;
+    virtual void receiveContents(std::string_view data) = 0;
+    virtual void isExecutable() = 0;
+    virtual void closeRegularFile() = 0;
+
+    virtual void createSymlink(const Path & path, const std::string & target) = 0;
+
+    /**
+     * An optimization. By default, do nothing.
+     */
     virtual void preallocateContents(uint64_t size) { };
-    virtual void receiveContents(std::string_view data) { };
-
-    virtual void createSymlink(const Path & path, const std::string & target) { };
 };
 
+/**
+ * Recusively copy file system objects from the source into the sink.
+ */
+void copyRecursive(
+    SourceAccessor & accessor, const CanonPath & sourcePath,
+    ParseSink & sink, const Path & destPath);
+
+/**
+ * Ignore everything and do nothing
+ */
+struct NullParseSink : ParseSink
+{
+    void createDirectory(const Path & path) override { }
+    void receiveContents(std::string_view data) override { }
+    void createSymlink(const Path & path, const std::string & target) override { }
+    void createRegularFile(const Path & path) override { }
+    void closeRegularFile() override { }
+    void isExecutable() override { }
+};
+
+/**
+ * Write files at the given path
+ */
 struct RestoreSink : ParseSink
 {
     Path dstPath;
-    AutoCloseFD fd;
-
 
     void createDirectory(const Path & path) override;
 
     void createRegularFile(const Path & path) override;
-    void closeRegularFile() override;
-    void isExecutable() override;
-    void preallocateContents(uint64_t size) override;
     void receiveContents(std::string_view data) override;
+    void isExecutable() override;
+    void closeRegularFile() override;
 
     void createSymlink(const Path & path, const std::string & target) override;
+
+    void preallocateContents(uint64_t size) override;
+
+private:
+    AutoCloseFD fd;
+};
+
+/**
+ * Restore a single file at the top level, passing along
+ * `receiveContents` to the underlying `Sink`. For anything but a single
+ * file, set `regular = true` so the caller can fail accordingly.
+ */
+struct RegularFileSink : ParseSink
+{
+    bool regular = true;
+    Sink & sink;
+
+    RegularFileSink(Sink & sink) : sink(sink) { }
+
+    void createDirectory(const Path & path) override
+    {
+        regular = false;
+    }
+
+    void receiveContents(std::string_view data) override
+    {
+        sink(data);
+    }
+
+    void createSymlink(const Path & path, const std::string & target) override
+    {
+        regular = false;
+    }
+
+    void createRegularFile(const Path & path) override { }
+    void closeRegularFile() override { }
+    void isExecutable() override { }
 };
 
 }

src/libutil/git.cc

@@ -1,9 +1,263 @@
-#include "git.hh"
-
+#include <cerrno>
+#include <algorithm>
+#include <vector>
+#include <map>
 #include <regex>
+#include <strings.h> // for strcasecmp
+
+#include "signals.hh"
+#include "config.hh"
+#include "hash.hh"
+#include "posix-source-accessor.hh"
+
+#include "git.hh"
+#include "serialise.hh"
+
+namespace nix::git {
+
+using namespace nix;
+using namespace std::string_literals;
+
+std::optional<Mode> decodeMode(RawMode m) {
+    switch (m) {
+        case (RawMode) Mode::Directory:
+        case (RawMode) Mode::Executable:
+        case (RawMode) Mode::Regular:
+        case (RawMode) Mode::Symlink:
+            return (Mode) m;
+        default:
+            return std::nullopt;
+    }
+}
+
+
+static std::string getStringUntil(Source & source, char byte)
+{
+    std::string s;
+    char n[1];
+    source(std::string_view { n, 1 });
+    while (*n != byte) {
+        s += *n;
+        source(std::string_view { n, 1 });
+    }
+    return s;
+}
+
+
+static std::string getString(Source & source, int n)
+{
+    std::string v;
+    v.resize(n);
+    source(v);
+    return v;
+}
+
+
+void parse(
+    ParseSink & sink,
+    const Path & sinkPath,
+    Source & source,
+    std::function<SinkHook> hook,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    xpSettings.require(Xp::GitHashing);
+
+    auto type = getString(source, 5);
+
+    if (type == "blob ") {
+        sink.createRegularFile(sinkPath);
+
+        unsigned long long size = std::stoi(getStringUntil(source, 0));
+
+        sink.preallocateContents(size);
+
+        unsigned long long left = size;
+        std::string buf;
+        buf.reserve(65536);
+
+        while (left) {
+            checkInterrupt();
+            buf.resize(std::min((unsigned long long)buf.capacity(), left));
+            source(buf);
+            sink.receiveContents(buf);
+            left -= buf.size();
+        }
+    } else if (type == "tree ") {
+        unsigned long long size = std::stoi(getStringUntil(source, 0));
+        unsigned long long left = size;
+
+        sink.createDirectory(sinkPath);
+
+        while (left) {
+            std::string perms = getStringUntil(source, ' ');
+            left -= perms.size();
+            left -= 1;
+
+            RawMode rawMode = std::stoi(perms, 0, 8);
+            auto modeOpt = decodeMode(rawMode);
+            if (!modeOpt)
+                throw Error("Unknown Git permission: %o", perms);
+            auto mode = std::move(*modeOpt);
+
+            std::string name = getStringUntil(source, '\0');
+            left -= name.size();
+            left -= 1;
+
+            std::string hashs = getString(source, 20);
+            left -= 20;
+
+            Hash hash(HashAlgorithm::SHA1);
+            std::copy(hashs.begin(), hashs.end(), hash.hash);
+
+            hook(name, TreeEntry {
+                .mode = mode,
+                .hash = hash,
+            });
+
+            if (mode == Mode::Executable)
+                sink.isExecutable();
+        }
+    } else throw Error("input doesn't look like a Git object");
+}
+
+
+std::optional<Mode> convertMode(SourceAccessor::Type type)
+{
+    switch (type) {
+    case SourceAccessor::tSymlink:   return Mode::Symlink;
+    case SourceAccessor::tRegular:   return Mode::Regular;
+    case SourceAccessor::tDirectory: return Mode::Directory;
+    case SourceAccessor::tMisc:      return std::nullopt;
+    default: abort();
+    }
+}
+
+
+void restore(ParseSink & sink, Source & source, std::function<RestoreHook> hook)
+{
+    parse(sink, "", source, [&](Path name, TreeEntry entry) {
+        auto [accessor, from] = hook(entry.hash);
+        auto stat = accessor->lstat(from);
+        auto gotOpt = convertMode(stat.type);
+        if (!gotOpt)
+            throw Error("file '%s' (git hash %s) has an unsupported type",
+                from,
+                entry.hash.to_string(HashFormat::Base16, false));
+        auto & got = *gotOpt;
+        if (got != entry.mode)
+            throw Error("git mode of file '%s' (git hash %s) is %o but expected %o",
+                from,
+                entry.hash.to_string(HashFormat::Base16, false),
+                (RawMode) got,
+                (RawMode) entry.mode);
+        copyRecursive(
+            *accessor, from,
+            sink, name);
+    });
+}
+
+
+void dumpBlobPrefix(
+    uint64_t size, Sink & sink,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    xpSettings.require(Xp::GitHashing);
+    auto s = fmt("blob %d\0"s, std::to_string(size));
+    sink(s);
+}
+
+
+void dumpTree(const Tree & entries, Sink & sink,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    xpSettings.require(Xp::GitHashing);
+
+    std::string v1;
+
+    for (auto & [name, entry] : entries) {
+        auto name2 = name;
+        if (entry.mode == Mode::Directory) {
+            assert(name2.back() == '/');
+            name2.pop_back();
+        }
+        v1 += fmt("%o %s\0"s, static_cast<RawMode>(entry.mode), name2);
+        std::copy(entry.hash.hash, entry.hash.hash + entry.hash.hashSize, std::back_inserter(v1));
+    }
+
+    {
+        auto s = fmt("tree %d\0"s, v1.size());
+        sink(s);
+    }
+
+    sink(v1);
+}
+
+
+Mode dump(
+    SourceAccessor & accessor, const CanonPath & path,
+    Sink & sink,
+    std::function<DumpHook> hook,
+    PathFilter & filter,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    auto st = accessor.lstat(path);
+
+    switch (st.type) {
+    case SourceAccessor::tRegular:
+    {
+        accessor.readFile(path, sink, [&](uint64_t size) {
+            dumpBlobPrefix(size, sink, xpSettings);
+        });
+        return st.isExecutable
+            ? Mode::Executable
+            : Mode::Regular;
+    }
+
+    case SourceAccessor::tDirectory:
+    {
+        Tree entries;
+        for (auto & [name, _] : accessor.readDirectory(path)) {
+            auto child = path + name;
+            if (!filter(child.abs())) continue;
+
+            auto entry = hook(child);
+
+            auto name2 = name;
+            if (entry.mode == Mode::Directory)
+                name2 += "/";
+
+            entries.insert_or_assign(std::move(name2), std::move(entry));
+        }
+        dumpTree(entries, sink, xpSettings);
+        return Mode::Directory;
+    }
+
+    case SourceAccessor::tSymlink:
+    case SourceAccessor::tMisc:
+    default:
+        throw Error("file '%1%' has an unsupported type", path);
+    }
+}
+
+
+TreeEntry dumpHash(
+        HashAlgorithm ha,
+        SourceAccessor & accessor, const CanonPath & path, PathFilter & filter)
+{
+    std::function<DumpHook> hook;
+    hook = [&](const CanonPath & path) -> TreeEntry {
+        auto hashSink = HashSink(ha);
+        auto mode = dump(accessor, path, hashSink, hook, filter);
+        auto hash = hashSink.finish().first;
+        return {
+            .mode = mode,
+            .hash = hash,
+        };
+    };
+
+    return hook(path);
+}
 
-namespace nix {
-namespace git {
 
 std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line)
 {
@@ -22,4 +276,3 @@ std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line)
 }
 
 }
-}

src/libutil/git.hh

@@ -5,9 +5,127 @@
 #include <string_view>
 #include <optional>
 
-namespace nix {
+#include "types.hh"
+#include "serialise.hh"
+#include "hash.hh"
+#include "source-accessor.hh"
+#include "fs-sink.hh"
 
-namespace git {
+namespace nix::git {
+
+using RawMode = uint32_t;
+
+enum struct Mode : RawMode {
+    Directory = 0040000,
+    Executable = 0100755,
+    Regular = 0100644,
+    Symlink = 0120000,
+};
+
+std::optional<Mode> decodeMode(RawMode m);
+
+/**
+ * An anonymous Git tree object entry (no name part).
+ */
+struct TreeEntry
+{
+    Mode mode;
+    Hash hash;
+
+    GENERATE_CMP(TreeEntry, me->mode, me->hash);
+};
+
+/**
+ * A Git tree object, fully decoded and stored in memory.
+ *
+ * Directory names must end in a `/` for sake of sorting. See
+ * https://github.com/mirage/irmin/issues/352
+ */
+using Tree = std::map<std::string, TreeEntry>;
+
+/**
+ * Callback for processing a child hash with `parse`
+ *
+ * The function should
+ *
+ * 1. Obtain the file system objects denoted by `gitHash`
+ *
+ * 2. Ensure they match `mode`
+ *
+ * 3. Feed them into the same sink `parse` was called with
+ *
+ * Implementations may seek to memoize resources (bandwidth, storage,
+ * etc.) for the same Git hash.
+ */
+using SinkHook = void(const Path & name, TreeEntry entry);
+
+void parse(
+    ParseSink & sink, const Path & sinkPath,
+    Source & source,
+    std::function<SinkHook> hook,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+/**
+ * Assists with writing a `SinkHook` step (2).
+ */
+std::optional<Mode> convertMode(SourceAccessor::Type type);
+
+/**
+ * Simplified version of `SinkHook` for `restore`.
+ *
+ * Given a `Hash`, return a `SourceAccessor` and `CanonPath` pointing to
+ * the file system object with that path.
+ */
+using RestoreHook = std::pair<SourceAccessor *, CanonPath>(Hash);
+
+/**
+ * Wrapper around `parse` and `RestoreSink`
+ */
+void restore(ParseSink & sink, Source & source, std::function<RestoreHook> hook);
+
+/**
+ * Dumps a single file to a sink
+ *
+ * @param xpSettings for testing purposes
+ */
+void dumpBlobPrefix(
+    uint64_t size, Sink & sink,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+/**
+ * Dumps a representation of a git tree to a sink
+ */
+void dumpTree(
+    const Tree & entries, Sink & sink,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+/**
+ * Callback for processing a child with `dump`
+ *
+ * The function should return the Git hash and mode of the file at the
+ * given path in the accessor passed to `dump`.
+ *
+ * Note that if the child is a directory, its child in must also be so
+ * processed in order to compute this information.
+ */
+using DumpHook = TreeEntry(const CanonPath & path);
+
+Mode dump(
+    SourceAccessor & accessor, const CanonPath & path,
+    Sink & sink,
+    std::function<DumpHook> hook,
+    PathFilter & filter = defaultPathFilter,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+/**
+ * Recursively dumps path, hashing as we go.
+ *
+ * A smaller wrapper around `dump`.
+ */
+TreeEntry dumpHash(
+            HashAlgorithm ha,
+            SourceAccessor & accessor, const CanonPath & path,
+            PathFilter & filter = defaultPathFilter);
 
 /**
  * A line from the output of `git ls-remote --symref`.
@@ -16,15 +134,17 @@ namespace git {
  *
  * - Symbolic references of the form
  *
- *      ref: {target}	{reference}
- *
- *    where {target} is itself a reference and {reference} is optional
+ *   ```
+ *   ref: {target} {reference}
+ *   ```
+ *   where {target} is itself a reference and {reference} is optional
  *
  * - Object references of the form
  *
- *      {target}	{reference}
- *
- *    where {target} is a commit id and {reference} is mandatory
+ *   ```
+ *   {target}  {reference}
+ *   ```
+ *   where {target} is a commit id and {reference} is mandatory
  */
 struct LsRemoteRefLine {
     enum struct Kind {
@@ -36,8 +156,9 @@ struct LsRemoteRefLine {
     std::optional<std::string> reference;
 };
 
+/**
+ * Parse an `LsRemoteRefLine`
+ */
 std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line);
 
 }
-
-}

src/libutil/hash.cc

@@ -9,7 +9,6 @@
 #include "hash.hh"
 #include "archive.hh"
 #include "split.hh"
-#include "util.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -17,23 +16,24 @@
 
 namespace nix {
 
-static size_t regularHashSize(HashType type) {
+static size_t regularHashSize(HashAlgorithm type) {
     switch (type) {
-    case htMD5: return md5HashSize;
-    case htSHA1: return sha1HashSize;
-    case htSHA256: return sha256HashSize;
-    case htSHA512: return sha512HashSize;
+    case HashAlgorithm::MD5: return md5HashSize;
+    case HashAlgorithm::SHA1: return sha1HashSize;
+    case HashAlgorithm::SHA256: return sha256HashSize;
+    case HashAlgorithm::SHA512: return sha512HashSize;
     }
     abort();
 }
 
 
-std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" };
+const std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
 
+const std::set<std::string> hashFormats = {"base64", "nix32", "base16", "sri" };
 
-Hash::Hash(HashType type) : type(type)
+Hash::Hash(HashAlgorithm algo) : algo(algo)
 {
-    hashSize = regularHashSize(type);
+    hashSize = regularHashSize(algo);
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
 }
@@ -82,7 +82,7 @@ static std::string printHash16(const Hash & hash)
 
 
 // omitted: E O U T
-const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
+const std::string nix32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
 
 
 static std::string printHash32(const Hash & hash)
@@ -101,7 +101,7 @@ static std::string printHash32(const Hash & hash)
         unsigned char c =
             (hash.hash[i] >> j)
             | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j));
-        s.push_back(base32Chars[c & 0x1f]);
+        s.push_back(nix32Chars[c & 0x1f]);
     }
 
     return s;
@@ -110,23 +110,23 @@ static std::string printHash32(const Hash & hash)
 
 std::string printHash16or32(const Hash & hash)
 {
-    assert(hash.type);
-    return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false);
+    assert(static_cast<char>(hash.algo));
+    return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Nix32, false);
 }
 
 
-std::string Hash::to_string(HashFormat hashFormat, bool includeType) const
+std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const
 {
     std::string s;
-    if (hashFormat == HashFormat::SRI || includeType) {
-        s += printHashType(type);
+    if (hashFormat == HashFormat::SRI || includeAlgo) {
+        s += printHashAlgo(algo);
         s += hashFormat == HashFormat::SRI ? '-' : ':';
     }
     switch (hashFormat) {
     case HashFormat::Base16:
         s += printHash16(*this);
         break;
-    case HashFormat::Base32:
+    case HashFormat::Nix32:
         s += printHash32(*this);
         break;
     case HashFormat::Base64:
@@ -137,7 +137,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeType) const
     return s;
 }
 
-Hash Hash::dummy(htSHA256);
+Hash Hash::dummy(HashAlgorithm::SHA256);
 
 Hash Hash::parseSRI(std::string_view original) {
     auto rest = original;
@@ -146,18 +146,18 @@ Hash Hash::parseSRI(std::string_view original) {
     auto hashRaw = splitPrefixTo(rest, '-');
     if (!hashRaw)
         throw BadHash("hash '%s' is not SRI", original);
-    HashType parsedType = parseHashType(*hashRaw);
+    HashAlgorithm parsedType = parseHashAlgo(*hashRaw);
 
     return Hash(rest, parsedType, true);
 }
 
 // Mutates the string to eliminate the prefixes when found
-static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest)
+static std::pair<std::optional<HashAlgorithm>, bool> getParsedTypeAndSRI(std::string_view & rest)
 {
     bool isSRI = false;
 
     // Parse the hash type before the separator, if there was one.
-    std::optional<HashType> optParsedType;
+    std::optional<HashAlgorithm> optParsedType;
     {
         auto hashRaw = splitPrefixTo(rest, ':');
 
@@ -167,7 +167,7 @@ static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_
                 isSRI = true;
         }
         if (hashRaw)
-            optParsedType = parseHashType(*hashRaw);
+            optParsedType = parseHashAlgo(*hashRaw);
     }
 
     return {optParsedType, isSRI};
@@ -186,29 +186,29 @@ Hash Hash::parseAnyPrefixed(std::string_view original)
     return Hash(rest, *optParsedType, isSRI);
 }
 
-Hash Hash::parseAny(std::string_view original, std::optional<HashType> optType)
+Hash Hash::parseAny(std::string_view original, std::optional<HashAlgorithm> optAlgo)
 {
     auto rest = original;
     auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest);
 
     // Either the string or user must provide the type, if they both do they
     // must agree.
-    if (!optParsedType && !optType)
+    if (!optParsedType && !optAlgo)
         throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest);
-    else if (optParsedType && optType && *optParsedType != *optType)
-        throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
+    else if (optParsedType && optAlgo && *optParsedType != *optAlgo)
+        throw BadHash("hash '%s' should have type '%s'", original, printHashAlgo(*optAlgo));
 
-    HashType hashType = optParsedType ? *optParsedType : *optType;
-    return Hash(rest, hashType, isSRI);
+    HashAlgorithm hashAlgo = optParsedType ? *optParsedType : *optAlgo;
+    return Hash(rest, hashAlgo, isSRI);
 }
 
-Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type)
+Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo)
 {
-    return Hash(s, type, false);
+    return Hash(s, algo, false);
 }
 
-Hash::Hash(std::string_view rest, HashType type, bool isSRI)
-    : Hash(type)
+Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
+    : Hash(algo)
 {
     if (!isSRI && rest.size() == base16Len()) {
 
@@ -231,8 +231,8 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI)
         for (unsigned int n = 0; n < rest.size(); ++n) {
             char c = rest[rest.size() - n - 1];
             unsigned char digit;
-            for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
-                if (base32Chars[digit] == c) break;
+            for (digit = 0; digit < nix32Chars.size(); ++digit) /* !!! slow */
+                if (nix32Chars[digit] == c) break;
             if (digit >= 32)
                 throw BadHash("invalid base-32 hash '%s'", rest);
             unsigned int b = n * 5;
@@ -258,19 +258,19 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI)
     }
 
     else
-        throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type));
+        throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo));
 }
 
-Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht)
+Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha)
 {
     if (hashStr.empty()) {
-        if (!ht)
+        if (!ha)
             throw BadHash("empty hash requires explicit hash type");
-        Hash h(*ht);
+        Hash h(*ha);
         warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true));
         return h;
     } else
-        return Hash::parseAny(hashStr, ht);
+        return Hash::parseAny(hashStr, ha);
 }
 
 
@@ -283,58 +283,58 @@ union Ctx
 };
 
 
-static void start(HashType ht, Ctx & ctx)
+static void start(HashAlgorithm ha, Ctx & ctx)
 {
-    if (ht == htMD5) MD5_Init(&ctx.md5);
-    else if (ht == htSHA1) SHA1_Init(&ctx.sha1);
-    else if (ht == htSHA256) SHA256_Init(&ctx.sha256);
-    else if (ht == htSHA512) SHA512_Init(&ctx.sha512);
+    if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
+    else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1);
+    else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256);
+    else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512);
 }
 
 
-static void update(HashType ht, Ctx & ctx,
-    std::string_view data)
+static void update(HashAlgorithm ha, Ctx & ctx,
+                   std::string_view data)
 {
-    if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size());
-    else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
-    else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
-    else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
+    if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
 }
 
 
-static void finish(HashType ht, Ctx & ctx, unsigned char * hash)
+static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash)
 {
-    if (ht == htMD5) MD5_Final(hash, &ctx.md5);
-    else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1);
-    else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256);
-    else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512);
+    if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
+    else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1);
+    else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256);
+    else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512);
 }
 
 
-Hash hashString(HashType ht, std::string_view s)
+Hash hashString(HashAlgorithm ha, std::string_view s)
 {
     Ctx ctx;
-    Hash hash(ht);
-    start(ht, ctx);
-    update(ht, ctx, s);
-    finish(ht, ctx, hash.hash);
+    Hash hash(ha);
+    start(ha, ctx);
+    update(ha, ctx, s);
+    finish(ha, ctx, hash.hash);
     return hash;
 }
 
 
-Hash hashFile(HashType ht, const Path & path)
+Hash hashFile(HashAlgorithm ha, const Path & path)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     readFile(path, sink);
     return sink.finish().first;
 }
 
 
-HashSink::HashSink(HashType ht) : ht(ht)
+HashSink::HashSink(HashAlgorithm ha) : ha(ha)
 {
     ctx = new Ctx;
     bytes = 0;
-    start(ht, *ctx);
+    start(ha, *ctx);
 }
 
 HashSink::~HashSink()
@@ -346,14 +346,14 @@ HashSink::~HashSink()
 void HashSink::writeUnbuffered(std::string_view data)
 {
     bytes += data.size();
-    update(ht, *ctx, data);
+    update(ha, *ctx, data);
 }
 
 HashResult HashSink::finish()
 {
     flush();
-    Hash hash(ht);
-    nix::finish(ht, *ctx, hash.hash);
+    Hash hash(ha);
+    nix::finish(ha, *ctx, hash.hash);
     return HashResult(hash, bytes);
 }
 
@@ -361,16 +361,16 @@ HashResult HashSink::currentHash()
 {
     flush();
     Ctx ctx2 = *ctx;
-    Hash hash(ht);
-    nix::finish(ht, ctx2, hash.hash);
+    Hash hash(ha);
+    nix::finish(ha, ctx2, hash.hash);
     return HashResult(hash, bytes);
 }
 
 
 HashResult hashPath(
-    HashType ht, const Path & path, PathFilter & filter)
+        HashAlgorithm ha, const Path & path, PathFilter & filter)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     dumpPath(path, sink, filter);
     return sink.finish();
 }
@@ -378,7 +378,7 @@ HashResult hashPath(
 
 Hash compressHash(const Hash & hash, unsigned int newSize)
 {
-    Hash h(hash.type);
+    Hash h(hash.algo);
     h.hashSize = newSize;
     for (unsigned int i = 0; i < hash.hashSize; ++i)
         h.hash[i % newSize] ^= hash.hash[i];
@@ -389,7 +389,11 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
 std::optional<HashFormat> parseHashFormatOpt(std::string_view hashFormatName)
 {
     if (hashFormatName == "base16") return HashFormat::Base16;
-    if (hashFormatName == "base32") return HashFormat::Base32;
+    if (hashFormatName == "nix32") return HashFormat::Nix32;
+    if (hashFormatName == "base32") {
+        warn(R"("base32" is a deprecated alias for hash format "nix32".)");
+        return HashFormat::Nix32;
+    }
     if (hashFormatName == "base64") return HashFormat::Base64;
     if (hashFormatName == "sri") return HashFormat::SRI;
     return std::nullopt;
@@ -408,8 +412,8 @@ std::string_view printHashFormat(HashFormat HashFormat)
     switch (HashFormat) {
     case HashFormat::Base64:
         return "base64";
-    case HashFormat::Base32:
-        return "base32";
+    case HashFormat::Nix32:
+        return "nix32";
     case HashFormat::Base16:
         return "base16";
     case HashFormat::SRI:
@@ -421,31 +425,31 @@ std::string_view printHashFormat(HashFormat HashFormat)
     }
 }
 
-std::optional<HashType> parseHashTypeOpt(std::string_view s)
+std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
 {
-    if (s == "md5") return htMD5;
-    if (s == "sha1") return htSHA1;
-    if (s == "sha256") return htSHA256;
-    if (s == "sha512") return htSHA512;
+    if (s == "md5") return HashAlgorithm::MD5;
+    if (s == "sha1") return HashAlgorithm::SHA1;
+    if (s == "sha256") return HashAlgorithm::SHA256;
+    if (s == "sha512") return HashAlgorithm::SHA512;
     return std::nullopt;
 }
 
-HashType parseHashType(std::string_view s)
+HashAlgorithm parseHashAlgo(std::string_view s)
 {
-    auto opt_h = parseHashTypeOpt(s);
+    auto opt_h = parseHashAlgoOpt(s);
     if (opt_h)
         return *opt_h;
     else
         throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s);
 }
 
-std::string_view printHashType(HashType ht)
+std::string_view printHashAlgo(HashAlgorithm ha)
 {
-    switch (ht) {
-    case htMD5: return "md5";
-    case htSHA1: return "sha1";
-    case htSHA256: return "sha256";
-    case htSHA512: return "sha512";
+    switch (ha) {
+    case HashAlgorithm::MD5: return "md5";
+    case HashAlgorithm::SHA1: return "sha1";
+    case HashAlgorithm::SHA256: return "sha256";
+    case HashAlgorithm::SHA512: return "sha512";
     default:
         // illegal hash type enum value internally, as opposed to external input
         // which should be validated with nice error message.

src/libutil/hash.hh

@@ -3,6 +3,7 @@
 
 #include "types.hh"
 #include "serialise.hh"
+#include "file-system.hh"
 
 
 namespace nix {
@@ -11,7 +12,7 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 };
+enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 };
 
 
 const int md5HashSize = 16;
@@ -19,9 +20,9 @@ const int sha1HashSize = 20;
 const int sha256HashSize = 32;
 const int sha512HashSize = 64;
 
-extern std::set<std::string> hashTypes;
+extern const std::set<std::string> hashAlgorithms;
 
-extern const std::string base32Chars;
+extern const std::string nix32Chars;
 
 /**
  * @brief Enumeration representing the hash formats.
@@ -30,8 +31,8 @@ enum struct HashFormat : int {
     /// @brief Base 64 encoding.
     /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4).
     Base64,
-    /// @brief Nix-specific base-32 encoding. @see base32Chars
-    Base32,
+    /// @brief Nix-specific base-32 encoding. @see nix32Chars
+    Nix32,
     /// @brief Lowercase hexadecimal encoding. @see base16Chars
     Base16,
     /// @brief "<hash algo>:<Base 64 hash>", format of the SRI integrity attribute.
@@ -39,6 +40,7 @@ enum struct HashFormat : int {
     SRI
 };
 
+extern const std::set<std::string> hashFormats;
 
 struct Hash
 {
@@ -46,12 +48,12 @@ struct Hash
     size_t hashSize = 0;
     uint8_t hash[maxHashSize] = {};
 
-    HashType type;
+    HashAlgorithm algo;
 
     /**
      * Create a zero-filled hash object.
      */
-    Hash(HashType type);
+    explicit Hash(HashAlgorithm algo);
 
     /**
      * Parse the hash from a string representation in the format
@@ -60,7 +62,7 @@ struct Hash
      * is not present, then the hash type must be specified in the
      * string.
      */
-    static Hash parseAny(std::string_view s, std::optional<HashType> type);
+    static Hash parseAny(std::string_view s, std::optional<HashAlgorithm> optAlgo);
 
     /**
      * Parse a hash from a string representation like the above, except the
@@ -72,7 +74,7 @@ struct Hash
      * Parse a plain hash that musst not have any prefix indicating the type.
      * The type is passed in to disambiguate.
      */
-    static Hash parseNonSRIUnprefixed(std::string_view s, HashType type);
+    static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo);
 
     static Hash parseSRI(std::string_view original);
 
@@ -81,7 +83,7 @@ private:
      * The type must be provided, the string view must not include <type>
      * prefix. `isSRI` helps disambigate the various base-* encodings.
      */
-    Hash(std::string_view s, HashType type, bool isSRI);
+    Hash(std::string_view s, HashAlgorithm algo, bool isSRI);
 
 public:
     /**
@@ -102,7 +104,7 @@ public:
     /**
      * Returns the length of a base-16 representation of this hash.
      */
-    size_t base16Len() const
+    [[nodiscard]] size_t base16Len() const
     {
         return hashSize * 2;
     }
@@ -110,7 +112,7 @@ public:
     /**
      * Returns the length of a base-32 representation of this hash.
      */
-    size_t base32Len() const
+    [[nodiscard]] size_t base32Len() const
     {
         return (hashSize * 8 - 1) / 5 + 1;
     }
@@ -118,24 +120,24 @@ public:
     /**
      * Returns the length of a base-64 representation of this hash.
      */
-    size_t base64Len() const
+    [[nodiscard]] size_t base64Len() const
     {
         return ((4 * hashSize / 3) + 3) & ~3;
     }
 
     /**
      * Return a string representation of the hash, in base-16, base-32
-     * or base-64. By default, this is prefixed by the hash type
+     * or base-64. By default, this is prefixed by the hash algo
      * (e.g. "sha256:").
      */
-    std::string to_string(HashFormat hashFormat, bool includeType) const;
+    [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeAlgo) const;
 
-    std::string gitRev() const
+    [[nodiscard]] std::string gitRev() const
     {
         return to_string(HashFormat::Base16, false);
     }
 
-    std::string gitShortRev() const
+    [[nodiscard]] std::string gitShortRev() const
     {
         return std::string(to_string(HashFormat::Base16, false), 0, 7);
     }
@@ -146,7 +148,7 @@ public:
 /**
  * Helper that defaults empty hashes to the 0 hash.
  */
-Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht);
+Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha);
 
 /**
  * Print a hash in base-16 if it's MD5, or base-32 otherwise.
@@ -156,14 +158,14 @@ std::string printHash16or32(const Hash & hash);
 /**
  * Compute the hash of the given string.
  */
-Hash hashString(HashType ht, std::string_view s);
+Hash hashString(HashAlgorithm ha, std::string_view s);
 
 /**
  * Compute the hash of the given file, hashing its contents directly.
  *
  * (Metadata, such as the executable permission bit, is ignored.)
  */
-Hash hashFile(HashType ht, const Path & path);
+Hash hashFile(HashAlgorithm ha, const Path & path);
 
 /**
  * Compute the hash of the given path, serializing as a Nix Archive and
@@ -172,8 +174,8 @@ Hash hashFile(HashType ht, const Path & path);
  * The hash is defined as (essentially) hashString(ht, dumpPath(path)).
  */
 typedef std::pair<Hash, uint64_t> HashResult;
-HashResult hashPath(HashType ht, const Path & path,
-    PathFilter & filter = defaultPathFilter);
+HashResult hashPath(HashAlgorithm ha, const Path & path,
+                    PathFilter & filter = defaultPathFilter);
 
 /**
  * Compress a hash to the specified number of bytes by cyclically
@@ -199,17 +201,17 @@ std::string_view printHashFormat(HashFormat hashFormat);
 /**
  * Parse a string representing a hash type.
  */
-HashType parseHashType(std::string_view s);
+HashAlgorithm parseHashAlgo(std::string_view s);
 
 /**
  * Will return nothing on parse error
  */
-std::optional<HashType> parseHashTypeOpt(std::string_view s);
+std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s);
 
 /**
  * And the reverse.
  */
-std::string_view printHashType(HashType ht);
+std::string_view printHashAlgo(HashAlgorithm ha);
 
 
 union Ctx;
@@ -222,12 +224,12 @@ struct AbstractHashSink : virtual Sink
 class HashSink : public BufferedSink, public AbstractHashSink
 {
 private:
-    HashType ht;
+    HashAlgorithm ha;
     Ctx * ctx;
     uint64_t bytes;
 
 public:
-    HashSink(HashType ht);
+    HashSink(HashAlgorithm ha);
     HashSink(const HashSink & h);
     ~HashSink();
     void writeUnbuffered(std::string_view data) override;

src/libutil/json-utils.hh

@@ -78,20 +78,29 @@ namespace nlohmann {
  */
 template<typename T>
 struct adl_serializer<std::optional<T>> {
-    static std::optional<T> from_json(const json & json) {
+    /**
+     * @brief Convert a JSON type to an `optional<T>` treating
+     *        `null` as `std::nullopt`.
+     */
+    static void from_json(const json & json, std::optional<T> & t) {
         static_assert(
             nix::json_avoids_null<T>::value,
             "null is already in use for underlying type's JSON");
-        return json.is_null()
+        t = json.is_null()
             ? std::nullopt
-            : std::optional { adl_serializer<T>::from_json(json) };
+            : std::make_optional(json.template get<T>());
     }
-    static void to_json(json & json, std::optional<T> t) {
+
+    /**
+     *  @brief Convert an optional type to a JSON type  treating `std::nullopt`
+     *         as `null`.
+     */
+    static void to_json(json & json, const std::optional<T> & t) {
         static_assert(
             nix::json_avoids_null<T>::value,
             "null is already in use for underlying type's JSON");
         if (t)
-            adl_serializer<T>::to_json(json, *t);
+            json = *t;
         else
             json = nullptr;
     }

src/libutil/logging.cc

@@ -1,4 +1,7 @@
 #include "logging.hh"
+#include "file-descriptor.hh"
+#include "environment-variables.hh"
+#include "terminal.hh"
 #include "util.hh"
 #include "config.hh"
 

src/libutil/logging.hh

@@ -23,6 +23,7 @@ typedef enum {
     actQueryPathInfo = 109,
     actPostBuildHook = 110,
     actBuildWaiting = 111,
+    actFetchTree = 112,
 } ActivityType;
 
 typedef enum {
@@ -34,6 +35,7 @@ typedef enum {
     resProgress = 105,
     resSetExpected = 106,
     resPostBuildLogLine = 107,
+    resFetchStatus = 108,
 } ResultType;
 
 typedef uint64_t ActivityId;

src/libutil/memory-source-accessor.cc

@@ -0,0 +1,180 @@
+#include "memory-source-accessor.hh"
+
+namespace nix {
+
+MemorySourceAccessor::File *
+MemorySourceAccessor::open(const CanonPath & path, std::optional<File> create)
+{
+    File * cur = &root;
+
+    bool newF = false;
+
+    for (std::string_view name : path)
+    {
+        auto * curDirP = std::get_if<File::Directory>(&cur->raw);
+        if (!curDirP)
+            return nullptr;
+        auto & curDir = *curDirP;
+
+        auto i = curDir.contents.find(name);
+        if (i == curDir.contents.end()) {
+            if (!create)
+                return nullptr;
+            else {
+                newF = true;
+                i = curDir.contents.insert(i, {
+                    std::string { name },
+                    File::Directory {},
+                });
+            }
+        }
+        cur = &i->second;
+    }
+
+    if (newF && create) *cur = std::move(*create);
+
+    return cur;
+}
+
+std::string MemorySourceAccessor::readFile(const CanonPath & path)
+{
+    auto * f = open(path, std::nullopt);
+    if (!f)
+        throw Error("file '%s' does not exist", path);
+    if (auto * r = std::get_if<File::Regular>(&f->raw))
+        return r->contents;
+    else
+        throw Error("file '%s' is not a regular file", path);
+}
+
+bool MemorySourceAccessor::pathExists(const CanonPath & path)
+{
+    return open(path, std::nullopt);
+}
+
+MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const
+{
+    return std::visit(overloaded {
+        [](const Regular & r) {
+            return Stat {
+                .type = tRegular,
+                .fileSize = r.contents.size(),
+                .isExecutable = r.executable,
+            };
+        },
+        [](const Directory &) {
+            return Stat {
+                .type = tDirectory,
+            };
+        },
+        [](const Symlink &) {
+            return Stat {
+                .type = tSymlink,
+            };
+        },
+    }, this->raw);
+}
+
+std::optional<MemorySourceAccessor::Stat>
+MemorySourceAccessor::maybeLstat(const CanonPath & path)
+{
+    const auto * f = open(path, std::nullopt);
+    return f ? std::optional { f->lstat() } : std::nullopt;
+}
+
+MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const CanonPath & path)
+{
+    auto * f = open(path, std::nullopt);
+    if (!f)
+        throw Error("file '%s' does not exist", path);
+    if (auto * d = std::get_if<File::Directory>(&f->raw)) {
+        DirEntries res;
+        for (auto & [name, file] : d->contents)
+            res.insert_or_assign(name, file.lstat().type);
+        return res;
+    } else
+        throw Error("file '%s' is not a directory", path);
+    return {};
+}
+
+std::string MemorySourceAccessor::readLink(const CanonPath & path)
+{
+    auto * f = open(path, std::nullopt);
+    if (!f)
+        throw Error("file '%s' does not exist", path);
+    if (auto * s = std::get_if<File::Symlink>(&f->raw))
+        return s->target;
+    else
+        throw Error("file '%s' is not a symbolic link", path);
+}
+
+CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents)
+{
+    auto * f = open(path, File { File::Regular {} });
+    if (!f)
+        throw Error("file '%s' cannot be made because some parent file is not a directory", path);
+    if (auto * r = std::get_if<File::Regular>(&f->raw))
+        r->contents = std::move(contents);
+    else
+        throw Error("file '%s' is not a regular file", path);
+
+    return path;
+}
+
+
+using File = MemorySourceAccessor::File;
+
+void MemorySink::createDirectory(const Path & path)
+{
+    auto * f = dst.open(CanonPath{path}, File { File::Directory { } });
+    if (!f)
+        throw Error("file '%s' cannot be made because some parent file is not a directory", path);
+
+    if (!std::holds_alternative<File::Directory>(f->raw))
+        throw Error("file '%s' is not a directory", path);
+};
+
+void MemorySink::createRegularFile(const Path & path)
+{
+    auto * f = dst.open(CanonPath{path}, File { File::Regular {} });
+    if (!f)
+        throw Error("file '%s' cannot be made because some parent file is not a directory", path);
+    if (!(r = std::get_if<File::Regular>(&f->raw)))
+        throw Error("file '%s' is not a regular file", path);
+}
+
+void MemorySink::closeRegularFile()
+{
+    r = nullptr;
+}
+
+void MemorySink::isExecutable()
+{
+    assert(r);
+    r->executable = true;
+}
+
+void MemorySink::preallocateContents(uint64_t len)
+{
+    assert(r);
+    r->contents.reserve(len);
+}
+
+void MemorySink::receiveContents(std::string_view data)
+{
+    assert(r);
+    r->contents += data;
+}
+
+void MemorySink::createSymlink(const Path & path, const std::string & target)
+{
+    auto * f = dst.open(CanonPath{path}, File { File::Symlink { } });
+    if (!f)
+        throw Error("file '%s' cannot be made because some parent file is not a directory", path);
+    if (auto * s = std::get_if<File::Symlink>(&f->raw))
+        s->target = target;
+    else
+        throw Error("file '%s' is not a symbolic link", path);
+}
+
+}

src/libutil/memory-source-accessor.hh

@@ -0,0 +1,99 @@
+#include "source-accessor.hh"
+#include "fs-sink.hh"
+#include "variant-wrapper.hh"
+
+namespace nix {
+
+/**
+ * An source accessor for an in-memory file system.
+ */
+struct MemorySourceAccessor : virtual SourceAccessor
+{
+    /**
+     * In addition to being part of the implementation of
+     * `MemorySourceAccessor`, this has a side benefit of nicely
+     * defining what a "file system object" is in Nix.
+     */
+    struct File {
+        struct Regular {
+            bool executable = false;
+            std::string contents;
+
+            GENERATE_CMP(Regular, me->executable, me->contents);
+        };
+
+        struct Directory {
+            using Name = std::string;
+
+            std::map<Name, File, std::less<>> contents;
+
+            GENERATE_CMP(Directory, me->contents);
+        };
+
+        struct Symlink {
+            std::string target;
+
+            GENERATE_CMP(Symlink, me->target);
+        };
+
+        using Raw = std::variant<Regular, Directory, Symlink>;
+        Raw raw;
+
+        MAKE_WRAPPER_CONSTRUCTOR(File);
+
+        GENERATE_CMP(File, me->raw);
+
+        Stat lstat() const;
+    };
+
+    File root { File::Directory {} };
+
+    GENERATE_CMP(MemorySourceAccessor, me->root);
+
+    std::string readFile(const CanonPath & path) override;
+    bool pathExists(const CanonPath & path) override;
+    std::optional<Stat> maybeLstat(const CanonPath & path) override;
+    DirEntries readDirectory(const CanonPath & path) override;
+    std::string readLink(const CanonPath & path) override;
+
+    /**
+     * @param create If present, create this file and any parent directories
+     * that are needed.
+     *
+     * Return null if
+     *
+     * - `create = false`: File does not exist.
+     *
+     * - `create = true`: some parent file was not a dir, so couldn't
+     *   look/create inside.
+     */
+    File * open(const CanonPath & path, std::optional<File> create);
+
+    CanonPath addFile(CanonPath path, std::string && contents);
+};
+
+/**
+ * Write to a `MemorySourceAccessor` at the given path
+ */
+struct MemorySink : ParseSink
+{
+    MemorySourceAccessor & dst;
+
+    MemorySink(MemorySourceAccessor & dst) : dst(dst) { }
+
+    void createDirectory(const Path & path) override;
+
+    void createRegularFile(const Path & path) override;
+    void receiveContents(std::string_view data) override;
+    void isExecutable() override;
+    void closeRegularFile() override;
+
+    void createSymlink(const Path & path, const std::string & target) override;
+
+    void preallocateContents(uint64_t size) override;
+
+private:
+    MemorySourceAccessor::File::Regular * r;
+};
+
+}

src/libutil/monitor-fd.hh

@@ -10,6 +10,8 @@
 #include <unistd.h>
 #include <signal.h>
 
+#include "signals.hh"
+
 namespace nix {
 
 

src/libutil/namespaces.cc

@@ -1,13 +1,22 @@
-#if __linux__
-
-#include "namespaces.hh"
+#include "current-process.hh"
 #include "util.hh"
 #include "finally.hh"
+#include "file-system.hh"
+#include "processes.hh"
+#include "signals.hh"
+
+#if __linux__
+# include <mutex>
+# include <sys/resource.h>
+# include "cgroup.hh"
+#endif
 
 #include <sys/mount.h>
 
 namespace nix {
 
+#if __linux__
+
 bool userNamespacesSupported()
 {
     static auto res = [&]() -> bool
@@ -92,6 +101,60 @@ bool mountAndPidNamespacesSupported()
     return res;
 }
 
+#endif
+
+
+//////////////////////////////////////////////////////////////////////
+
+#if __linux__
+static AutoCloseFD fdSavedMountNamespace;
+static AutoCloseFD fdSavedRoot;
+#endif
+
+void saveMountNamespace()
+{
+#if __linux__
+    static std::once_flag done;
+    std::call_once(done, []() {
+        fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
+        if (!fdSavedMountNamespace)
+            throw SysError("saving parent mount namespace");
+
+        fdSavedRoot = open("/proc/self/root", O_RDONLY);
+    });
+#endif
 }
 
+void restoreMountNamespace()
+{
+#if __linux__
+    try {
+        auto savedCwd = absPath(".");
+
+        if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
+            throw SysError("restoring parent mount namespace");
+
+        if (fdSavedRoot) {
+            if (fchdir(fdSavedRoot.get()))
+                throw SysError("chdir into saved root");
+            if (chroot("."))
+                throw SysError("chroot into saved root");
+        }
+
+        if (chdir(savedCwd.c_str()) == -1)
+            throw SysError("restoring cwd");
+    } catch (Error & e) {
+        debug(e.msg());
+    }
 #endif
+}
+
+void unshareFilesystem()
+{
+#ifdef __linux__
+    if (unshare(CLONE_FS) != 0 && errno != EPERM)
+        throw SysError("unsharing filesystem state in download thread");
+#endif
+}
+
+}

src/libutil/namespaces.hh

@@ -1,8 +1,31 @@
 #pragma once
 ///@file
 
+#include <optional>
+
+#include "types.hh"
+
 namespace nix {
 
+/**
+ * Save the current mount namespace. Ignored if called more than
+ * once.
+ */
+void saveMountNamespace();
+
+/**
+ * Restore the mount namespace saved by saveMountNamespace(). Ignored
+ * if saveMountNamespace() was never called.
+ */
+void restoreMountNamespace();
+
+/**
+ * Cause this thread to not share any FS attributes with the main
+ * thread, because this causes setns() in restoreMountNamespace() to
+ * fail.
+ */
+void unshareFilesystem();
+
 #if __linux__
 
 bool userNamespacesSupported();

src/libutil/posix-source-accessor.cc

@@ -1,4 +1,8 @@
 #include "posix-source-accessor.hh"
+#include "signals.hh"
+#include "sync.hh"
+
+#include <unordered_map>
 
 namespace nix {
 
@@ -7,9 +11,9 @@ void PosixSourceAccessor::readFile(
     Sink & sink,
     std::function<void(uint64_t)> sizeCallback)
 {
-    // FIXME: add O_NOFOLLOW since symlinks should be resolved by the
-    // caller?
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    assertNoSymlinks(path);
+
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW);
     if (!fd)
         throw SysError("opening file '%1%'", path);
 
@@ -41,25 +45,55 @@ void PosixSourceAccessor::readFile(
 
 bool PosixSourceAccessor::pathExists(const CanonPath & path)
 {
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
     return nix::pathExists(path.abs());
 }
 
-SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path)
+std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & path)
 {
-    auto st = nix::lstat(path.abs());
-    mtime = std::max(mtime, st.st_mtime);
+    static Sync<std::unordered_map<CanonPath, std::optional<struct stat>>> _cache;
+
+    {
+        auto cache(_cache.lock());
+        auto i = cache->find(path);
+        if (i != cache->end()) return i->second;
+    }
+
+    std::optional<struct stat> st{std::in_place};
+    if (::lstat(path.c_str(), &*st)) {
+        if (errno == ENOENT || errno == ENOTDIR)
+            st.reset();
+        else
+            throw SysError("getting status of '%s'", showPath(path));
+    }
+
+    auto cache(_cache.lock());
+    if (cache->size() >= 16384) cache->clear();
+    cache->emplace(path, st);
+
+    return st;
+}
+
+std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonPath & path)
+{
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
+    auto st = cachedLstat(path);
+    if (!st) return std::nullopt;
+    mtime = std::max(mtime, st->st_mtime);
     return Stat {
         .type =
-            S_ISREG(st.st_mode) ? tRegular :
-            S_ISDIR(st.st_mode) ? tDirectory :
-            S_ISLNK(st.st_mode) ? tSymlink :
+            S_ISREG(st->st_mode) ? tRegular :
+            S_ISDIR(st->st_mode) ? tDirectory :
+            S_ISLNK(st->st_mode) ? tSymlink :
             tMisc,
-        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
+        .fileSize = S_ISREG(st->st_mode) ? std::optional<uint64_t>(st->st_size) : std::nullopt,
+        .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR,
     };
 }
 
 SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path)
 {
+    assertNoSymlinks(path);
     DirEntries res;
     for (auto & entry : nix::readDirectory(path.abs())) {
         std::optional<Type> type;
@@ -75,6 +109,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath &
 
 std::string PosixSourceAccessor::readLink(const CanonPath & path)
 {
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
     return nix::readLink(path.abs());
 }
 
@@ -83,4 +118,14 @@ std::optional<CanonPath> PosixSourceAccessor::getPhysicalPath(const CanonPath &
     return path;
 }
 
+void PosixSourceAccessor::assertNoSymlinks(CanonPath path)
+{
+    while (!path.isRoot()) {
+        auto st = cachedLstat(path);
+        if (st && S_ISLNK(st->st_mode))
+            throw Error("path '%s' is a symlink", showPath(path));
+        path.pop();
+    }
+}
+
 }

src/libutil/posix-source-accessor.hh

@@ -7,7 +7,7 @@ namespace nix {
 /**
  * A source accessor that uses the Unix filesystem.
  */
-struct PosixSourceAccessor : SourceAccessor
+struct PosixSourceAccessor : virtual SourceAccessor
 {
     /**
      * The most recent mtime seen by lstat(). This is a hack to
@@ -22,13 +22,22 @@ struct PosixSourceAccessor : SourceAccessor
 
     bool pathExists(const CanonPath & path) override;
 
-    Stat lstat(const CanonPath & path) override;
+    std::optional<Stat> maybeLstat(const CanonPath & path) override;
 
     DirEntries readDirectory(const CanonPath & path) override;
 
     std::string readLink(const CanonPath & path) override;
 
     std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override;
+
+private:
+
+    /**
+     * Throw an error if `path` or any of its ancestors are symlinks.
+     */
+    void assertNoSymlinks(CanonPath path);
+
+    std::optional<struct stat> cachedLstat(const CanonPath & path);
 };
 
 }

src/libutil/processes.cc

@@ -0,0 +1,421 @@
+#include "current-process.hh"
+#include "environment-variables.hh"
+#include "signals.hh"
+#include "processes.hh"
+#include "finally.hh"
+#include "serialise.hh"
+
+#include <cerrno>
+#include <cstdlib>
+#include <cstring>
+#include <future>
+#include <iostream>
+#include <sstream>
+#include <thread>
+
+#include <grp.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#ifdef __APPLE__
+# include <sys/syscall.h>
+#endif
+
+#ifdef __linux__
+# include <sys/prctl.h>
+# include <sys/mman.h>
+#endif
+
+
+namespace nix {
+
+Pid::Pid()
+{
+}
+
+
+Pid::Pid(pid_t pid)
+    : pid(pid)
+{
+}
+
+
+Pid::~Pid()
+{
+    if (pid != -1) kill();
+}
+
+
+void Pid::operator =(pid_t pid)
+{
+    if (this->pid != -1 && this->pid != pid) kill();
+    this->pid = pid;
+    killSignal = SIGKILL; // reset signal to default
+}
+
+
+Pid::operator pid_t()
+{
+    return pid;
+}
+
+
+int Pid::kill()
+{
+    assert(pid != -1);
+
+    debug("killing process %1%", pid);
+
+    /* Send the requested signal to the child.  If it has its own
+       process group, send the signal to every process in the child
+       process group (which hopefully includes *all* its children). */
+    if (::kill(separatePG ? -pid : pid, killSignal) != 0) {
+        /* On BSDs, killing a process group will return EPERM if all
+           processes in the group are zombies (or something like
+           that). So try to detect and ignore that situation. */
+#if __FreeBSD__ || __APPLE__
+        if (errno != EPERM || ::kill(pid, 0) != 0)
+#endif
+            logError(SysError("killing process %d", pid).info());
+    }
+
+    return wait();
+}
+
+
+int Pid::wait()
+{
+    assert(pid != -1);
+    while (1) {
+        int status;
+        int res = waitpid(pid, &status, 0);
+        if (res == pid) {
+            pid = -1;
+            return status;
+        }
+        if (errno != EINTR)
+            throw SysError("cannot get exit status of PID %d", pid);
+        checkInterrupt();
+    }
+}
+
+
+void Pid::setSeparatePG(bool separatePG)
+{
+    this->separatePG = separatePG;
+}
+
+
+void Pid::setKillSignal(int signal)
+{
+    this->killSignal = signal;
+}
+
+
+pid_t Pid::release()
+{
+    pid_t p = pid;
+    pid = -1;
+    return p;
+}
+
+
+void killUser(uid_t uid)
+{
+    debug("killing all processes running under uid '%1%'", uid);
+
+    assert(uid != 0); /* just to be safe... */
+
+    /* The system call kill(-1, sig) sends the signal `sig' to all
+       users to which the current process can send signals.  So we
+       fork a process, switch to uid, and send a mass kill. */
+
+    Pid pid = startProcess([&]() {
+
+        if (setuid(uid) == -1)
+            throw SysError("setting uid");
+
+        while (true) {
+#ifdef __APPLE__
+            /* OSX's kill syscall takes a third parameter that, among
+               other things, determines if kill(-1, signo) affects the
+               calling process. In the OSX libc, it's set to true,
+               which means "follow POSIX", which we don't want here
+                 */
+            if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
+#else
+            if (kill(-1, SIGKILL) == 0) break;
+#endif
+            if (errno == ESRCH || errno == EPERM) break; /* no more processes */
+            if (errno != EINTR)
+                throw SysError("cannot kill processes for uid '%1%'", uid);
+        }
+
+        _exit(0);
+    });
+
+    int status = pid.wait();
+    if (status != 0)
+        throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status));
+
+    /* !!! We should really do some check to make sure that there are
+       no processes left running under `uid', but there is no portable
+       way to do so (I think).  The most reliable way may be `ps -eo
+       uid | grep -q $uid'. */
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+/* Wrapper around vfork to prevent the child process from clobbering
+   the caller's stack frame in the parent. */
+static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
+static pid_t doFork(bool allowVfork, std::function<void()> fun)
+{
+#ifdef __linux__
+    pid_t pid = allowVfork ? vfork() : fork();
+#else
+    pid_t pid = fork();
+#endif
+    if (pid != 0) return pid;
+    fun();
+    abort();
+}
+
+
+#if __linux__
+static int childEntry(void * arg)
+{
+    auto main = (std::function<void()> *) arg;
+    (*main)();
+    return 1;
+}
+#endif
+
+
+pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
+{
+    std::function<void()> wrapper = [&]() {
+        if (!options.allowVfork)
+            logger = makeSimpleLogger();
+        try {
+#if __linux__
+            if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
+                throw SysError("setting death signal");
+#endif
+            fun();
+        } catch (std::exception & e) {
+            try {
+                std::cerr << options.errorPrefix << e.what() << "\n";
+            } catch (...) { }
+        } catch (...) { }
+        if (options.runExitHandlers)
+            exit(1);
+        else
+            _exit(1);
+    };
+
+    pid_t pid = -1;
+
+    if (options.cloneFlags) {
+        #ifdef __linux__
+        // Not supported, since then we don't know when to free the stack.
+        assert(!(options.cloneFlags & CLONE_VM));
+
+        size_t stackSize = 1 * 1024 * 1024;
+        auto stack = (char *) mmap(0, stackSize,
+            PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+        if (stack == MAP_FAILED) throw SysError("allocating stack");
+
+        Finally freeStack([&]() { munmap(stack, stackSize); });
+
+        pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper);
+        #else
+        throw Error("clone flags are only supported on Linux");
+        #endif
+    } else
+        pid = doFork(options.allowVfork, wrapper);
+
+    if (pid == -1) throw SysError("unable to fork");
+
+    return pid;
+}
+
+
+std::string runProgram(Path program, bool searchPath, const Strings & args,
+    const std::optional<std::string> & input, bool isInteractive)
+{
+    auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive});
+
+    if (!statusOk(res.first))
+        throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
+
+    return res.second;
+}
+
+// Output = error code + "standard out" output stream
+std::pair<int, std::string> runProgram(RunOptions && options)
+{
+    StringSink sink;
+    options.standardOut = &sink;
+
+    int status = 0;
+
+    try {
+        runProgram2(options);
+    } catch (ExecError & e) {
+        status = e.status;
+    }
+
+    return {status, std::move(sink.s)};
+}
+
+void runProgram2(const RunOptions & options)
+{
+    checkInterrupt();
+
+    assert(!(options.standardIn && options.input));
+
+    std::unique_ptr<Source> source_;
+    Source * source = options.standardIn;
+
+    if (options.input) {
+        source_ = std::make_unique<StringSource>(*options.input);
+        source = source_.get();
+    }
+
+    /* Create a pipe. */
+    Pipe out, in;
+    if (options.standardOut) out.create();
+    if (source) in.create();
+
+    ProcessOptions processOptions;
+    // vfork implies that the environment of the main process and the fork will
+    // be shared (technically this is undefined, but in practice that's the
+    // case), so we can't use it if we alter the environment
+    processOptions.allowVfork = !options.environment;
+
+    std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
+    if (options.isInteractive) {
+        logger->pause();
+        resumeLoggerDefer.emplace(
+            []() {
+                logger->resume();
+            }
+        );
+    }
+
+    /* Fork. */
+    Pid pid = startProcess([&]() {
+        if (options.environment)
+            replaceEnv(*options.environment);
+        if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+            throw SysError("dupping stdout");
+        if (options.mergeStderrToStdout)
+            if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
+                throw SysError("cannot dup stdout into stderr");
+        if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
+            throw SysError("dupping stdin");
+
+        if (options.chdir && chdir((*options.chdir).c_str()) == -1)
+            throw SysError("chdir failed");
+        if (options.gid && setgid(*options.gid) == -1)
+            throw SysError("setgid failed");
+        /* Drop all other groups if we're setgid. */
+        if (options.gid && setgroups(0, 0) == -1)
+            throw SysError("setgroups failed");
+        if (options.uid && setuid(*options.uid) == -1)
+            throw SysError("setuid failed");
+
+        Strings args_(options.args);
+        args_.push_front(options.program);
+
+        restoreProcessContext();
+
+        if (options.searchPath)
+            execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
+            // This allows you to refer to a program with a pathname relative
+            // to the PATH variable.
+        else
+            execv(options.program.c_str(), stringsToCharPtrs(args_).data());
+
+        throw SysError("executing '%1%'", options.program);
+    }, processOptions);
+
+    out.writeSide.close();
+
+    std::thread writerThread;
+
+    std::promise<void> promise;
+
+    Finally doJoin([&]() {
+        if (writerThread.joinable())
+            writerThread.join();
+    });
+
+
+    if (source) {
+        in.readSide.close();
+        writerThread = std::thread([&]() {
+            try {
+                std::vector<char> buf(8 * 1024);
+                while (true) {
+                    size_t n;
+                    try {
+                        n = source->read(buf.data(), buf.size());
+                    } catch (EndOfFile &) {
+                        break;
+                    }
+                    writeFull(in.writeSide.get(), {buf.data(), n});
+                }
+                promise.set_value();
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+            in.writeSide.close();
+        });
+    }
+
+    if (options.standardOut)
+        drainFD(out.readSide.get(), *options.standardOut);
+
+    /* Wait for the child to finish. */
+    int status = pid.wait();
+
+    /* Wait for the writer thread to finish. */
+    if (source) promise.get_future().get();
+
+    if (status)
+        throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status));
+}
+
+//////////////////////////////////////////////////////////////////////
+
+std::string statusToString(int status)
+{
+    if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+        if (WIFEXITED(status))
+            return fmt("failed with exit code %1%", WEXITSTATUS(status));
+        else if (WIFSIGNALED(status)) {
+            int sig = WTERMSIG(status);
+#if HAVE_STRSIGNAL
+            const char * description = strsignal(sig);
+            return fmt("failed due to signal %1% (%2%)", sig, description);
+#else
+            return fmt("failed due to signal %1%", sig);
+#endif
+        }
+        else
+            return "died abnormally";
+    } else return "succeeded";
+}
+
+
+bool statusOk(int status)
+{
+    return WIFEXITED(status) && WEXITSTATUS(status) == 0;
+}
+
+}

src/libutil/processes.hh

@@ -0,0 +1,123 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "error.hh"
+#include "logging.hh"
+#include "ansicolor.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <sstream>
+#include <optional>
+
+namespace nix {
+
+struct Sink;
+struct Source;
+
+class Pid
+{
+    pid_t pid = -1;
+    bool separatePG = false;
+    int killSignal = SIGKILL;
+public:
+    Pid();
+    Pid(pid_t pid);
+    ~Pid();
+    void operator =(pid_t pid);
+    operator pid_t();
+    int kill();
+    int wait();
+
+    void setSeparatePG(bool separatePG);
+    void setKillSignal(int signal);
+    pid_t release();
+};
+
+
+/**
+ * Kill all processes running under the specified uid by sending them
+ * a SIGKILL.
+ */
+void killUser(uid_t uid);
+
+
+/**
+ * Fork a process that runs the given function, and return the child
+ * pid to the caller.
+ */
+struct ProcessOptions
+{
+    std::string errorPrefix = "";
+    bool dieWithParent = true;
+    bool runExitHandlers = false;
+    bool allowVfork = false;
+    /**
+     * use clone() with the specified flags (Linux only)
+     */
+    int cloneFlags = 0;
+};
+
+pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
+
+
+/**
+ * Run a program and return its stdout in a string (i.e., like the
+ * shell backtick operator).
+ */
+std::string runProgram(Path program, bool searchPath = false,
+    const Strings & args = Strings(),
+    const std::optional<std::string> & input = {}, bool isInteractive = false);
+
+struct RunOptions
+{
+    Path program;
+    bool searchPath = true;
+    Strings args;
+    std::optional<uid_t> uid;
+    std::optional<uid_t> gid;
+    std::optional<Path> chdir;
+    std::optional<std::map<std::string, std::string>> environment;
+    std::optional<std::string> input;
+    Source * standardIn = nullptr;
+    Sink * standardOut = nullptr;
+    bool mergeStderrToStdout = false;
+    bool isInteractive = false;
+};
+
+std::pair<int, std::string> runProgram(RunOptions && options);
+
+void runProgram2(const RunOptions & options);
+
+
+class ExecError : public Error
+{
+public:
+    int status;
+
+    template<typename... Args>
+    ExecError(int status, const Args & ... args)
+        : Error(args...), status(status)
+    { }
+};
+
+
+/**
+ * Convert the exit status of a child as returned by wait() into an
+ * error string.
+ */
+std::string statusToString(int status);
+
+bool statusOk(int status);
+
+}

src/libutil/references.cc

@@ -1,6 +1,5 @@
 #include "references.hh"
 #include "hash.hh"
-#include "util.hh"
 #include "archive.hh"
 
 #include <map>
@@ -24,8 +23,8 @@ static void search(
     static bool isBase32[256];
     std::call_once(initialised, [](){
         for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false;
-        for (unsigned int i = 0; i < base32Chars.size(); ++i)
-            isBase32[(unsigned char) base32Chars[i]] = true;
+        for (unsigned int i = 0; i < nix32Chars.size(); ++i)
+            isBase32[(unsigned char) nix32Chars[i]] = true;
     });
 
     for (size_t i = 0; i + refLength <= s.size(); ) {
@@ -111,8 +110,8 @@ void RewritingSink::flush()
     prev.clear();
 }
 
-HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
-    : hashSink(ht)
+HashModuloSink::HashModuloSink(HashAlgorithm ha, const std::string & modulus)
+    : hashSink(ha)
     , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
 {
 }

src/libutil/references.hh

@@ -46,7 +46,7 @@ struct HashModuloSink : AbstractHashSink
     HashSink hashSink;
     RewritingSink rewritingSink;
 
-    HashModuloSink(HashType ht, const std::string & modulus);
+    HashModuloSink(HashAlgorithm ha, const std::string & modulus);
 
     void operator () (std::string_view data) override;
 

src/libutil/serialise.cc

@@ -1,5 +1,5 @@
 #include "serialise.hh"
-#include "util.hh"
+#include "signals.hh"
 
 #include <cstring>
 #include <cerrno>
@@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len)
     }
 }
 
+void Source::operator () (std::string_view data)
+{
+    (*this)((char *)data.data(), data.size());
+}
 
 void Source::drainInto(Sink & sink)
 {
@@ -444,7 +448,7 @@ Error readError(Source & source)
     auto msg = readString(source);
     ErrorInfo info {
         .level = level,
-        .msg = hintformat(fmt("%s", msg)),
+        .msg = hintfmt(msg),
     };
     auto havePos = readNum<size_t>(source);
     assert(havePos == 0);
@@ -453,7 +457,7 @@ Error readError(Source & source)
         havePos = readNum<size_t>(source);
         assert(havePos == 0);
         info.traces.push_back(Trace {
-            .hint = hintformat(fmt("%s", readString(source)))
+            .hint = hintfmt(readString(source))
         });
     }
     return Error(std::move(info));

src/libutil/serialise.hh

@@ -5,6 +5,7 @@
 
 #include "types.hh"
 #include "util.hh"
+#include "file-descriptor.hh"
 
 namespace boost::context { struct stack_context; }
 
@@ -72,6 +73,7 @@ struct Source
      * an error if it is not going to be available.
      */
     void operator () (char * data, size_t len);
+    void operator () (std::string_view data);
 
     /**
      * Store up to ‘len’ in the buffer pointed to by ‘data’, and

src/libutil/signals.cc

@@ -0,0 +1,188 @@
+#include "signals.hh"
+#include "util.hh"
+#include "error.hh"
+#include "sync.hh"
+#include "terminal.hh"
+
+#include <thread>
+
+namespace nix {
+
+std::atomic<bool> _isInterrupted = false;
+
+static thread_local bool interruptThrown = false;
+thread_local std::function<bool()> interruptCheck;
+
+void setInterruptThrown()
+{
+    interruptThrown = true;
+}
+
+void _interrupted()
+{
+    /* Block user interrupts while an exception is being handled.
+       Throwing an exception while another exception is being handled
+       kills the program! */
+    if (!interruptThrown && !std::uncaught_exceptions()) {
+        interruptThrown = true;
+        throw Interrupted("interrupted by the user");
+    }
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+/* We keep track of interrupt callbacks using integer tokens, so we can iterate
+   safely without having to lock the data structure while executing arbitrary
+   functions.
+ */
+struct InterruptCallbacks {
+    typedef int64_t Token;
+
+    /* We use unique tokens so that we can't accidentally delete the wrong
+       handler because of an erroneous double delete. */
+    Token nextToken = 0;
+
+    /* Used as a list, see InterruptCallbacks comment. */
+    std::map<Token, std::function<void()>> callbacks;
+};
+
+static Sync<InterruptCallbacks> _interruptCallbacks;
+
+static void signalHandlerThread(sigset_t set)
+{
+    while (true) {
+        int signal = 0;
+        sigwait(&set, &signal);
+
+        if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP)
+            triggerInterrupt();
+
+        else if (signal == SIGWINCH) {
+            updateWindowSize();
+        }
+    }
+}
+
+void triggerInterrupt()
+{
+    _isInterrupted = true;
+
+    {
+        InterruptCallbacks::Token i = 0;
+        while (true) {
+            std::function<void()> callback;
+            {
+                auto interruptCallbacks(_interruptCallbacks.lock());
+                auto lb = interruptCallbacks->callbacks.lower_bound(i);
+                if (lb == interruptCallbacks->callbacks.end())
+                    break;
+
+                callback = lb->second;
+                i = lb->first + 1;
+            }
+
+            try {
+                callback();
+            } catch (...) {
+                ignoreException();
+            }
+        }
+    }
+}
+
+
+static sigset_t savedSignalMask;
+static bool savedSignalMaskIsSet = false;
+
+void setChildSignalMask(sigset_t * sigs)
+{
+    assert(sigs); // C style function, but think of sigs as a reference
+
+#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
+    sigemptyset(&savedSignalMask);
+    // There's no "assign" or "copy" function, so we rely on (math) idempotence
+    // of the or operator: a or a = a.
+    sigorset(&savedSignalMask, sigs, sigs);
+#else
+    // Without sigorset, our best bet is to assume that sigset_t is a type that
+    // can be assigned directly, such as is the case for a sigset_t defined as
+    // an integer type.
+    savedSignalMask = *sigs;
+#endif
+
+    savedSignalMaskIsSet = true;
+}
+
+void saveSignalMask() {
+    if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
+        throw SysError("querying signal mask");
+
+    savedSignalMaskIsSet = true;
+}
+
+void startSignalHandlerThread()
+{
+    updateWindowSize();
+
+    saveSignalMask();
+
+    sigset_t set;
+    sigemptyset(&set);
+    sigaddset(&set, SIGINT);
+    sigaddset(&set, SIGTERM);
+    sigaddset(&set, SIGHUP);
+    sigaddset(&set, SIGPIPE);
+    sigaddset(&set, SIGWINCH);
+    if (pthread_sigmask(SIG_BLOCK, &set, nullptr))
+        throw SysError("blocking signals");
+
+    std::thread(signalHandlerThread, set).detach();
+}
+
+void restoreSignals()
+{
+    // If startSignalHandlerThread wasn't called, that means we're not running
+    // in a proper libmain process, but a process that presumably manages its
+    // own signal handlers. Such a process should call either
+    //  - initNix(), to be a proper libmain process
+    //  - startSignalHandlerThread(), to resemble libmain regarding signal
+    //    handling only
+    //  - saveSignalMask(), for processes that define their own signal handling
+    //    thread
+    // TODO: Warn about this? Have a default signal mask? The latter depends on
+    //       whether we should generally inherit signal masks from the caller.
+    //       I don't know what the larger unix ecosystem expects from us here.
+    if (!savedSignalMaskIsSet)
+        return;
+
+    if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+        throw SysError("restoring signals");
+}
+
+
+/* RAII helper to automatically deregister a callback. */
+struct InterruptCallbackImpl : InterruptCallback
+{
+    InterruptCallbacks::Token token;
+    ~InterruptCallbackImpl() override
+    {
+        auto interruptCallbacks(_interruptCallbacks.lock());
+        interruptCallbacks->callbacks.erase(token);
+    }
+};
+
+std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback)
+{
+    auto interruptCallbacks(_interruptCallbacks.lock());
+    auto token = interruptCallbacks->nextToken++;
+    interruptCallbacks->callbacks.emplace(token, callback);
+
+    auto res = std::make_unique<InterruptCallbackImpl>();
+    res->token = token;
+
+    return std::unique_ptr<InterruptCallback>(res.release());
+}
+
+}

src/libutil/signals.hh

@@ -0,0 +1,104 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "error.hh"
+#include "logging.hh"
+#include "ansicolor.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <sstream>
+#include <optional>
+
+namespace nix {
+
+/* User interruption. */
+
+extern std::atomic<bool> _isInterrupted;
+
+extern thread_local std::function<bool()> interruptCheck;
+
+void setInterruptThrown();
+
+void _interrupted();
+
+void inline checkInterrupt()
+{
+    if (_isInterrupted || (interruptCheck && interruptCheck()))
+        _interrupted();
+}
+
+MakeError(Interrupted, BaseError);
+
+
+/**
+ * Start a thread that handles various signals. Also block those signals
+ * on the current thread (and thus any threads created by it).
+ * Saves the signal mask before changing the mask to block those signals.
+ * See saveSignalMask().
+ */
+void startSignalHandlerThread();
+
+/**
+ * Saves the signal mask, which is the signal mask that nix will restore
+ * before creating child processes.
+ * See setChildSignalMask() to set an arbitrary signal mask instead of the
+ * current mask.
+ */
+void saveSignalMask();
+
+/**
+ * To use in a process that already called `startSignalHandlerThread()`
+ * or `saveSignalMask()` first.
+ */
+void restoreSignals();
+
+/**
+ * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
+ * necessarily match the current thread's mask.
+ * See saveSignalMask() to set the saved mask to the current mask.
+ */
+void setChildSignalMask(sigset_t *sigs);
+
+struct InterruptCallback
+{
+    virtual ~InterruptCallback() { };
+};
+
+/**
+ * Register a function that gets called on SIGINT (in a non-signal
+ * context).
+ */
+std::unique_ptr<InterruptCallback> createInterruptCallback(
+    std::function<void()> callback);
+
+void triggerInterrupt();
+
+/**
+ * A RAII class that causes the current thread to receive SIGUSR1 when
+ * the signal handler thread receives SIGINT. That is, this allows
+ * SIGINT to be multiplexed to multiple threads.
+ */
+struct ReceiveInterrupts
+{
+    pthread_t target;
+    std::unique_ptr<InterruptCallback> callback;
+
+    ReceiveInterrupts()
+        : target(pthread_self())
+        , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); }))
+    { }
+};
+
+
+}

src/libutil/source-accessor.cc

@@ -7,9 +7,15 @@ static std::atomic<size_t> nextNumber{0};
 
 SourceAccessor::SourceAccessor()
     : number(++nextNumber)
+    , displayPrefix{"«unknown»"}
 {
 }
 
+bool SourceAccessor::pathExists(const CanonPath & path)
+{
+    return maybeLstat(path).has_value();
+}
+
 std::string SourceAccessor::readFile(const CanonPath & path)
 {
     StringSink sink;
@@ -33,26 +39,32 @@ void SourceAccessor::readFile(
 }
 
 Hash SourceAccessor::hashPath(
-    const CanonPath & path,
-    PathFilter & filter,
-    HashType ht)
+        const CanonPath & path,
+        PathFilter & filter,
+        HashAlgorithm ha)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     dumpPath(path, sink, filter);
     return sink.finish().first;
 }
 
-std::optional<SourceAccessor::Stat> SourceAccessor::maybeLstat(const CanonPath & path)
+SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
 {
-    // FIXME: merge these into one operation.
-    if (!pathExists(path))
-        return {};
-    return lstat(path);
+    if (auto st = maybeLstat(path))
+        return *st;
+    else
+        throw Error("path '%s' does not exist", showPath(path));
+}
+
+void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix)
+{
+    this->displayPrefix = std::move(displayPrefix);
+    this->displaySuffix = std::move(displaySuffix);
 }
 
 std::string SourceAccessor::showPath(const CanonPath & path)
 {
-    return path.abs();
+    return displayPrefix + path.abs() + displaySuffix;
 }
 
 }

src/libutil/source-accessor.hh

@@ -17,6 +17,8 @@ struct SourceAccessor
 {
     const size_t number;
 
+    std::string displayPrefix, displaySuffix;
+
     SourceAccessor();
 
     virtual ~SourceAccessor()
@@ -24,6 +26,13 @@ struct SourceAccessor
 
     /**
      * Return the contents of a file as a string.
+     *
+     * @note Unlike Unix, this method should *not* follow symlinks. Nix
+     * by default wants to manipulate symlinks explicitly, and not
+     * implictly follow them, as they are frequently untrusted user data
+     * and thus may point to arbitrary locations. Acting on the targets
+     * targets of symlinks should only occasionally be done, and only
+     * with care.
      */
     virtual std::string readFile(const CanonPath & path);
 
@@ -32,7 +41,10 @@ struct SourceAccessor
      * called with the size of the file before any data is written to
      * the sink.
      *
-     * Note: subclasses of `SourceAccessor` need to implement at least
+     * @note Like the other `readFile`, this method should *not* follow
+     * symlinks.
+     *
+     * @note subclasses of `SourceAccessor` need to implement at least
      * one of the `readFile()` variants.
      */
     virtual void readFile(
@@ -40,7 +52,7 @@ struct SourceAccessor
         Sink & sink,
         std::function<void(uint64_t)> sizeCallback = [](uint64_t size){});
 
-    virtual bool pathExists(const CanonPath & path) = 0;
+    virtual bool pathExists(const CanonPath & path);
 
     enum Type {
       tRegular, tSymlink, tDirectory,
@@ -57,18 +69,37 @@ struct SourceAccessor
     struct Stat
     {
         Type type = tMisc;
-        //uint64_t fileSize = 0; // regular files only
-        bool isExecutable = false; // regular files only
+
+        /**
+         * For regular files only: the size of the file. Not all
+         * accessors return this since it may be too expensive to
+         * compute.
+         */
+        std::optional<uint64_t> fileSize;
+
+        /**
+         * For regular files only: whether this is an executable.
+         */
+        bool isExecutable = false;
+
+        /**
+         * For regular files only: the position of the contents of this
+         * file in the NAR. Only returned by NAR accessors.
+         */
+        std::optional<uint64_t> narOffset;
     };
 
-    virtual Stat lstat(const CanonPath & path) = 0;
+    Stat lstat(const CanonPath & path);
 
-    std::optional<Stat> maybeLstat(const CanonPath & path);
+    virtual std::optional<Stat> maybeLstat(const CanonPath & path) = 0;
 
     typedef std::optional<Type> DirEntry;
 
     typedef std::map<std::string, DirEntry> DirEntries;
 
+    /**
+     * @note Like `readFile`, this method should *not* follow symlinks.
+     */
     virtual DirEntries readDirectory(const CanonPath & path) = 0;
 
     virtual std::string readLink(const CanonPath & path) = 0;
@@ -79,9 +110,9 @@ struct SourceAccessor
         PathFilter & filter = defaultPathFilter);
 
     Hash hashPath(
-        const CanonPath & path,
-        PathFilter & filter = defaultPathFilter,
-        HashType ht = htSHA256);
+            const CanonPath & path,
+            PathFilter & filter = defaultPathFilter,
+            HashAlgorithm ha = HashAlgorithm::SHA256);
 
     /**
      * Return a corresponding path in the root filesystem, if
@@ -101,6 +132,8 @@ struct SourceAccessor
         return number < x.number;
     }
 
+    void setPathDisplay(std::string displayPrefix, std::string displaySuffix = "");
+
     virtual std::string showPath(const CanonPath & path);
 };
 

src/libutil/suggestions.cc

@@ -1,7 +1,9 @@
 #include "suggestions.hh"
 #include "ansicolor.hh"
-#include "util.hh"
+#include "terminal.hh"
+
 #include <algorithm>
+#include <sstream>
 
 namespace nix {
 

src/libutil/tarfile.cc

@@ -3,6 +3,7 @@
 
 #include "serialise.hh"
 #include "tarfile.hh"
+#include "file-system.hh"
 
 namespace nix {
 
@@ -52,6 +53,7 @@ TarArchive::TarArchive(Source & source, bool raw) : buffer(65536)
         archive_read_support_format_raw(archive);
         archive_read_support_format_empty(archive);
     }
+    archive_read_set_option(archive, NULL, "mac-ext", NULL);
     check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
 }
 
@@ -62,6 +64,7 @@ TarArchive::TarArchive(const Path & path)
 
     archive_read_support_filter_all(archive);
     archive_read_support_format_all(archive);
+    archive_read_set_option(archive, NULL, "mac-ext", NULL);
     check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
 }
 

src/libutil/terminal.cc

@@ -0,0 +1,108 @@
+#include "terminal.hh"
+#include "environment-variables.hh"
+#include "sync.hh"
+
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+namespace nix {
+
+bool shouldANSI()
+{
+    return isatty(STDERR_FILENO)
+        && getEnv("TERM").value_or("dumb") != "dumb"
+        && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value());
+}
+
+std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width)
+{
+    std::string t, e;
+    size_t w = 0;
+    auto i = s.begin();
+
+    while (w < (size_t) width && i != s.end()) {
+
+        if (*i == '\e') {
+            std::string e;
+            e += *i++;
+            char last = 0;
+
+            if (i != s.end() && *i == '[') {
+                e += *i++;
+                // eat parameter bytes
+                while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
+                // eat intermediate bytes
+                while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
+                // eat final byte
+                if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
+            } else {
+                if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
+            }
+
+            if (!filterAll && last == 'm')
+                t += e;
+        }
+
+        else if (*i == '\t') {
+            i++; t += ' '; w++;
+            while (w < (size_t) width && w % 8) {
+                t += ' '; w++;
+            }
+        }
+
+        else if (*i == '\r' || *i == '\a')
+            // do nothing for now
+            i++;
+
+        else {
+            w++;
+            // Copy one UTF-8 character.
+            if ((*i & 0xe0) == 0xc0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+            } else if ((*i & 0xf0) == 0xe0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                    t += *i++;
+                    if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+                }
+            } else if ((*i & 0xf8) == 0xf0) {
+                t += *i++;
+                if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                    t += *i++;
+                    if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+                        t += *i++;
+                        if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+                    }
+                }
+            } else
+                t += *i++;
+        }
+    }
+
+    return t;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
+
+
+void updateWindowSize()
+{
+    struct winsize ws;
+    if (ioctl(2, TIOCGWINSZ, &ws) == 0) {
+        auto windowSize_(windowSize.lock());
+        windowSize_->first = ws.ws_row;
+        windowSize_->second = ws.ws_col;
+    }
+}
+
+
+std::pair<unsigned short, unsigned short> getWindowSize()
+{
+    return *windowSize.lock();
+}
+
+}

src/libutil/terminal.hh

@@ -0,0 +1,38 @@
+#pragma once
+///@file
+
+#include "types.hh"
+
+namespace nix {
+/**
+ * Determine whether ANSI escape sequences are appropriate for the
+ * present output.
+ */
+bool shouldANSI();
+
+/**
+ * Truncate a string to 'width' printable characters. If 'filterAll'
+ * is true, all ANSI escape sequences are filtered out. Otherwise,
+ * some escape sequences (such as colour setting) are copied but not
+ * included in the character count. Also, tabs are expanded to
+ * spaces.
+ */
+std::string filterANSIEscapes(std::string_view s,
+    bool filterAll = false,
+    unsigned int width = std::numeric_limits<unsigned int>::max());
+
+/**
+ * Recalculate the window size, updating a global variable. Used in the
+ * `SIGWINCH` signal handler.
+ */
+void updateWindowSize();
+
+/**
+ * @return the number of rows and columns of the terminal.
+ *
+ * The value is cached so this is quick. The cached result is computed
+ * by `updateWindowSize()`.
+ */
+std::pair<unsigned short, unsigned short> getWindowSize();
+
+}

src/libutil/tests/canon-path.cc

@@ -1,162 +0,0 @@
-#include "canon-path.hh"
-
-#include <gtest/gtest.h>
-
-namespace nix {
-
-    TEST(CanonPath, basic) {
-        {
-            CanonPath p("/");
-            ASSERT_EQ(p.abs(), "/");
-            ASSERT_EQ(p.rel(), "");
-            ASSERT_EQ(p.baseName(), std::nullopt);
-            ASSERT_EQ(p.dirOf(), std::nullopt);
-            ASSERT_FALSE(p.parent());
-        }
-
-        {
-            CanonPath p("/foo//");
-            ASSERT_EQ(p.abs(), "/foo");
-            ASSERT_EQ(p.rel(), "foo");
-            ASSERT_EQ(*p.baseName(), "foo");
-            ASSERT_EQ(*p.dirOf(), ""); // FIXME: do we want this?
-            ASSERT_EQ(p.parent()->abs(), "/");
-        }
-
-        {
-            CanonPath p("foo/bar");
-            ASSERT_EQ(p.abs(), "/foo/bar");
-            ASSERT_EQ(p.rel(), "foo/bar");
-            ASSERT_EQ(*p.baseName(), "bar");
-            ASSERT_EQ(*p.dirOf(), "/foo");
-            ASSERT_EQ(p.parent()->abs(), "/foo");
-        }
-
-        {
-            CanonPath p("foo//bar/");
-            ASSERT_EQ(p.abs(), "/foo/bar");
-            ASSERT_EQ(p.rel(), "foo/bar");
-            ASSERT_EQ(*p.baseName(), "bar");
-            ASSERT_EQ(*p.dirOf(), "/foo");
-        }
-    }
-
-    TEST(CanonPath, pop) {
-        CanonPath p("foo/bar/x");
-        ASSERT_EQ(p.abs(), "/foo/bar/x");
-        p.pop();
-        ASSERT_EQ(p.abs(), "/foo/bar");
-        p.pop();
-        ASSERT_EQ(p.abs(), "/foo");
-        p.pop();
-        ASSERT_EQ(p.abs(), "/");
-    }
-
-    TEST(CanonPath, removePrefix) {
-        CanonPath p1("foo/bar");
-        CanonPath p2("foo/bar/a/b/c");
-        ASSERT_EQ(p2.removePrefix(p1).abs(), "/a/b/c");
-        ASSERT_EQ(p1.removePrefix(p1).abs(), "/");
-        ASSERT_EQ(p1.removePrefix(CanonPath("/")).abs(), "/foo/bar");
-    }
-
-    TEST(CanonPath, iter) {
-        {
-            CanonPath p("a//foo/bar//");
-            std::vector<std::string_view> ss;
-            for (auto & c : p) ss.push_back(c);
-            ASSERT_EQ(ss, std::vector<std::string_view>({"a", "foo", "bar"}));
-        }
-
-        {
-            CanonPath p("/");
-            std::vector<std::string_view> ss;
-            for (auto & c : p) ss.push_back(c);
-            ASSERT_EQ(ss, std::vector<std::string_view>());
-        }
-    }
-
-    TEST(CanonPath, concat) {
-        {
-            CanonPath p1("a//foo/bar//");
-            CanonPath p2("xyzzy/bla");
-            ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla");
-        }
-
-        {
-            CanonPath p1("/");
-            CanonPath p2("/a/b");
-            ASSERT_EQ((p1 + p2).abs(), "/a/b");
-        }
-
-        {
-            CanonPath p1("/a/b");
-            CanonPath p2("/");
-            ASSERT_EQ((p1 + p2).abs(), "/a/b");
-        }
-
-        {
-            CanonPath p("/foo/bar");
-            ASSERT_EQ((p + "x").abs(), "/foo/bar/x");
-        }
-
-        {
-            CanonPath p("/");
-            ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar");
-        }
-    }
-
-    TEST(CanonPath, within) {
-        ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo")));
-        ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar")));
-        ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo")));
-        ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo")));
-        ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar")));
-        ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/")));
-        ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/")));
-    }
-
-    TEST(CanonPath, sort) {
-        ASSERT_FALSE(CanonPath("foo") < CanonPath("foo"));
-        ASSERT_TRUE (CanonPath("foo") < CanonPath("foo/bar"));
-        ASSERT_TRUE (CanonPath("foo/bar") < CanonPath("foo!"));
-        ASSERT_FALSE(CanonPath("foo!") < CanonPath("foo"));
-        ASSERT_TRUE (CanonPath("foo") < CanonPath("foo!"));
-    }
-
-    TEST(CanonPath, allowed) {
-        std::set<CanonPath> allowed {
-            CanonPath("foo/bar"),
-            CanonPath("foo!"),
-            CanonPath("xyzzy"),
-            CanonPath("a/b/c"),
-        };
-
-        ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed));
-        ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed));
-        ASSERT_TRUE (CanonPath("foo").isAllowed(allowed));
-        ASSERT_FALSE(CanonPath("bar").isAllowed(allowed));
-        ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed));
-        ASSERT_TRUE (CanonPath("a").isAllowed(allowed));
-        ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed));
-        ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed));
-        ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed));
-        ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed));
-        ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed));
-        ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed));
-        ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed));
-        ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed));
-        ASSERT_TRUE (CanonPath("/").isAllowed(allowed));
-    }
-
-    TEST(CanonPath, makeRelative) {
-        CanonPath d("/foo/bar");
-        ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar")), ".");
-        ASSERT_EQ(d.makeRelative(CanonPath("/foo")), "..");
-        ASSERT_EQ(d.makeRelative(CanonPath("/")), "../..");
-        ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy")), "xyzzy");
-        ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy/bla")), "xyzzy/bla");
-        ASSERT_EQ(d.makeRelative(CanonPath("/foo/xyzzy/bla")), "../xyzzy/bla");
-        ASSERT_EQ(d.makeRelative(CanonPath("/xyzzy/bla")), "../../xyzzy/bla");
-    }
-}

src/libutil/tests/chunked-vector.cc

@@ -1,54 +0,0 @@
-#include "chunked-vector.hh"
-
-#include <gtest/gtest.h>
-
-namespace nix {
-    TEST(ChunkedVector, InitEmpty) {
-        auto v = ChunkedVector<int, 2>(100);
-        ASSERT_EQ(v.size(), 0);
-    }
-
-    TEST(ChunkedVector, GrowsCorrectly) {
-        auto v = ChunkedVector<int, 2>(100);
-        for (auto i = 1; i < 20; i++) {
-            v.add(i);
-            ASSERT_EQ(v.size(), i);
-        }
-    }
-
-    TEST(ChunkedVector, AddAndGet) {
-        auto v = ChunkedVector<int, 2>(100);
-        for (auto i = 1; i < 20; i++) {
-            auto [i2, idx] = v.add(i);
-            auto & i3 = v[idx];
-            ASSERT_EQ(i, i2);
-            ASSERT_EQ(&i2, &i3);
-        }
-    }
-
-    TEST(ChunkedVector, ForEach) {
-        auto v = ChunkedVector<int, 2>(100);
-        for (auto i = 1; i < 20; i++) {
-            v.add(i);
-        }
-        int count = 0;
-        v.forEach([&count](int elt) {
-            count++;
-        });
-        ASSERT_EQ(count, v.size());
-    }
-
-    TEST(ChunkedVector, OverflowOK) {
-        // Similar to the AddAndGet, but intentionnally use a small
-        // initial ChunkedVector to force it to overflow
-        auto v = ChunkedVector<int, 2>(2);
-        for (auto i = 1; i < 20; i++) {
-            auto [i2, idx] = v.add(i);
-            auto & i3 = v[idx];
-            ASSERT_EQ(i, i2);
-            ASSERT_EQ(&i2, &i3);
-        }
-    }
-
-}
-

src/libutil/tests/closure.cc

@@ -1,70 +0,0 @@
-#include "closure.hh"
-#include <gtest/gtest.h>
-
-namespace nix {
-
-using namespace std;
-
-map<string, set<string>> testGraph = {
-    { "A", { "B", "C", "G" } },
-    { "B", { "A" } }, // Loops back to A
-    { "C", { "F" } }, // Indirect reference
-    { "D", { "A" } }, // Not reachable, but has backreferences
-    { "E", {} }, // Just not reachable
-    { "F", {} },
-    { "G", { "G" } }, // Self reference
-};
-
-TEST(closure, correctClosure) {
-    set<string> aClosure;
-    set<string> expectedClosure = {"A", "B", "C", "F", "G"};
-    computeClosure<string>(
-        {"A"},
-        aClosure,
-        [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
-            promise<set<string>> promisedNodes;
-            promisedNodes.set_value(testGraph[currentNode]);
-            processEdges(promisedNodes);
-        }
-    );
-
-    ASSERT_EQ(aClosure, expectedClosure);
-}
-
-TEST(closure, properlyHandlesDirectExceptions) {
-    struct TestExn {};
-    set<string> aClosure;
-    EXPECT_THROW(
-        computeClosure<string>(
-            {"A"},
-            aClosure,
-            [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
-                throw TestExn();
-            }
-        ),
-        TestExn
-    );
-}
-
-TEST(closure, properlyHandlesExceptionsInPromise) {
-    struct TestExn {};
-    set<string> aClosure;
-    EXPECT_THROW(
-        computeClosure<string>(
-            {"A"},
-            aClosure,
-            [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
-                promise<set<string>> promise;
-                try {
-                    throw TestExn();
-                } catch (...) {
-                    promise.set_exception(std::current_exception());
-                }
-                processEdges(promise);
-            }
-        ),
-        TestExn
-    );
-}
-
-}

src/libutil/tests/compression.cc

@@ -1,96 +0,0 @@
-#include "compression.hh"
-#include <gtest/gtest.h>
-
-namespace nix {
-
-    /* ----------------------------------------------------------------------------
-     * compress / decompress
-     * --------------------------------------------------------------------------*/
-
-    TEST(compress, compressWithUnknownMethod) {
-        ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod);
-    }
-
-    TEST(compress, noneMethodDoesNothingToTheInput) {
-        auto o = compress("none", "this-is-a-test");
-
-        ASSERT_EQ(o, "this-is-a-test");
-    }
-
-    TEST(decompress, decompressNoneCompressed) {
-        auto method = "none";
-        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
-        auto o = decompress(method, str);
-
-        ASSERT_EQ(o, str);
-    }
-
-    TEST(decompress, decompressEmptyCompressed) {
-        // Empty-method decompression used e.g. by S3 store
-        // (Content-Encoding == "").
-        auto method = "";
-        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
-        auto o = decompress(method, str);
-
-        ASSERT_EQ(o, str);
-    }
-
-    TEST(decompress, decompressXzCompressed) {
-        auto method = "xz";
-        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
-        auto o = decompress(method, compress(method, str));
-
-        ASSERT_EQ(o, str);
-    }
-
-    TEST(decompress, decompressBzip2Compressed) {
-        auto method = "bzip2";
-        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
-        auto o = decompress(method, compress(method, str));
-
-        ASSERT_EQ(o, str);
-    }
-
-    TEST(decompress, decompressBrCompressed) {
-        auto method = "br";
-        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
-        auto o = decompress(method, compress(method, str));
-
-        ASSERT_EQ(o, str);
-    }
-
-    TEST(decompress, decompressInvalidInputThrowsCompressionError) {
-        auto method = "bzip2";
-        auto str = "this is a string that does not qualify as valid bzip2 data";
-
-        ASSERT_THROW(decompress(method, str), CompressionError);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * compression sinks
-     * --------------------------------------------------------------------------*/
-
-    TEST(makeCompressionSink, noneSinkDoesNothingToInput) {
-        StringSink strSink;
-        auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
-        auto sink = makeCompressionSink("none", strSink);
-        (*sink)(inputString);
-        sink->finish();
-
-        ASSERT_STREQ(strSink.s.c_str(), inputString);
-    }
-
-    TEST(makeCompressionSink, compressAndDecompress) {
-        StringSink strSink;
-        auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
-        auto decompressionSink = makeDecompressionSink("bzip2", strSink);
-        auto sink = makeCompressionSink("bzip2", *decompressionSink);
-
-        (*sink)(inputString);
-        sink->finish();
-        decompressionSink->finish();
-
-        ASSERT_STREQ(strSink.s.c_str(), inputString);
-    }
-
-}

src/libutil/tests/config.cc

@@ -1,295 +0,0 @@
-#include "config.hh"
-#include "args.hh"
-
-#include <sstream>
-#include <gtest/gtest.h>
-#include <nlohmann/json.hpp>
-
-namespace nix {
-
-    /* ----------------------------------------------------------------------------
-     * Config
-     * --------------------------------------------------------------------------*/
-
-    TEST(Config, setUndefinedSetting) {
-        Config config;
-        ASSERT_EQ(config.set("undefined-key", "value"), false);
-    }
-
-    TEST(Config, setDefinedSetting) {
-        Config config;
-        std::string value;
-        Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
-        ASSERT_EQ(config.set("name-of-the-setting", "value"), true);
-    }
-
-    TEST(Config, getDefinedSetting) {
-        Config config;
-        std::string value;
-        std::map<std::string, Config::SettingInfo> settings;
-        Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
-
-        config.getSettings(settings, /* overriddenOnly = */ false);
-        const auto iter = settings.find("name-of-the-setting");
-        ASSERT_NE(iter, settings.end());
-        ASSERT_EQ(iter->second.value, "");
-        ASSERT_EQ(iter->second.description, "description\n");
-    }
-
-    TEST(Config, getDefinedOverriddenSettingNotSet) {
-        Config config;
-        std::string value;
-        std::map<std::string, Config::SettingInfo> settings;
-        Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
-
-        config.getSettings(settings, /* overriddenOnly = */ true);
-        const auto e = settings.find("name-of-the-setting");
-        ASSERT_EQ(e, settings.end());
-    }
-
-    TEST(Config, getDefinedSettingSet1) {
-        Config config;
-        std::string value;
-        std::map<std::string, Config::SettingInfo> settings;
-        Setting<std::string> setting{&config, value, "name-of-the-setting", "description"};
-
-        setting.assign("value");
-
-        config.getSettings(settings, /* overriddenOnly = */ false);
-        const auto iter = settings.find("name-of-the-setting");
-        ASSERT_NE(iter, settings.end());
-        ASSERT_EQ(iter->second.value, "value");
-        ASSERT_EQ(iter->second.description, "description\n");
-    }
-
-    TEST(Config, getDefinedSettingSet2) {
-        Config config;
-        std::map<std::string, Config::SettingInfo> settings;
-        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
-
-        ASSERT_TRUE(config.set("name-of-the-setting", "value"));
-
-        config.getSettings(settings, /* overriddenOnly = */ false);
-        const auto e = settings.find("name-of-the-setting");
-        ASSERT_NE(e, settings.end());
-        ASSERT_EQ(e->second.value, "value");
-        ASSERT_EQ(e->second.description, "description\n");
-    }
-
-    TEST(Config, addSetting) {
-        class TestSetting : public AbstractSetting {
-            public:
-            TestSetting() : AbstractSetting("test", "test", {}) {}
-            void set(const std::string & value, bool append) override {}
-            std::string to_string() const override { return {}; }
-            bool isAppendable() override { return false; }
-        };
-
-        Config config;
-        TestSetting setting;
-
-        ASSERT_FALSE(config.set("test", "value"));
-        config.addSetting(&setting);
-        ASSERT_TRUE(config.set("test", "value"));
-        ASSERT_FALSE(config.set("extra-test", "value"));
-    }
-
-    TEST(Config, withInitialValue) {
-        const StringMap initials = {
-            { "key", "value" },
-        };
-        Config config(initials);
-
-        {
-            std::map<std::string, Config::SettingInfo> settings;
-            config.getSettings(settings, /* overriddenOnly = */ false);
-            ASSERT_EQ(settings.find("key"), settings.end());
-        }
-
-        Setting<std::string> setting{&config, "default-value", "key", "description"};
-
-        {
-            std::map<std::string, Config::SettingInfo> settings;
-            config.getSettings(settings, /* overriddenOnly = */ false);
-            ASSERT_EQ(settings["key"].value, "value");
-        }
-    }
-
-    TEST(Config, resetOverridden) {
-        Config config;
-        config.resetOverridden();
-    }
-
-    TEST(Config, resetOverriddenWithSetting) {
-        Config config;
-        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
-
-        {
-            std::map<std::string, Config::SettingInfo> settings;
-
-            setting.set("foo");
-            ASSERT_EQ(setting.get(), "foo");
-            config.getSettings(settings, /* overriddenOnly = */ true);
-            ASSERT_TRUE(settings.empty());
-        }
-
-        {
-            std::map<std::string, Config::SettingInfo> settings;
-
-            setting.override("bar");
-            ASSERT_TRUE(setting.overridden);
-            ASSERT_EQ(setting.get(), "bar");
-            config.getSettings(settings, /* overriddenOnly = */ true);
-            ASSERT_FALSE(settings.empty());
-        }
-
-        {
-            std::map<std::string, Config::SettingInfo> settings;
-
-            config.resetOverridden();
-            ASSERT_FALSE(setting.overridden);
-            config.getSettings(settings, /* overriddenOnly = */ true);
-            ASSERT_TRUE(settings.empty());
-        }
-    }
-
-    TEST(Config, toJSONOnEmptyConfig) {
-        ASSERT_EQ(Config().toJSON().dump(), "{}");
-    }
-
-    TEST(Config, toJSONOnNonEmptyConfig) {
-        using nlohmann::literals::operator "" _json;
-        Config config;
-        Setting<std::string> setting{
-            &config,
-            "",
-            "name-of-the-setting",
-            "description",
-        };
-        setting.assign("value");
-
-        ASSERT_EQ(config.toJSON(),
-          R"#({
-            "name-of-the-setting": {
-              "aliases": [],
-              "defaultValue": "",
-              "description": "description\n",
-              "documentDefault": true,
-              "value": "value",
-              "experimentalFeature": null
-            }
-          })#"_json);
-    }
-
-    TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) {
-        using nlohmann::literals::operator "" _json;
-        Config config;
-        Setting<std::string> setting{
-            &config,
-            "",
-            "name-of-the-setting",
-            "description",
-            {},
-            true,
-            Xp::Flakes,
-        };
-        setting.assign("value");
-
-        ASSERT_EQ(config.toJSON(),
-          R"#({
-            "name-of-the-setting": {
-              "aliases": [],
-              "defaultValue": "",
-              "description": "description\n",
-              "documentDefault": true,
-              "value": "value",
-              "experimentalFeature": "flakes"
-            }
-          })#"_json);
-    }
-
-    TEST(Config, setSettingAlias) {
-        Config config;
-        Setting<std::string> setting{&config, "", "some-int", "best number", { "another-int" }};
-        ASSERT_TRUE(config.set("some-int", "1"));
-        ASSERT_EQ(setting.get(), "1");
-        ASSERT_TRUE(config.set("another-int", "2"));
-        ASSERT_EQ(setting.get(), "2");
-        ASSERT_TRUE(config.set("some-int", "3"));
-        ASSERT_EQ(setting.get(), "3");
-    }
-
-    /* FIXME: The reapplyUnknownSettings method doesn't seem to do anything
-     * useful (these days).  Whenever we add a new setting to Config the
-     * unknown settings are always considered.  In which case is this function
-     * actually useful? Is there some way to register a Setting without calling
-     * addSetting? */
-    TEST(Config, DISABLED_reapplyUnknownSettings) {
-        Config config;
-        ASSERT_FALSE(config.set("name-of-the-setting", "unknownvalue"));
-        Setting<std::string> setting{&config, "default", "name-of-the-setting", "description"};
-        ASSERT_EQ(setting.get(), "default");
-        config.reapplyUnknownSettings();
-        ASSERT_EQ(setting.get(), "unknownvalue");
-    }
-
-    TEST(Config, applyConfigEmpty) {
-        Config config;
-        std::map<std::string, Config::SettingInfo> settings;
-        config.applyConfig("");
-        config.getSettings(settings);
-        ASSERT_TRUE(settings.empty());
-    }
-
-    TEST(Config, applyConfigEmptyWithComment) {
-        Config config;
-        std::map<std::string, Config::SettingInfo> settings;
-        config.applyConfig("# just a comment");
-        config.getSettings(settings);
-        ASSERT_TRUE(settings.empty());
-    }
-
-    TEST(Config, applyConfigAssignment) {
-        Config config;
-        std::map<std::string, Config::SettingInfo> settings;
-        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
-        config.applyConfig(
-            "name-of-the-setting = value-from-file #useful comment\n"
-            "# name-of-the-setting = foo\n"
-        );
-        config.getSettings(settings);
-        ASSERT_FALSE(settings.empty());
-        ASSERT_EQ(settings["name-of-the-setting"].value, "value-from-file");
-    }
-
-    TEST(Config, applyConfigWithReassignedSetting) {
-        Config config;
-        std::map<std::string, Config::SettingInfo> settings;
-        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
-        config.applyConfig(
-            "name-of-the-setting = first-value\n"
-            "name-of-the-setting = second-value\n"
-        );
-        config.getSettings(settings);
-        ASSERT_FALSE(settings.empty());
-        ASSERT_EQ(settings["name-of-the-setting"].value, "second-value");
-    }
-
-    TEST(Config, applyConfigFailsOnMissingIncludes) {
-        Config config;
-        std::map<std::string, Config::SettingInfo> settings;
-        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
-
-        ASSERT_THROW(config.applyConfig(
-            "name-of-the-setting = value-from-file\n"
-            "# name-of-the-setting = foo\n"
-            "include /nix/store/does/not/exist.nix"
-        ), Error);
-    }
-
-    TEST(Config, applyConfigInvalidThrows) {
-        Config config;
-        ASSERT_THROW(config.applyConfig("value == key"), UsageError);
-        ASSERT_THROW(config.applyConfig("value "), UsageError);
-    }
-}

src/libutil/tests/git.cc

@@ -1,33 +0,0 @@
-#include "git.hh"
-#include <gtest/gtest.h>
-
-namespace nix {
-
-    TEST(GitLsRemote, parseSymrefLineWithReference) {
-        auto line = "ref: refs/head/main	HEAD";
-        auto res = git::parseLsRemoteLine(line);
-        ASSERT_TRUE(res.has_value());
-        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic);
-        ASSERT_EQ(res->target, "refs/head/main");
-        ASSERT_EQ(res->reference, "HEAD");
-    }
-
-    TEST(GitLsRemote, parseSymrefLineWithNoReference) {
-        auto line = "ref: refs/head/main";
-        auto res = git::parseLsRemoteLine(line);
-        ASSERT_TRUE(res.has_value());
-        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic);
-        ASSERT_EQ(res->target, "refs/head/main");
-        ASSERT_EQ(res->reference, std::nullopt);
-    }
-
-    TEST(GitLsRemote, parseObjectRefLine) {
-        auto line = "abc123	refs/head/main";
-        auto res = git::parseLsRemoteLine(line);
-        ASSERT_TRUE(res.has_value());
-        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object);
-        ASSERT_EQ(res->target, "abc123");
-        ASSERT_EQ(res->reference, "refs/head/main");
-    }
-}
-

src/libutil/tests/hash.cc

@@ -1,110 +0,0 @@
-#include <regex>
-
-#include <nlohmann/json.hpp>
-#include <gtest/gtest.h>
-#include <rapidcheck/gtest.h>
-
-#include <hash.hh>
-
-#include "tests/hash.hh"
-
-namespace nix {
-
-    /* ----------------------------------------------------------------------------
-     * hashString
-     * --------------------------------------------------------------------------*/
-
-    TEST(hashString, testKnownMD5Hashes1) {
-        // values taken from: https://tools.ietf.org/html/rfc1321
-        auto s1 = "";
-        auto hash = hashString(HashType::htMD5, s1);
-        ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e");
-    }
-
-    TEST(hashString, testKnownMD5Hashes2) {
-        // values taken from: https://tools.ietf.org/html/rfc1321
-        auto s2 = "abc";
-        auto hash = hashString(HashType::htMD5, s2);
-        ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72");
-    }
-
-    TEST(hashString, testKnownSHA1Hashes1) {
-        // values taken from: https://tools.ietf.org/html/rfc3174
-        auto s = "abc";
-        auto hash = hashString(HashType::htSHA1, s);
-        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d");
-    }
-
-    TEST(hashString, testKnownSHA1Hashes2) {
-        // values taken from: https://tools.ietf.org/html/rfc3174
-        auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
-        auto hash = hashString(HashType::htSHA1, s);
-        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1");
-    }
-
-    TEST(hashString, testKnownSHA256Hashes1) {
-        // values taken from: https://tools.ietf.org/html/rfc4634
-        auto s = "abc";
-
-        auto hash = hashString(HashType::htSHA256, s);
-        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
-                "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
-    }
-
-    TEST(hashString, testKnownSHA256Hashes2) {
-        // values taken from: https://tools.ietf.org/html/rfc4634
-        auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
-        auto hash = hashString(HashType::htSHA256, s);
-        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
-                "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
-    }
-
-    TEST(hashString, testKnownSHA512Hashes1) {
-        // values taken from: https://tools.ietf.org/html/rfc4634
-        auto s = "abc";
-        auto hash = hashString(HashType::htSHA512, s);
-        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
-                "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9"
-                "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd"
-                "454d4423643ce80e2a9ac94fa54ca49f");
-    }
-
-    TEST(hashString, testKnownSHA512Hashes2) {
-        // values taken from: https://tools.ietf.org/html/rfc4634
-        auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu";
-
-        auto hash = hashString(HashType::htSHA512, s);
-        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
-                "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1"
-                "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a"
-                "c7d329eeb6dd26545e96e55b874be909");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * parseHashFormat, parseHashFormatOpt, printHashFormat
-     * --------------------------------------------------------------------------*/
-
-    TEST(hashFormat, testRoundTripPrintParse) {
-        for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) {
-            ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat);
-            ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat);
-        }
-    }
-
-    TEST(hashFormat, testParseHashFormatOptException) {
-        ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt);
-    }
-}
-
-namespace rc {
-using namespace nix;
-
-Gen<Hash> Arbitrary<Hash>::arbitrary()
-{
-    Hash hash(htSHA1);
-    for (size_t i = 0; i < hash.hashSize; ++i)
-        hash.hash[i] = *gen::arbitrary<uint8_t>();
-    return gen::just(hash);
-}
-
-}

src/libutil/tests/hash.hh

@@ -1,16 +0,0 @@
-#pragma once
-///@file
-
-#include <rapidcheck/gen/Arbitrary.h>
-
-#include <hash.hh>
-
-namespace rc {
-using namespace nix;
-
-template<>
-struct Arbitrary<Hash> {
-    static Gen<Hash> arbitrary();
-};
-
-}

src/libutil/tests/hilite.cc

@@ -1,66 +0,0 @@
-#include "hilite.hh"
-
-#include <gtest/gtest.h>
-
-namespace nix {
-/* ----------- tests for fmt.hh -------------------------------------------------*/
-
-    TEST(hiliteMatches, noHighlight) {
-        ASSERT_STREQ(hiliteMatches("Hello, world!", std::vector<std::smatch>(), "(", ")").c_str(), "Hello, world!");
-    }
-
-    TEST(hiliteMatches, simpleHighlight) {
-        std::string str = "Hello, world!";
-        std::regex re = std::regex("world");
-        auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator());
-        ASSERT_STREQ(
-                    hiliteMatches(str, matches, "(", ")").c_str(),
-                    "Hello, (world)!"
-        );
-    }
-
-    TEST(hiliteMatches, multipleMatches) {
-        std::string str = "Hello, world, world, world, world, world, world, Hello!";
-        std::regex re = std::regex("world");
-        auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator());
-        ASSERT_STREQ(
-                    hiliteMatches(str, matches, "(", ")").c_str(),
-                    "Hello, (world), (world), (world), (world), (world), (world), Hello!"
-        );
-    }
-
-    TEST(hiliteMatches, overlappingMatches) {
-        std::string str = "world, Hello, world, Hello, world, Hello, world, Hello, world!";
-        std::regex re = std::regex("Hello, world");
-        std::regex re2 = std::regex("world, Hello");
-        auto v = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator());
-        for(auto it = std::sregex_iterator(str.begin(), str.end(), re2); it != std::sregex_iterator(); ++it) {
-            v.push_back(*it);
-        }
-        ASSERT_STREQ(
-                    hiliteMatches(str, v, "(", ")").c_str(),
-                    "(world, Hello, world, Hello, world, Hello, world, Hello, world)!"
-        );
-    }
-
-    TEST(hiliteMatches, complexOverlappingMatches) {
-        std::string str = "legacyPackages.x86_64-linux.git-crypt";
-        std::vector regexes = {
-            std::regex("t-cry"),
-            std::regex("ux\\.git-cry"),
-            std::regex("git-c"),
-            std::regex("pt"),
-        };
-        std::vector<std::smatch> matches;
-        for(auto regex : regexes)
-        {
-            for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) {
-                matches.push_back(*it);
-            }
-        }
-        ASSERT_STREQ(
-                    hiliteMatches(str, matches, "(", ")").c_str(),
-                    "legacyPackages.x86_64-lin(ux.git-crypt)"
-        );
-    }
-}

src/libutil/tests/local.mk

@@ -1,29 +0,0 @@
-check: libutil-tests_RUN
-
-programs += libutil-tests
-
-libutil-tests-exe_NAME = libnixutil-tests
-
-libutil-tests-exe_DIR := $(d)
-
-libutil-tests-exe_INSTALL_DIR :=
-
-libutil-tests-exe_LIBS = libutil-tests
-
-libutil-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libutil-tests
-
-libutil-tests_NAME = libnixutil-tests
-
-libutil-tests_DIR := $(d)
-
-libutil-tests_INSTALL_DIR :=
-
-libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libutil-tests_CXXFLAGS += -I src/libutil
-
-libutil-tests_LIBS = libutil
-
-libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

src/libutil/tests/logging.cc

@@ -1,370 +0,0 @@
-#if 0
-
-#include "logging.hh"
-#include "nixexpr.hh"
-#include "util.hh"
-#include <fstream>
-
-#include <gtest/gtest.h>
-
-namespace nix {
-
-    /* ----------------------------------------------------------------------------
-     * logEI
-     * --------------------------------------------------------------------------*/
-
-    const char *test_file =
-        "previous line of code\n"
-        "this is the problem line of code\n"
-        "next line of code\n";
-    const char *one_liner =
-        "this is the other problem line of code";
-
-    TEST(logEI, catpuresBasicProperties) {
-
-        MakeError(TestError, Error);
-        ErrorInfo::programName = std::optional("error-unit-test");
-
-        try {
-            throw TestError("an error for testing purposes");
-        } catch (Error &e) {
-            testing::internal::CaptureStderr();
-            logger->logEI(e.info());
-            auto str = testing::internal::GetCapturedStderr();
-
-            ASSERT_STREQ(str.c_str(),"\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\nan error for testing purposes\n");
-        }
-    }
-
-    TEST(logEI, jsonOutput) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create("random.nix");
-        testing::internal::CaptureStderr();
-
-        makeJSONLogger(*logger)->logEI({
-                .name = "error name",
-                .msg = hintfmt("this hint has %1% templated %2%!!",
-                    "yellow",
-                    "values"),
-                .errPos = Pos(foFile, problem_file, 02, 13)
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
-    }
-
-    TEST(logEI, appendingHintsToPreviousError) {
-
-        MakeError(TestError, Error);
-        ErrorInfo::programName = std::optional("error-unit-test");
-
-        try {
-            auto e = Error("initial error");
-            throw TestError(e.info());
-        } catch (Error &e) {
-            ErrorInfo ei = e.info();
-            ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str()));
-
-            testing::internal::CaptureStderr();
-            logger->logEI(ei);
-            auto str = testing::internal::GetCapturedStderr();
-
-            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\ninitial error; subsequent error message.\n");
-        }
-
-    }
-
-    TEST(logEI, picksUpSysErrorExitCode) {
-
-        MakeError(TestError, Error);
-        ErrorInfo::programName = std::optional("error-unit-test");
-
-        try {
-            auto x = readFile(-1);
-        }
-        catch (SysError &e) {
-            testing::internal::CaptureStderr();
-            logError(e.info());
-            auto str = testing::internal::GetCapturedStderr();
-
-            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n");
-        }
-    }
-
-    TEST(logEI, loggingErrorOnInfoLevel) {
-        testing::internal::CaptureStderr();
-
-        logger->logEI({ .level = lvlInfo,
-                        .name = "Info name",
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name --- error-unit-test\x1B[0m\nInfo description\n");
-    }
-
-    TEST(logEI, loggingErrorOnTalkativeLevel) {
-        verbosity = lvlTalkative;
-
-        testing::internal::CaptureStderr();
-
-        logger->logEI({ .level = lvlTalkative,
-                        .name = "Talkative name",
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name --- error-unit-test\x1B[0m\nTalkative description\n");
-    }
-
-    TEST(logEI, loggingErrorOnChattyLevel) {
-        verbosity = lvlChatty;
-
-        testing::internal::CaptureStderr();
-
-        logger->logEI({ .level = lvlChatty,
-                        .name = "Chatty name",
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name --- error-unit-test\x1B[0m\nTalkative description\n");
-    }
-
-    TEST(logEI, loggingErrorOnDebugLevel) {
-        verbosity = lvlDebug;
-
-        testing::internal::CaptureStderr();
-
-        logger->logEI({ .level = lvlDebug,
-                        .name = "Debug name",
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name --- error-unit-test\x1B[0m\nDebug description\n");
-    }
-
-    TEST(logEI, loggingErrorOnVomitLevel) {
-        verbosity = lvlVomit;
-
-        testing::internal::CaptureStderr();
-
-        logger->logEI({ .level = lvlVomit,
-                        .name = "Vomit name",
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name --- error-unit-test\x1B[0m\nVomit description\n");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * logError
-     * --------------------------------------------------------------------------*/
-
-    TEST(logError, logErrorWithoutHintOrCode) {
-        testing::internal::CaptureStderr();
-
-        logError({
-                .name = "name",
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n");
-    }
-
-    TEST(logError, logErrorWithPreviousAndNextLinesOfCode) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create(test_file);
-
-        testing::internal::CaptureStderr();
-
-        logError({
-                .name = "error name",
-                .msg = hintfmt("this hint has %1% templated %2%!!",
-                    "yellow",
-                    "values"),
-                .errPos = Pos(foString, problem_file, 02, 13),
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nerror with code lines\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
-    }
-
-    TEST(logError, logErrorWithInvalidFile) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create("invalid filename");
-        testing::internal::CaptureStderr();
-
-        logError({
-                .name = "error name",
-                .msg = hintfmt("this hint has %1% templated %2%!!",
-                    "yellow",
-                    "values"),
-                .errPos = Pos(foFile, problem_file, 02, 13)
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
-    }
-
-    TEST(logError, logErrorWithOnlyHintAndName) {
-        testing::internal::CaptureStderr();
-
-        logError({
-                .name = "error name",
-                .msg = hintfmt("hint %1%", "only"),
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nhint \x1B[33;1monly\x1B[0m\n");
-
-    }
-
-    /* ----------------------------------------------------------------------------
-     * logWarning
-     * --------------------------------------------------------------------------*/
-
-    TEST(logWarning, logWarningWithNameDescriptionAndHint) {
-        testing::internal::CaptureStderr();
-
-        logWarning({
-                .name = "name",
-                .msg = hintfmt("there was a %1%", "warning"),
-            });
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nwarning description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
-    }
-
-    TEST(logWarning, logWarningWithFileLineNumAndCode) {
-
-        SymbolTable testTable;
-        auto problem_file = testTable.create(test_file);
-
-        testing::internal::CaptureStderr();
-
-        logWarning({
-                .name = "warning name",
-                .msg = hintfmt("this hint has %1% templated %2%!!",
-                    "yellow",
-                    "values"),
-                .errPos = Pos(foStdin, problem_file, 2, 13),
-            });
-
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from stdin\x1B[0m\n\nwarning description\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * traces
-     * --------------------------------------------------------------------------*/
-
-    TEST(addTrace, showTracesWithShowTrace) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create(test_file);
-        auto oneliner_file = testTable.create(one_liner);
-        auto invalidfilename = testTable.create("invalid filename");
-
-        auto e = AssertionError(ErrorInfo {
-                .name = "wat",
-                .msg = hintfmt("it has been %1% days since our last error", "zero"),
-                .errPos = Pos(foString, problem_file, 2, 13),
-            });
-
-        e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
-        e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
-        e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
-
-        testing::internal::CaptureStderr();
-
-        loggerSettings.showTrace.assign(true);
-
-        logError(e.info());
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n     1| this is the other problem line of code\n      |                   \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
-    }
-
-    TEST(addTrace, hideTracesWithoutShowTrace) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create(test_file);
-        auto oneliner_file = testTable.create(one_liner);
-        auto invalidfilename = testTable.create("invalid filename");
-
-        auto e = AssertionError(ErrorInfo {
-                .name = "wat",
-                .msg = hintfmt("it has been %1% days since our last error", "zero"),
-                .errPos = Pos(foString, problem_file, 2, 13),
-            });
-
-        e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
-        e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
-        e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
-
-        testing::internal::CaptureStderr();
-
-        loggerSettings.showTrace.assign(false);
-
-        logError(e.info());
-
-        auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nhide traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n");
-    }
-
-
-    /* ----------------------------------------------------------------------------
-     * hintfmt
-     * --------------------------------------------------------------------------*/
-
-    TEST(hintfmt, percentStringWithoutArgs) {
-
-        const char *teststr = "this is 100%s correct!";
-
-        ASSERT_STREQ(
-            hintfmt(teststr).str().c_str(),
-            teststr);
-
-    }
-
-    TEST(hintfmt, fmtToHintfmt) {
-
-        ASSERT_STREQ(
-            hintfmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(),
-            "the color of this this text is not yellow");
-
-    }
-
-    TEST(hintfmt, tooFewArguments) {
-
-        ASSERT_STREQ(
-            hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(),
-            "only one arg " ANSI_WARNING "fulfilled" ANSI_NORMAL " ");
-
-    }
-
-    TEST(hintfmt, tooManyArguments) {
-
-        ASSERT_STREQ(
-            hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(),
-            "what about this " ANSI_WARNING "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
-
-    }
-
-    /* ----------------------------------------------------------------------------
-     * ErrPos
-     * --------------------------------------------------------------------------*/
-
-    TEST(errpos, invalidPos) {
-
-      // contains an invalid symbol, which we should not dereference!
-      Pos invalid;
-
-      // constructing without access violation.
-      ErrPos ep(invalid);
-
-      // assignment without access violation.
-      ep = invalid;
-
-    }
-
-}
-
-#endif

src/libutil/tests/lru-cache.cc

@@ -1,130 +0,0 @@
-#include "lru-cache.hh"
-#include <gtest/gtest.h>
-
-namespace nix {
-
-    /* ----------------------------------------------------------------------------
-     * size
-     * --------------------------------------------------------------------------*/
-
-    TEST(LRUCache, sizeOfEmptyCacheIsZero) {
-        LRUCache<std::string, std::string> c(10);
-        ASSERT_EQ(c.size(), 0);
-    }
-
-    TEST(LRUCache, sizeOfSingleElementCacheIsOne) {
-        LRUCache<std::string, std::string> c(10);
-        c.upsert("foo", "bar");
-        ASSERT_EQ(c.size(), 1);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * upsert / get
-     * --------------------------------------------------------------------------*/
-
-    TEST(LRUCache, getFromEmptyCache) {
-        LRUCache<std::string, std::string> c(10);
-        auto val = c.get("x");
-        ASSERT_EQ(val.has_value(), false);
-    }
-
-    TEST(LRUCache, getExistingValue) {
-        LRUCache<std::string, std::string> c(10);
-        c.upsert("foo", "bar");
-        auto val = c.get("foo");
-        ASSERT_EQ(val, "bar");
-    }
-
-    TEST(LRUCache, getNonExistingValueFromNonEmptyCache) {
-        LRUCache<std::string, std::string> c(10);
-        c.upsert("foo", "bar");
-        auto val = c.get("another");
-        ASSERT_EQ(val.has_value(), false);
-    }
-
-    TEST(LRUCache, upsertOnZeroCapacityCache) {
-        LRUCache<std::string, std::string> c(0);
-        c.upsert("foo", "bar");
-        auto val = c.get("foo");
-        ASSERT_EQ(val.has_value(), false);
-    }
-
-    TEST(LRUCache, updateExistingValue) {
-        LRUCache<std::string, std::string> c(1);
-        c.upsert("foo", "bar");
-
-        auto val = c.get("foo");
-        ASSERT_EQ(val.value_or("error"), "bar");
-        ASSERT_EQ(c.size(), 1);
-
-        c.upsert("foo", "changed");
-        val = c.get("foo");
-        ASSERT_EQ(val.value_or("error"), "changed");
-        ASSERT_EQ(c.size(), 1);
-    }
-
-    TEST(LRUCache, overwriteOldestWhenCapacityIsReached) {
-        LRUCache<std::string, std::string> c(3);
-        c.upsert("one", "eins");
-        c.upsert("two", "zwei");
-        c.upsert("three", "drei");
-
-        ASSERT_EQ(c.size(), 3);
-        ASSERT_EQ(c.get("one").value_or("error"), "eins");
-
-        // exceed capacity
-        c.upsert("another", "whatever");
-
-        ASSERT_EQ(c.size(), 3);
-        // Retrieving "one" makes it the most recent element thus
-        // two will be the oldest one and thus replaced.
-        ASSERT_EQ(c.get("two").has_value(), false);
-        ASSERT_EQ(c.get("another").value(), "whatever");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * clear
-     * --------------------------------------------------------------------------*/
-
-    TEST(LRUCache, clearEmptyCache) {
-        LRUCache<std::string, std::string> c(10);
-        c.clear();
-        ASSERT_EQ(c.size(), 0);
-    }
-
-    TEST(LRUCache, clearNonEmptyCache) {
-        LRUCache<std::string, std::string> c(10);
-        c.upsert("one", "eins");
-        c.upsert("two", "zwei");
-        c.upsert("three", "drei");
-        ASSERT_EQ(c.size(), 3);
-        c.clear();
-        ASSERT_EQ(c.size(), 0);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * erase
-     * --------------------------------------------------------------------------*/
-
-    TEST(LRUCache, eraseFromEmptyCache) {
-        LRUCache<std::string, std::string> c(10);
-        ASSERT_EQ(c.erase("foo"), false);
-        ASSERT_EQ(c.size(), 0);
-    }
-
-    TEST(LRUCache, eraseMissingFromNonEmptyCache) {
-        LRUCache<std::string, std::string> c(10);
-        c.upsert("one", "eins");
-        ASSERT_EQ(c.erase("foo"), false);
-        ASSERT_EQ(c.size(), 1);
-        ASSERT_EQ(c.get("one").value_or("error"), "eins");
-    }
-
-    TEST(LRUCache, eraseFromNonEmptyCache) {
-        LRUCache<std::string, std::string> c(10);
-        c.upsert("one", "eins");
-        ASSERT_EQ(c.erase("one"), true);
-        ASSERT_EQ(c.size(), 0);
-        ASSERT_EQ(c.get("one").value_or("empty"), "empty");
-    }
-}

src/libutil/tests/pool.cc

@@ -1,127 +0,0 @@
-#include "pool.hh"
-#include <gtest/gtest.h>
-
-namespace nix {
-
-    struct TestResource
-    {
-
-        TestResource() {
-            static int counter = 0;
-            num = counter++;
-        }
-
-        int dummyValue = 1;
-        bool good = true;
-        int num;
-    };
-
-    /* ----------------------------------------------------------------------------
-     * Pool
-     * --------------------------------------------------------------------------*/
-
-    TEST(Pool, freshPoolHasZeroCountAndSpecifiedCapacity) {
-        auto isGood = [](const ref<TestResource> & r) { return r->good; };
-        auto createResource = []() { return make_ref<TestResource>(); };
-
-        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
-
-        ASSERT_EQ(pool.count(), 0);
-        ASSERT_EQ(pool.capacity(), 1);
-    }
-
-    TEST(Pool, freshPoolCanGetAResource) {
-        auto isGood = [](const ref<TestResource> & r) { return r->good; };
-        auto createResource = []() { return make_ref<TestResource>(); };
-
-        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
-        ASSERT_EQ(pool.count(), 0);
-
-        TestResource r = *(pool.get());
-
-        ASSERT_EQ(pool.count(), 1);
-        ASSERT_EQ(pool.capacity(), 1);
-        ASSERT_EQ(r.dummyValue, 1);
-        ASSERT_EQ(r.good, true);
-    }
-
-    TEST(Pool, capacityCanBeIncremented) {
-        auto isGood = [](const ref<TestResource> & r) { return r->good; };
-        auto createResource = []() { return make_ref<TestResource>(); };
-
-        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
-        ASSERT_EQ(pool.capacity(), 1);
-        pool.incCapacity();
-        ASSERT_EQ(pool.capacity(), 2);
-    }
-
-    TEST(Pool, capacityCanBeDecremented) {
-        auto isGood = [](const ref<TestResource> & r) { return r->good; };
-        auto createResource = []() { return make_ref<TestResource>(); };
-
-        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
-        ASSERT_EQ(pool.capacity(), 1);
-        pool.decCapacity();
-        ASSERT_EQ(pool.capacity(), 0);
-    }
-
-    TEST(Pool, flushBadDropsOutOfScopeResources) {
-        auto isGood = [](const ref<TestResource> & r) { return false; };
-        auto createResource = []() { return make_ref<TestResource>(); };
-
-        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
-
-        {
-            auto _r = pool.get();
-            ASSERT_EQ(pool.count(), 1);
-        }
-
-        pool.flushBad();
-        ASSERT_EQ(pool.count(), 0);
-    }
-
-    // Test that the resources we allocate are being reused when they are still good.
-    TEST(Pool, reuseResource) {
-        auto isGood = [](const ref<TestResource> & r) { return true; };
-        auto createResource = []() { return make_ref<TestResource>(); };
-
-        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
-
-        // Compare the instance counter between the two handles. We expect them to be equal
-        // as the pool should hand out the same (still) good one again.
-        int counter = -1;
-        {
-            Pool<TestResource>::Handle h = pool.get();
-            counter = h->num;
-        } // the first handle goes out of scope
-
-        { // the second handle should contain the same resource (with the same counter value)
-            Pool<TestResource>::Handle h = pool.get();
-            ASSERT_EQ(h->num, counter);
-        }
-    }
-
-    // Test that the resources we allocate are being thrown away when they are no longer good.
-    TEST(Pool, badResourceIsNotReused) {
-        auto isGood = [](const ref<TestResource> & r) { return false; };
-        auto createResource = []() { return make_ref<TestResource>(); };
-
-        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
-
-        // Compare the instance counter between the two handles. We expect them
-        // to *not* be equal as the pool should hand out a new instance after
-        // the first one was returned.
-        int counter = -1;
-        {
-            Pool<TestResource>::Handle h = pool.get();
-            counter = h->num;
-        } // the first handle goes out of scope
-
-        {
-          // the second handle should contain a different resource (with a
-          //different counter value)
-            Pool<TestResource>::Handle h = pool.get();
-            ASSERT_NE(h->num, counter);
-        }
-    }
-}

src/libutil/tests/references.cc

@@ -1,46 +0,0 @@
-#include "references.hh"
-#include <gtest/gtest.h>
-
-namespace nix {
-
-using std::string;
-
-struct RewriteParams {
-    string originalString, finalString;
-    StringMap rewrites;
-
-    friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) {
-        StringSet strRewrites;
-        for (auto & [from, to] : bar.rewrites)
-            strRewrites.insert(from + "->" + to);
-        return os <<
-            "OriginalString: " << bar.originalString << std::endl <<
-            "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl <<
-            "Expected result: " << bar.finalString;
-    }
-};
-
-class RewriteTest : public ::testing::TestWithParam<RewriteParams> {
-};
-
-TEST_P(RewriteTest, IdentityRewriteIsIdentity) {
-    RewriteParams param = GetParam();
-    StringSink rewritten;
-    auto rewriter = RewritingSink(param.rewrites, rewritten);
-    rewriter(param.originalString);
-    rewriter.flush();
-    ASSERT_EQ(rewritten.s, param.finalString);
-}
-
-INSTANTIATE_TEST_CASE_P(
-    references,
-    RewriteTest,
-    ::testing::Values(
-        RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}},
-        RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}},
-        RewriteParams{ "foooo", "foooo", {}}
-    )
-);
-
-}
-

src/libutil/tests/suggestions.cc

@@ -1,43 +0,0 @@
-#include "suggestions.hh"
-#include <gtest/gtest.h>
-
-namespace nix {
-
-    struct LevenshteinDistanceParam {
-        std::string s1, s2;
-        int distance;
-    };
-
-    class LevenshteinDistanceTest :
-        public testing::TestWithParam<LevenshteinDistanceParam> {
-    };
-
-    TEST_P(LevenshteinDistanceTest, CorrectlyComputed) {
-        auto params = GetParam();
-
-        ASSERT_EQ(levenshteinDistance(params.s1, params.s2), params.distance);
-        ASSERT_EQ(levenshteinDistance(params.s2, params.s1), params.distance);
-    }
-
-    INSTANTIATE_TEST_SUITE_P(LevenshteinDistance, LevenshteinDistanceTest,
-            testing::Values(
-                LevenshteinDistanceParam{"foo", "foo", 0},
-                LevenshteinDistanceParam{"foo", "", 3},
-                LevenshteinDistanceParam{"", "", 0},
-                LevenshteinDistanceParam{"foo", "fo", 1},
-                LevenshteinDistanceParam{"foo", "oo", 1},
-                LevenshteinDistanceParam{"foo", "fao", 1},
-                LevenshteinDistanceParam{"foo", "abc", 3}
-            )
-    );
-
-    TEST(Suggestions, Trim) {
-        auto suggestions = Suggestions::bestMatches({"foooo", "bar", "fo", "gao"}, "foo");
-        auto onlyOne = suggestions.trim(1);
-        ASSERT_EQ(onlyOne.suggestions.size(), 1);
-        ASSERT_TRUE(onlyOne.suggestions.begin()->suggestion == "fo");
-
-        auto closest = suggestions.trim(999, 2);
-        ASSERT_EQ(closest.suggestions.size(), 3);
-    }
-}

src/libutil/tests/tests.cc

@@ -1,659 +0,0 @@
-#include "util.hh"
-#include "types.hh"
-
-#include <limits.h>
-#include <gtest/gtest.h>
-
-#include <numeric>
-
-namespace nix {
-
-/* ----------- tests for util.hh ------------------------------------------------*/
-
-    /* ----------------------------------------------------------------------------
-     * absPath
-     * --------------------------------------------------------------------------*/
-
-    TEST(absPath, doesntChangeRoot) {
-        auto p = absPath("/");
-
-        ASSERT_EQ(p, "/");
-    }
-
-
-
-
-    TEST(absPath, turnsEmptyPathIntoCWD) {
-        char cwd[PATH_MAX+1];
-        auto p = absPath("");
-
-        ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX));
-    }
-
-    TEST(absPath, usesOptionalBasePathWhenGiven) {
-        char _cwd[PATH_MAX+1];
-        char* cwd = getcwd((char*)&_cwd, PATH_MAX);
-
-        auto p = absPath("", cwd);
-
-        ASSERT_EQ(p, cwd);
-    }
-
-    TEST(absPath, isIdempotent) {
-        char _cwd[PATH_MAX+1];
-        char* cwd = getcwd((char*)&_cwd, PATH_MAX);
-        auto p1 = absPath(cwd);
-        auto p2 = absPath(p1);
-
-        ASSERT_EQ(p1, p2);
-    }
-
-
-    TEST(absPath, pathIsCanonicalised) {
-        auto path = "/some/path/with/trailing/dot/.";
-        auto p1 = absPath(path);
-        auto p2 = absPath(p1);
-
-        ASSERT_EQ(p1, "/some/path/with/trailing/dot");
-        ASSERT_EQ(p1, p2);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * canonPath
-     * --------------------------------------------------------------------------*/
-
-    TEST(canonPath, removesTrailingSlashes) {
-        auto path = "/this/is/a/path//";
-        auto p = canonPath(path);
-
-        ASSERT_EQ(p, "/this/is/a/path");
-    }
-
-    TEST(canonPath, removesDots) {
-        auto path = "/this/./is/a/path/./";
-        auto p = canonPath(path);
-
-        ASSERT_EQ(p, "/this/is/a/path");
-    }
-
-    TEST(canonPath, removesDots2) {
-        auto path = "/this/a/../is/a////path/foo/..";
-        auto p = canonPath(path);
-
-        ASSERT_EQ(p, "/this/is/a/path");
-    }
-
-    TEST(canonPath, requiresAbsolutePath) {
-        ASSERT_ANY_THROW(canonPath("."));
-        ASSERT_ANY_THROW(canonPath(".."));
-        ASSERT_ANY_THROW(canonPath("../"));
-        ASSERT_DEATH({ canonPath(""); }, "path != \"\"");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * dirOf
-     * --------------------------------------------------------------------------*/
-
-    TEST(dirOf, returnsEmptyStringForRoot) {
-        auto p = dirOf("/");
-
-        ASSERT_EQ(p, "/");
-    }
-
-    TEST(dirOf, returnsFirstPathComponent) {
-        auto p1 = dirOf("/dir/");
-        ASSERT_EQ(p1, "/dir");
-        auto p2 = dirOf("/dir");
-        ASSERT_EQ(p2, "/");
-        auto p3 = dirOf("/dir/..");
-        ASSERT_EQ(p3, "/dir");
-        auto p4 = dirOf("/dir/../");
-        ASSERT_EQ(p4, "/dir/..");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * baseNameOf
-     * --------------------------------------------------------------------------*/
-
-    TEST(baseNameOf, emptyPath) {
-        auto p1 = baseNameOf("");
-        ASSERT_EQ(p1, "");
-    }
-
-    TEST(baseNameOf, pathOnRoot) {
-        auto p1 = baseNameOf("/dir");
-        ASSERT_EQ(p1, "dir");
-    }
-
-    TEST(baseNameOf, relativePath) {
-        auto p1 = baseNameOf("dir/foo");
-        ASSERT_EQ(p1, "foo");
-    }
-
-    TEST(baseNameOf, pathWithTrailingSlashRoot) {
-        auto p1 = baseNameOf("/");
-        ASSERT_EQ(p1, "");
-    }
-
-    TEST(baseNameOf, trailingSlash) {
-        auto p1 = baseNameOf("/dir/");
-        ASSERT_EQ(p1, "dir");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * isInDir
-     * --------------------------------------------------------------------------*/
-
-    TEST(isInDir, trivialCase) {
-        auto p1 = isInDir("/foo/bar", "/foo");
-        ASSERT_EQ(p1, true);
-    }
-
-    TEST(isInDir, notInDir) {
-        auto p1 = isInDir("/zes/foo/bar", "/foo");
-        ASSERT_EQ(p1, false);
-    }
-
-    // XXX: hm, bug or feature? :) Looking at the implementation
-    // this might be problematic.
-    TEST(isInDir, emptyDir) {
-        auto p1 = isInDir("/zes/foo/bar", "");
-        ASSERT_EQ(p1, true);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * isDirOrInDir
-     * --------------------------------------------------------------------------*/
-
-    TEST(isDirOrInDir, trueForSameDirectory) {
-        ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true);
-        ASSERT_EQ(isDirOrInDir("/", "/"), true);
-    }
-
-    TEST(isDirOrInDir, trueForEmptyPaths) {
-        ASSERT_EQ(isDirOrInDir("", ""), true);
-    }
-
-    TEST(isDirOrInDir, falseForDisjunctPaths) {
-        ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false);
-    }
-
-    TEST(isDirOrInDir, relativePaths) {
-        ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true);
-    }
-
-    // XXX: while it is possible to use "." or ".." in the
-    // first argument this doesn't seem to work in the second.
-    TEST(isDirOrInDir, DISABLED_shouldWork) {
-        ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true);
-
-    }
-
-    /* ----------------------------------------------------------------------------
-     * pathExists
-     * --------------------------------------------------------------------------*/
-
-    TEST(pathExists, rootExists) {
-        ASSERT_TRUE(pathExists("/"));
-    }
-
-    TEST(pathExists, cwdExists) {
-        ASSERT_TRUE(pathExists("."));
-    }
-
-    TEST(pathExists, bogusPathDoesNotExist) {
-        ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes"));
-    }
-
-    /* ----------------------------------------------------------------------------
-     * concatStringsSep
-     * --------------------------------------------------------------------------*/
-
-    TEST(concatStringsSep, buildCommaSeparatedString) {
-        Strings strings;
-        strings.push_back("this");
-        strings.push_back("is");
-        strings.push_back("great");
-
-        ASSERT_EQ(concatStringsSep(",", strings), "this,is,great");
-    }
-
-    TEST(concatStringsSep, buildStringWithEmptySeparator) {
-        Strings strings;
-        strings.push_back("this");
-        strings.push_back("is");
-        strings.push_back("great");
-
-        ASSERT_EQ(concatStringsSep("", strings), "thisisgreat");
-    }
-
-    TEST(concatStringsSep, buildSingleString) {
-        Strings strings;
-        strings.push_back("this");
-
-        ASSERT_EQ(concatStringsSep(",", strings), "this");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * hasPrefix
-     * --------------------------------------------------------------------------*/
-
-    TEST(hasPrefix, emptyStringHasNoPrefix) {
-        ASSERT_FALSE(hasPrefix("", "foo"));
-    }
-
-    TEST(hasPrefix, emptyStringIsAlwaysPrefix) {
-        ASSERT_TRUE(hasPrefix("foo", ""));
-        ASSERT_TRUE(hasPrefix("jshjkfhsadf", ""));
-    }
-
-    TEST(hasPrefix, trivialCase) {
-        ASSERT_TRUE(hasPrefix("foobar", "foo"));
-    }
-
-    /* ----------------------------------------------------------------------------
-     * hasSuffix
-     * --------------------------------------------------------------------------*/
-
-    TEST(hasSuffix, emptyStringHasNoSuffix) {
-        ASSERT_FALSE(hasSuffix("", "foo"));
-    }
-
-    TEST(hasSuffix, trivialCase) {
-        ASSERT_TRUE(hasSuffix("foo", "foo"));
-        ASSERT_TRUE(hasSuffix("foobar", "bar"));
-    }
-
-    /* ----------------------------------------------------------------------------
-     * base64Encode
-     * --------------------------------------------------------------------------*/
-
-    TEST(base64Encode, emptyString) {
-        ASSERT_EQ(base64Encode(""), "");
-    }
-
-    TEST(base64Encode, encodesAString) {
-        ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=");
-    }
-
-    TEST(base64Encode, encodeAndDecode) {
-        auto s = "quod erat demonstrandum";
-        auto encoded = base64Encode(s);
-        auto decoded = base64Decode(encoded);
-
-        ASSERT_EQ(decoded, s);
-    }
-
-    TEST(base64Encode, encodeAndDecodeNonPrintable) {
-        char s[256];
-        std::iota(std::rbegin(s), std::rend(s), 0);
-
-        auto encoded = base64Encode(s);
-        auto decoded = base64Decode(encoded);
-
-        EXPECT_EQ(decoded.length(), 255);
-        ASSERT_EQ(decoded, s);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * base64Decode
-     * --------------------------------------------------------------------------*/
-
-    TEST(base64Decode, emptyString) {
-        ASSERT_EQ(base64Decode(""), "");
-    }
-
-    TEST(base64Decode, decodeAString) {
-        ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum");
-    }
-
-    TEST(base64Decode, decodeThrowsOnInvalidChar) {
-        ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * getLine
-     * --------------------------------------------------------------------------*/
-
-    TEST(getLine, all) {
-        {
-            auto [line, rest] = getLine("foo\nbar\nxyzzy");
-            ASSERT_EQ(line, "foo");
-            ASSERT_EQ(rest, "bar\nxyzzy");
-        }
-
-        {
-            auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy");
-            ASSERT_EQ(line, "foo");
-            ASSERT_EQ(rest, "bar\r\nxyzzy");
-        }
-
-        {
-            auto [line, rest] = getLine("foo\n");
-            ASSERT_EQ(line, "foo");
-            ASSERT_EQ(rest, "");
-        }
-
-        {
-            auto [line, rest] = getLine("foo");
-            ASSERT_EQ(line, "foo");
-            ASSERT_EQ(rest, "");
-        }
-
-        {
-            auto [line, rest] = getLine("");
-            ASSERT_EQ(line, "");
-            ASSERT_EQ(rest, "");
-        }
-    }
-
-    /* ----------------------------------------------------------------------------
-     * toLower
-     * --------------------------------------------------------------------------*/
-
-    TEST(toLower, emptyString) {
-        ASSERT_EQ(toLower(""), "");
-    }
-
-    TEST(toLower, nonLetters) {
-        auto s = "!@(*$#)(@#=\\234_";
-        ASSERT_EQ(toLower(s), s);
-    }
-
-    // std::tolower() doesn't handle unicode characters. In the context of
-    // store paths this isn't relevant but doesn't hurt to record this behavior
-    // here.
-    TEST(toLower, umlauts) {
-        auto s = "ÄÖÜ";
-        ASSERT_EQ(toLower(s), "ÄÖÜ");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * string2Float
-     * --------------------------------------------------------------------------*/
-
-    TEST(string2Float, emptyString) {
-        ASSERT_EQ(string2Float<double>(""), std::nullopt);
-    }
-
-    TEST(string2Float, trivialConversions) {
-        ASSERT_EQ(string2Float<double>("1.0"), 1.0);
-
-        ASSERT_EQ(string2Float<double>("0.0"), 0.0);
-
-        ASSERT_EQ(string2Float<double>("-100.25"), -100.25);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * string2Int
-     * --------------------------------------------------------------------------*/
-
-    TEST(string2Int, emptyString) {
-        ASSERT_EQ(string2Int<int>(""), std::nullopt);
-    }
-
-    TEST(string2Int, trivialConversions) {
-        ASSERT_EQ(string2Int<int>("1"), 1);
-
-        ASSERT_EQ(string2Int<int>("0"), 0);
-
-        ASSERT_EQ(string2Int<int>("-100"), -100);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * statusOk
-     * --------------------------------------------------------------------------*/
-
-    TEST(statusOk, zeroIsOk) {
-        ASSERT_EQ(statusOk(0), true);
-        ASSERT_EQ(statusOk(1), false);
-    }
-
-
-    /* ----------------------------------------------------------------------------
-     * rewriteStrings
-     * --------------------------------------------------------------------------*/
-
-    TEST(rewriteStrings, emptyString) {
-        StringMap rewrites;
-        rewrites["this"] = "that";
-
-        ASSERT_EQ(rewriteStrings("", rewrites), "");
-    }
-
-    TEST(rewriteStrings, emptyRewrites) {
-        StringMap rewrites;
-
-        ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that");
-    }
-
-    TEST(rewriteStrings, successfulRewrite) {
-        StringMap rewrites;
-        rewrites["this"] = "that";
-
-        ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that");
-    }
-
-    TEST(rewriteStrings, doesntOccur) {
-        StringMap rewrites;
-        rewrites["foo"] = "bar";
-
-        ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * replaceStrings
-     * --------------------------------------------------------------------------*/
-
-    TEST(replaceStrings, emptyString) {
-        ASSERT_EQ(replaceStrings("", "this", "that"), "");
-        ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that");
-    }
-
-    TEST(replaceStrings, successfulReplace) {
-        ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that");
-    }
-
-    TEST(replaceStrings, doesntOccur) {
-        ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * trim
-     * --------------------------------------------------------------------------*/
-
-    TEST(trim, emptyString) {
-        ASSERT_EQ(trim(""), "");
-    }
-
-    TEST(trim, removesWhitespace) {
-        ASSERT_EQ(trim("foo"), "foo");
-        ASSERT_EQ(trim("     foo "), "foo");
-        ASSERT_EQ(trim("     foo bar baz"), "foo bar baz");
-        ASSERT_EQ(trim("     \t foo bar baz\n"), "foo bar baz");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * chomp
-     * --------------------------------------------------------------------------*/
-
-    TEST(chomp, emptyString) {
-        ASSERT_EQ(chomp(""), "");
-    }
-
-    TEST(chomp, removesWhitespace) {
-        ASSERT_EQ(chomp("foo"), "foo");
-        ASSERT_EQ(chomp("foo "), "foo");
-        ASSERT_EQ(chomp(" foo "), " foo");
-        ASSERT_EQ(chomp(" foo bar baz  "), " foo bar baz");
-        ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * quoteStrings
-     * --------------------------------------------------------------------------*/
-
-    TEST(quoteStrings, empty) {
-        Strings s = { };
-        Strings expected = { };
-
-        ASSERT_EQ(quoteStrings(s), expected);
-    }
-
-    TEST(quoteStrings, emptyStrings) {
-        Strings s = { "", "", "" };
-        Strings expected = { "''", "''", "''" };
-        ASSERT_EQ(quoteStrings(s), expected);
-
-    }
-
-    TEST(quoteStrings, trivialQuote) {
-        Strings s = { "foo", "bar", "baz" };
-        Strings expected = { "'foo'", "'bar'", "'baz'" };
-
-        ASSERT_EQ(quoteStrings(s), expected);
-    }
-
-    TEST(quoteStrings, quotedStrings) {
-        Strings s = { "'foo'", "'bar'", "'baz'" };
-        Strings expected = { "''foo''", "''bar''", "''baz''" };
-
-        ASSERT_EQ(quoteStrings(s), expected);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * tokenizeString
-     * --------------------------------------------------------------------------*/
-
-    TEST(tokenizeString, empty) {
-        Strings expected = { };
-
-        ASSERT_EQ(tokenizeString<Strings>(""), expected);
-    }
-
-    TEST(tokenizeString, tokenizeSpacesWithDefaults) {
-        auto s = "foo bar baz";
-        Strings expected = { "foo", "bar", "baz" };
-
-        ASSERT_EQ(tokenizeString<Strings>(s), expected);
-    }
-
-    TEST(tokenizeString, tokenizeTabsWithDefaults) {
-        auto s = "foo\tbar\tbaz";
-        Strings expected = { "foo", "bar", "baz" };
-
-        ASSERT_EQ(tokenizeString<Strings>(s), expected);
-    }
-
-    TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) {
-        auto s = "foo\t bar\t baz";
-        Strings expected = { "foo", "bar", "baz" };
-
-        ASSERT_EQ(tokenizeString<Strings>(s), expected);
-    }
-
-    TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) {
-        auto s = "foo\t\n bar\t\n baz";
-        Strings expected = { "foo", "bar", "baz" };
-
-        ASSERT_EQ(tokenizeString<Strings>(s), expected);
-    }
-
-    TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) {
-        auto s = "foo\t\n\r bar\t\n\r baz";
-        Strings expected = { "foo", "bar", "baz" };
-
-        ASSERT_EQ(tokenizeString<Strings>(s), expected);
-
-        auto s2 = "foo \t\n\r bar \t\n\r baz";
-        Strings expected2 = { "foo", "bar", "baz" };
-
-        ASSERT_EQ(tokenizeString<Strings>(s2), expected2);
-    }
-
-    TEST(tokenizeString, tokenizeWithCustomSep) {
-        auto s = "foo\n,bar\n,baz\n";
-        Strings expected = { "foo\n", "bar\n", "baz\n" };
-
-        ASSERT_EQ(tokenizeString<Strings>(s, ","), expected);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * get
-     * --------------------------------------------------------------------------*/
-
-    TEST(get, emptyContainer) {
-        StringMap s = { };
-        auto expected = nullptr;
-
-        ASSERT_EQ(get(s, "one"), expected);
-    }
-
-    TEST(get, getFromContainer) {
-        StringMap s;
-        s["one"] = "yi";
-        s["two"] = "er";
-        auto expected = "yi";
-
-        ASSERT_EQ(*get(s, "one"), expected);
-    }
-
-    TEST(getOr, emptyContainer) {
-        StringMap s = { };
-        auto expected = "yi";
-
-        ASSERT_EQ(getOr(s, "one", "yi"), expected);
-    }
-
-    TEST(getOr, getFromContainer) {
-        StringMap s;
-        s["one"] = "yi";
-        s["two"] = "er";
-        auto expected = "yi";
-
-        ASSERT_EQ(getOr(s, "one", "nope"), expected);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * filterANSIEscapes
-     * --------------------------------------------------------------------------*/
-
-    TEST(filterANSIEscapes, emptyString) {
-        auto s = "";
-        auto expected = "";
-
-        ASSERT_EQ(filterANSIEscapes(s), expected);
-    }
-
-    TEST(filterANSIEscapes, doesntChangePrintableChars) {
-        auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f";
-
-        ASSERT_EQ(filterANSIEscapes(s), s);
-    }
-
-    TEST(filterANSIEscapes, filtersColorCodes) {
-        auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m";
-
-        ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" );
-        ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " );
-        ASSERT_EQ(filterANSIEscapes(s, true, 4), " A  " );
-        ASSERT_EQ(filterANSIEscapes(s, true, 5), " A  B" );
-        ASSERT_EQ(filterANSIEscapes(s, true, 8), " A  B  C" );
-    }
-
-    TEST(filterANSIEscapes, expandsTabs) {
-        auto s = "foo\tbar\tbaz";
-
-        ASSERT_EQ(filterANSIEscapes(s, true), "foo     bar     baz" );
-    }
-
-    TEST(filterANSIEscapes, utf8) {
-        ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba");
-        ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär");
-        ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä");
-        ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó");
-        ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b");
-        ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b");
-    }
-
-}

src/libutil/tests/url.cc

@@ -1,347 +0,0 @@
-#include "url.hh"
-#include <gtest/gtest.h>
-
-namespace nix {
-
-/* ----------- tests for url.hh --------------------------------------------------*/
-
-    std::string print_map(std::map<std::string, std::string> m) {
-        std::map<std::string, std::string>::iterator it;
-        std::string s = "{ ";
-        for (it = m.begin(); it != m.end(); ++it) {
-            s += "{ ";
-            s += it->first;
-            s += " = ";
-            s += it->second;
-            s += " } ";
-        }
-        s += "}";
-        return s;
-    }
-
-
-    std::ostream& operator<<(std::ostream& os, const ParsedURL& p) {
-        return os << "\n"
-                  << "url:       " << p.url << "\n"
-                  << "base:      " << p.base << "\n"
-                  << "scheme:    " << p.scheme << "\n"
-                  << "authority: " << p.authority.value() << "\n"
-                  << "path:      " << p.path << "\n"
-                  << "query:     " << print_map(p.query) << "\n"
-                  << "fragment:  " << p.fragment << "\n";
-    }
-
-    TEST(parseURL, parsesSimpleHttpUrl) {
-        auto s = "http://www.example.org/file.tar.gz";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "http://www.example.org/file.tar.gz",
-            .base = "http://www.example.org/file.tar.gz",
-            .scheme = "http",
-            .authority = "www.example.org",
-            .path = "/file.tar.gz",
-            .query = (StringMap) { },
-            .fragment = "",
-        };
-
-        ASSERT_EQ(parsed, expected);
-    }
-
-    TEST(parseURL, parsesSimpleHttpsUrl) {
-        auto s = "https://www.example.org/file.tar.gz";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "https://www.example.org/file.tar.gz",
-            .base = "https://www.example.org/file.tar.gz",
-            .scheme = "https",
-            .authority = "www.example.org",
-            .path = "/file.tar.gz",
-            .query = (StringMap) { },
-            .fragment = "",
-        };
-
-        ASSERT_EQ(parsed, expected);
-    }
-
-    TEST(parseURL, parsesSimpleHttpUrlWithQueryAndFragment) {
-        auto s = "https://www.example.org/file.tar.gz?download=fast&when=now#hello";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "https://www.example.org/file.tar.gz",
-            .base = "https://www.example.org/file.tar.gz",
-            .scheme = "https",
-            .authority = "www.example.org",
-            .path = "/file.tar.gz",
-            .query = (StringMap) { { "download", "fast" }, { "when", "now" } },
-            .fragment = "hello",
-        };
-
-        ASSERT_EQ(parsed, expected);
-    }
-
-    TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) {
-        auto s = "http://www.example.org/file.tar.gz?field=value#?foo=bar%23";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "http://www.example.org/file.tar.gz",
-            .base = "http://www.example.org/file.tar.gz",
-            .scheme = "http",
-            .authority = "www.example.org",
-            .path = "/file.tar.gz",
-            .query = (StringMap) { { "field", "value" } },
-            .fragment = "?foo=bar#",
-        };
-
-        ASSERT_EQ(parsed, expected);
-    }
-
-    TEST(parseURL, parsesFilePlusHttpsUrl) {
-        auto s = "file+https://www.example.org/video.mp4";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "file+https://www.example.org/video.mp4",
-            .base = "https://www.example.org/video.mp4",
-            .scheme = "file+https",
-            .authority = "www.example.org",
-            .path = "/video.mp4",
-            .query = (StringMap) { },
-            .fragment = "",
-        };
-
-        ASSERT_EQ(parsed, expected);
-    }
-
-    TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) {
-        auto s = "file://www.example.org/video.mp4";
-        ASSERT_THROW(parseURL(s), Error);
-    }
-
-    TEST(parseURL, parseIPv4Address) {
-        auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "http://127.0.0.1:8080/file.tar.gz",
-            .base = "https://127.0.0.1:8080/file.tar.gz",
-            .scheme = "http",
-            .authority = "127.0.0.1:8080",
-            .path = "/file.tar.gz",
-            .query = (StringMap) { { "download", "fast" }, { "when", "now" } },
-            .fragment = "hello",
-        };
-
-        ASSERT_EQ(parsed, expected);
-    }
-
-    TEST(parseURL, parseScopedRFC4007IPv6Address) {
-        auto s = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
-            .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
-            .scheme = "http",
-            .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
-            .path = "",
-            .query = (StringMap) { },
-            .fragment = "",
-        };
-
-        ASSERT_EQ(parsed, expected);
-
-    }
-
-    TEST(parseURL, parseIPv6Address) {
-        auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
-            .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
-            .scheme = "http",
-            .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
-            .path = "",
-            .query = (StringMap) { },
-            .fragment = "",
-        };
-
-        ASSERT_EQ(parsed, expected);
-
-    }
-
-    TEST(parseURL, parseEmptyQueryParams) {
-        auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&";
-        auto parsed = parseURL(s);
-        ASSERT_EQ(parsed.query, (StringMap) { });
-    }
-
-    TEST(parseURL, parseUserPassword) {
-        auto s = "http://user:pass@www.example.org:8080/file.tar.gz";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "http://user:pass@www.example.org/file.tar.gz",
-            .base = "http://user:pass@www.example.org/file.tar.gz",
-            .scheme = "http",
-            .authority = "user:pass@www.example.org:8080",
-            .path = "/file.tar.gz",
-            .query = (StringMap) { },
-            .fragment = "",
-        };
-
-
-        ASSERT_EQ(parsed, expected);
-    }
-
-    TEST(parseURL, parseFileURLWithQueryAndFragment) {
-        auto s = "file:///none/of//your/business";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "",
-            .base = "",
-            .scheme = "file",
-            .authority = "",
-            .path = "/none/of//your/business",
-            .query = (StringMap) { },
-            .fragment = "",
-        };
-
-        ASSERT_EQ(parsed, expected);
-
-    }
-
-    TEST(parseURL, parsedUrlsIsEqualToItself) {
-        auto s = "http://www.example.org/file.tar.gz";
-        auto url = parseURL(s);
-
-        ASSERT_TRUE(url == url);
-    }
-
-    TEST(parseURL, parseFTPUrl) {
-        auto s = "ftp://ftp.nixos.org/downloads/nixos.iso";
-        auto parsed = parseURL(s);
-
-        ParsedURL expected {
-            .url = "ftp://ftp.nixos.org/downloads/nixos.iso",
-            .base = "ftp://ftp.nixos.org/downloads/nixos.iso",
-            .scheme = "ftp",
-            .authority = "ftp.nixos.org",
-            .path = "/downloads/nixos.iso",
-            .query = (StringMap) { },
-            .fragment = "",
-        };
-
-        ASSERT_EQ(parsed, expected);
-    }
-
-    TEST(parseURL, parsesAnythingInUriFormat) {
-        auto s = "whatever://github.com/NixOS/nixpkgs.git";
-        auto parsed = parseURL(s);
-    }
-
-    TEST(parseURL, parsesAnythingInUriFormatWithoutDoubleSlash) {
-        auto s = "whatever:github.com/NixOS/nixpkgs.git";
-        auto parsed = parseURL(s);
-    }
-
-    TEST(parseURL, emptyStringIsInvalidURL) {
-        ASSERT_THROW(parseURL(""), Error);
-    }
-
-    /* ----------------------------------------------------------------------------
-     * decodeQuery
-     * --------------------------------------------------------------------------*/
-
-    TEST(decodeQuery, emptyStringYieldsEmptyMap) {
-        auto d = decodeQuery("");
-        ASSERT_EQ(d, (StringMap) { });
-    }
-
-    TEST(decodeQuery, simpleDecode) {
-        auto d = decodeQuery("yi=one&er=two");
-        ASSERT_EQ(d, ((StringMap) { { "yi", "one" }, { "er", "two" } }));
-    }
-
-    TEST(decodeQuery, decodeUrlEncodedArgs) {
-        auto d = decodeQuery("arg=%3D%3D%40%3D%3D");
-        ASSERT_EQ(d, ((StringMap) { { "arg", "==@==" } }));
-    }
-
-    TEST(decodeQuery, decodeArgWithEmptyValue) {
-        auto d = decodeQuery("arg=");
-        ASSERT_EQ(d, ((StringMap) { { "arg", ""} }));
-    }
-
-    /* ----------------------------------------------------------------------------
-     * percentDecode
-     * --------------------------------------------------------------------------*/
-
-    TEST(percentDecode, decodesUrlEncodedString) {
-        std::string s = "==@==";
-        std::string d = percentDecode("%3D%3D%40%3D%3D");
-        ASSERT_EQ(d, s);
-    }
-
-    TEST(percentDecode, multipleDecodesAreIdempotent) {
-        std::string once = percentDecode("%3D%3D%40%3D%3D");
-        std::string twice = percentDecode(once);
-
-        ASSERT_EQ(once, twice);
-    }
-
-    TEST(percentDecode, trailingPercent) {
-        std::string s = "==@==%";
-        std::string d = percentDecode("%3D%3D%40%3D%3D%25");
-
-        ASSERT_EQ(d, s);
-    }
-
-
-    /* ----------------------------------------------------------------------------
-     * percentEncode
-     * --------------------------------------------------------------------------*/
-
-    TEST(percentEncode, encodesUrlEncodedString) {
-        std::string s = percentEncode("==@==");
-        std::string d = "%3D%3D%40%3D%3D";
-        ASSERT_EQ(d, s);
-    }
-
-    TEST(percentEncode, keepArgument) {
-        std::string a = percentEncode("abd / def");
-        std::string b = percentEncode("abd / def", "/");
-        ASSERT_EQ(a, "abd%20%2F%20def");
-        ASSERT_EQ(b, "abd%20/%20def");
-    }
-
-    TEST(percentEncode, inverseOfDecode) {
-        std::string original = "%3D%3D%40%3D%3D";
-        std::string once = percentEncode(original);
-        std::string back = percentDecode(once);
-
-        ASSERT_EQ(back, original);
-    }
-
-    TEST(percentEncode, trailingPercent) {
-        std::string s = percentEncode("==@==%");
-        std::string d = "%3D%3D%40%3D%3D%25";
-
-        ASSERT_EQ(d, s);
-    }
-
-    TEST(percentEncode, yen) {
-        // https://en.wikipedia.org/wiki/Percent-encoding#Character_data
-        std::string s = reinterpret_cast<const char*>(u8"円");
-        std::string e = "%E5%86%86";
-
-        ASSERT_EQ(percentEncode(s), e);
-        ASSERT_EQ(percentDecode(e), s);
-    }
-
-}

src/libutil/tests/xml-writer.cc

@@ -1,105 +0,0 @@
-#include "xml-writer.hh"
-#include <gtest/gtest.h>
-#include <sstream>
-
-namespace nix {
-
-    /* ----------------------------------------------------------------------------
-     * XMLWriter
-     * --------------------------------------------------------------------------*/
-
-    TEST(XMLWriter, emptyObject) {
-        std::stringstream out;
-        {
-            XMLWriter t(false, out);
-        }
-
-        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n");
-    }
-
-    TEST(XMLWriter, objectWithEmptyElement) {
-        std::stringstream out;
-        {
-            XMLWriter t(false, out);
-            t.openElement("foobar");
-        }
-
-        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar></foobar>");
-    }
-
-    TEST(XMLWriter, objectWithElementWithAttrs) {
-        std::stringstream out;
-        {
-            XMLWriter t(false, out);
-            XMLAttrs attrs = {
-                { "foo", "bar" }
-            };
-            t.openElement("foobar", attrs);
-        }
-
-        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\"></foobar>");
-    }
-
-    TEST(XMLWriter, objectWithElementWithEmptyAttrs) {
-        std::stringstream out;
-        {
-            XMLWriter t(false, out);
-            XMLAttrs attrs = {};
-            t.openElement("foobar", attrs);
-        }
-
-        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar></foobar>");
-    }
-
-    TEST(XMLWriter, objectWithElementWithAttrsEscaping) {
-        std::stringstream out;
-        {
-            XMLWriter t(false, out);
-            XMLAttrs attrs = {
-                { "<key>", "<value>" }
-            };
-            t.openElement("foobar", attrs);
-        }
-
-        // XXX: While "<value>" is escaped, "<key>" isn't which I think is a bug.
-        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar <key>=\"&lt;value&gt;\"></foobar>");
-    }
-
-    TEST(XMLWriter, objectWithElementWithAttrsIndented) {
-        std::stringstream out;
-        {
-            XMLWriter t(true, out);
-            XMLAttrs attrs = {
-                { "foo", "bar" }
-            };
-            t.openElement("foobar", attrs);
-        }
-
-        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\">\n</foobar>\n");
-    }
-
-    TEST(XMLWriter, writeEmptyElement) {
-        std::stringstream out;
-        {
-            XMLWriter t(false, out);
-            t.writeEmptyElement("foobar");
-        }
-
-        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar />");
-    }
-
-    TEST(XMLWriter, writeEmptyElementWithAttributes) {
-        std::stringstream out;
-        {
-            XMLWriter t(false, out);
-            XMLAttrs attrs = {
-                { "foo", "bar" }
-            };
-            t.writeEmptyElement("foobar", attrs);
-
-        }
-
-        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\" />");
-    }
-
-}

src/libutil/thread-pool.cc

@@ -1,4 +1,6 @@
 #include "thread-pool.hh"
+#include "signals.hh"
+#include "util.hh"
 
 namespace nix {
 

src/libutil/thread-pool.hh

@@ -1,8 +1,8 @@
 #pragma once
 ///@file
 
+#include "error.hh"
 #include "sync.hh"
-#include "util.hh"
 
 #include <queue>
 #include <functional>

src/libutil/unix-domain-socket.cc

@@ -0,0 +1,100 @@
+#include "file-system.hh"
+#include "processes.hh"
+#include "unix-domain-socket.hh"
+
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+namespace nix {
+
+AutoCloseFD createUnixDomainSocket()
+{
+    AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM
+        #ifdef SOCK_CLOEXEC
+        | SOCK_CLOEXEC
+        #endif
+        , 0);
+    if (!fdSocket)
+        throw SysError("cannot create Unix domain socket");
+    closeOnExec(fdSocket.get());
+    return fdSocket;
+}
+
+
+AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
+{
+    auto fdSocket = nix::createUnixDomainSocket();
+
+    bind(fdSocket.get(), path);
+
+    if (chmod(path.c_str(), mode) == -1)
+        throw SysError("changing permissions on '%1%'", path);
+
+    if (listen(fdSocket.get(), 100) == -1)
+        throw SysError("cannot listen on socket '%1%'", path);
+
+    return fdSocket;
+}
+
+
+void bind(int fd, const std::string & path)
+{
+    unlink(path.c_str());
+
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+
+    if (path.size() + 1 >= sizeof(addr.sun_path)) {
+        Pid pid = startProcess([&]() {
+            Path dir = dirOf(path);
+            if (chdir(dir.c_str()) == -1)
+                throw SysError("chdir to '%s' failed", dir);
+            std::string base(baseNameOf(path));
+            if (base.size() + 1 >= sizeof(addr.sun_path))
+                throw Error("socket path '%s' is too long", base);
+            memcpy(addr.sun_path, base.c_str(), base.size() + 1);
+            if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+                throw SysError("cannot bind to socket '%s'", path);
+            _exit(0);
+        });
+        int status = pid.wait();
+        if (status != 0)
+            throw Error("cannot bind to socket '%s'", path);
+    } else {
+        memcpy(addr.sun_path, path.c_str(), path.size() + 1);
+        if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+            throw SysError("cannot bind to socket '%s'", path);
+    }
+}
+
+
+void connect(int fd, const std::string & path)
+{
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+
+    if (path.size() + 1 >= sizeof(addr.sun_path)) {
+        Pid pid = startProcess([&]() {
+            Path dir = dirOf(path);
+            if (chdir(dir.c_str()) == -1)
+                throw SysError("chdir to '%s' failed", dir);
+            std::string base(baseNameOf(path));
+            if (base.size() + 1 >= sizeof(addr.sun_path))
+                throw Error("socket path '%s' is too long", base);
+            memcpy(addr.sun_path, base.c_str(), base.size() + 1);
+            if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+                throw SysError("cannot connect to socket at '%s'", path);
+            _exit(0);
+        });
+        int status = pid.wait();
+        if (status != 0)
+            throw Error("cannot connect to socket at '%s'", path);
+    } else {
+        memcpy(addr.sun_path, path.c_str(), path.size() + 1);
+        if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+            throw SysError("cannot connect to socket at '%s'", path);
+    }
+}
+
+}

src/libutil/unix-domain-socket.hh

@@ -0,0 +1,31 @@
+#pragma once
+///@file
+
+#include "types.hh"
+#include "file-descriptor.hh"
+
+#include <unistd.h>
+
+namespace nix {
+
+/**
+ * Create a Unix domain socket.
+ */
+AutoCloseFD createUnixDomainSocket();
+
+/**
+ * Create a Unix domain socket in listen mode.
+ */
+AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
+
+/**
+ * Bind a Unix domain socket to a path.
+ */
+void bind(int fd, const std::string & path);
+
+/**
+ * Connect to a Unix domain socket.
+ */
+void connect(int fd, const std::string & path);
+
+}

src/libutil/url.cc

@@ -2,6 +2,7 @@
 #include "url-parts.hh"
 #include "util.hh"
 #include "split.hh"
+#include "canon-path.hh"
 
 namespace nix {
 
@@ -141,6 +142,13 @@ bool ParsedURL::operator ==(const ParsedURL & other) const
         && fragment == other.fragment;
 }
 
+ParsedURL ParsedURL::canonicalise()
+{
+    ParsedURL res(*this);
+    res.path = CanonPath(res.path).abs();
+    return res;
+}
+
 /**
  * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme'
  * into a tuple '(applicationScheme, transportScheme)'

src/libutil/url.hh

@@ -19,6 +19,11 @@ struct ParsedURL
     std::string to_string() const;
 
     bool operator ==(const ParsedURL & other) const;
+
+    /**
+     * Remove `.` and `..` path elements.
+     */
+    ParsedURL canonicalise();
 };
 
 MakeError(BadURL, Error);

src/libutil/users.cc

@@ -0,0 +1,116 @@
+#include "util.hh"
+#include "users.hh"
+#include "environment-variables.hh"
+#include "file-system.hh"
+
+#include <pwd.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+namespace nix {
+
+std::string getUserName()
+{
+    auto pw = getpwuid(geteuid());
+    std::string name = pw ? pw->pw_name : getEnv("USER").value_or("");
+    if (name.empty())
+        throw Error("cannot figure out user name");
+    return name;
+}
+
+Path getHomeOf(uid_t userId)
+{
+    std::vector<char> buf(16384);
+    struct passwd pwbuf;
+    struct passwd * pw;
+    if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0
+        || !pw || !pw->pw_dir || !pw->pw_dir[0])
+        throw Error("cannot determine user's home directory");
+    return pw->pw_dir;
+}
+
+Path getHome()
+{
+    static Path homeDir = []()
+    {
+        std::optional<std::string> unownedUserHomeDir = {};
+        auto homeDir = getEnv("HOME");
+        if (homeDir) {
+            // Only use $HOME if doesn't exist or is owned by the current user.
+            struct stat st;
+            int result = stat(homeDir->c_str(), &st);
+            if (result != 0) {
+                if (errno != ENOENT) {
+                    warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
+                    homeDir.reset();
+                }
+            } else if (st.st_uid != geteuid()) {
+                unownedUserHomeDir.swap(homeDir);
+            }
+        }
+        if (!homeDir) {
+            homeDir = getHomeOf(geteuid());
+            if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
+                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
+            }
+        }
+        return *homeDir;
+    }();
+    return homeDir;
+}
+
+
+Path getCacheDir()
+{
+    auto cacheDir = getEnv("XDG_CACHE_HOME");
+    return cacheDir ? *cacheDir : getHome() + "/.cache";
+}
+
+
+Path getConfigDir()
+{
+    auto configDir = getEnv("XDG_CONFIG_HOME");
+    return configDir ? *configDir : getHome() + "/.config";
+}
+
+std::vector<Path> getConfigDirs()
+{
+    Path configHome = getConfigDir();
+    auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg");
+    std::vector<Path> result = tokenizeString<std::vector<std::string>>(configDirs, ":");
+    result.insert(result.begin(), configHome);
+    return result;
+}
+
+
+Path getDataDir()
+{
+    auto dataDir = getEnv("XDG_DATA_HOME");
+    return dataDir ? *dataDir : getHome() + "/.local/share";
+}
+
+Path getStateDir()
+{
+    auto stateDir = getEnv("XDG_STATE_HOME");
+    return stateDir ? *stateDir : getHome() + "/.local/state";
+}
+
+Path createNixStateDir()
+{
+    Path dir = getStateDir() + "/nix";
+    createDirs(dir);
+    return dir;
+}
+
+
+std::string expandTilde(std::string_view path)
+{
+    // TODO: expand ~user ?
+    auto tilde = path.substr(0, 2);
+    if (tilde == "~/" || tilde == "~")
+        return getHome() + std::string(path.substr(1));
+    else
+        return std::string(path);
+}
+
+}

src/libutil/users.hh

@@ -0,0 +1,58 @@
+#pragma once
+///@file
+
+#include "types.hh"
+
+#include <sys/types.h>
+
+namespace nix {
+
+std::string getUserName();
+
+/**
+ * @return the given user's home directory from /etc/passwd.
+ */
+Path getHomeOf(uid_t userId);
+
+/**
+ * @return $HOME or the user's home directory from /etc/passwd.
+ */
+Path getHome();
+
+/**
+ * @return $XDG_CACHE_HOME or $HOME/.cache.
+ */
+Path getCacheDir();
+
+/**
+ * @return $XDG_CONFIG_HOME or $HOME/.config.
+ */
+Path getConfigDir();
+
+/**
+ * @return the directories to search for user configuration files
+ */
+std::vector<Path> getConfigDirs();
+
+/**
+ * @return $XDG_DATA_HOME or $HOME/.local/share.
+ */
+Path getDataDir();
+
+/**
+ * @return $XDG_STATE_HOME or $HOME/.local/state.
+ */
+Path getStateDir();
+
+/**
+ * Create the Nix state directory and return the path to it.
+ */
+Path createNixStateDir();
+
+/**
+ * Perform tilde expansion on a path, replacing tilde with the user's
+ * home directory.
+ */
+std::string expandTilde(std::string_view path);
+
+}

src/libutil/util.hh

@@ -4,491 +4,18 @@
 #include "types.hh"
 #include "error.hh"
 #include "logging.hh"
-#include "ansicolor.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <signal.h>
 
 #include <boost/lexical_cast.hpp>
 
-#include <atomic>
 #include <functional>
 #include <map>
 #include <sstream>
 #include <optional>
 
-#ifndef HAVE_STRUCT_DIRENT_D_TYPE
-#define DT_UNKNOWN 0
-#define DT_REG 1
-#define DT_LNK 2
-#define DT_DIR 3
-#endif
-
 namespace nix {
 
-struct Sink;
-struct Source;
-
 void initLibUtil();
 
-/**
- * The system for which Nix is compiled.
- */
-extern const std::string nativeSystem;
-
-
-/**
- * @return an environment variable.
- */
-std::optional<std::string> getEnv(const std::string & key);
-
-/**
- * @return a non empty environment variable. Returns nullopt if the env
- * variable is set to ""
- */
-std::optional<std::string> getEnvNonEmpty(const std::string & key);
-
-/**
- * Get the entire environment.
- */
-std::map<std::string, std::string> getEnv();
-
-/**
- * Clear the environment.
- */
-void clearEnv();
-
-/**
- * @return An absolutized path, resolving paths relative to the
- * specified directory, or the current directory otherwise.  The path
- * is also canonicalised.
- */
-Path absPath(Path path,
-    std::optional<PathView> dir = {},
-    bool resolveSymlinks = false);
-
-/**
- * Canonicalise a path by removing all `.` or `..` components and
- * double or trailing slashes.  Optionally resolves all symlink
- * components such that each component of the resulting path is *not*
- * a symbolic link.
- */
-Path canonPath(PathView path, bool resolveSymlinks = false);
-
-/**
- * @return The directory part of the given canonical path, i.e.,
- * everything before the final `/`.  If the path is the root or an
- * immediate child thereof (e.g., `/foo`), this means `/`
- * is returned.
- */
-Path dirOf(const PathView path);
-
-/**
- * @return the base name of the given canonical path, i.e., everything
- * following the final `/` (trailing slashes are removed).
- */
-std::string_view baseNameOf(std::string_view path);
-
-/**
- * Perform tilde expansion on a path.
- */
-std::string expandTilde(std::string_view path);
-
-/**
- * Check whether 'path' is a descendant of 'dir'. Both paths must be
- * canonicalized.
- */
-bool isInDir(std::string_view path, std::string_view dir);
-
-/**
- * Check whether 'path' is equal to 'dir' or a descendant of
- * 'dir'. Both paths must be canonicalized.
- */
-bool isDirOrInDir(std::string_view path, std::string_view dir);
-
-/**
- * Get status of `path`.
- */
-struct stat stat(const Path & path);
-struct stat lstat(const Path & path);
-
-/**
- * @return true iff the given path exists.
- */
-bool pathExists(const Path & path);
-
-/**
- * A version of pathExists that returns false on a permission error.
- * Useful for inferring default paths across directories that might not
- * be readable.
- * @return true iff the given path can be accessed and exists
- */
-bool pathAccessible(const Path & path);
-
-/**
- * Read the contents (target) of a symbolic link.  The result is not
- * in any way canonicalised.
- */
-Path readLink(const Path & path);
-
-bool isLink(const Path & path);
-
-/**
- * Read the contents of a directory.  The entries `.` and `..` are
- * removed.
- */
-struct DirEntry
-{
-    std::string name;
-    ino_t ino;
-    /**
-     * one of DT_*
-     */
-    unsigned char type;
-    DirEntry(std::string name, ino_t ino, unsigned char type)
-        : name(std::move(name)), ino(ino), type(type) { }
-};
-
-typedef std::vector<DirEntry> DirEntries;
-
-DirEntries readDirectory(const Path & path);
-
-unsigned char getFileType(const Path & path);
-
-/**
- * Read the contents of a file into a string.
- */
-std::string readFile(int fd);
-std::string readFile(const Path & path);
-void readFile(const Path & path, Sink & sink);
-
-/**
- * Write a string to a file.
- */
-void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false);
-
-void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false);
-
-/**
- * Flush a file's parent directory to disk
- */
-void syncParent(const Path & path);
-
-/**
- * Read a line from a file descriptor.
- */
-std::string readLine(int fd);
-
-/**
- * Write a line to a file descriptor.
- */
-void writeLine(int fd, std::string s);
-
-/**
- * Delete a path; i.e., in the case of a directory, it is deleted
- * recursively. It's not an error if the path does not exist. The
- * second variant returns the number of bytes and blocks freed.
- */
-void deletePath(const Path & path);
-
-void deletePath(const Path & path, uint64_t & bytesFreed);
-
-std::string getUserName();
-
-/**
- * @return the given user's home directory from /etc/passwd.
- */
-Path getHomeOf(uid_t userId);
-
-/**
- * @return $HOME or the user's home directory from /etc/passwd.
- */
-Path getHome();
-
-/**
- * @return $XDG_CACHE_HOME or $HOME/.cache.
- */
-Path getCacheDir();
-
-/**
- * @return $XDG_CONFIG_HOME or $HOME/.config.
- */
-Path getConfigDir();
-
-/**
- * @return the directories to search for user configuration files
- */
-std::vector<Path> getConfigDirs();
-
-/**
- * @return $XDG_DATA_HOME or $HOME/.local/share.
- */
-Path getDataDir();
-
-/**
- * @return the path of the current executable.
- */
-std::optional<Path> getSelfExe();
-
-/**
- * @return $XDG_STATE_HOME or $HOME/.local/state.
- */
-Path getStateDir();
-
-/**
- * Create the Nix state directory and return the path to it.
- */
-Path createNixStateDir();
-
-/**
- * Create a directory and all its parents, if necessary.  Returns the
- * list of created directories, in order of creation.
- */
-Paths createDirs(const Path & path);
-inline Paths createDirs(PathView path)
-{
-    return createDirs(Path(path));
-}
-
-/**
- * Create a symlink.
- */
-void createSymlink(const Path & target, const Path & link);
-
-/**
- * Atomically create or replace a symlink.
- */
-void replaceSymlink(const Path & target, const Path & link);
-
-void renameFile(const Path & src, const Path & dst);
-
-/**
- * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
- * are on a different filesystem.
- *
- * Beware that this might not be atomic because of the copy that happens behind
- * the scenes
- */
-void moveFile(const Path & src, const Path & dst);
-
-
-/**
- * Wrappers arount read()/write() that read/write exactly the
- * requested number of bytes.
- */
-void readFull(int fd, char * buf, size_t count);
-void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
-
-MakeError(EndOfFile, Error);
-
-
-/**
- * Read a file descriptor until EOF occurs.
- */
-std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
-
-void drainFD(int fd, Sink & sink, bool block = true);
-
-/**
- * If cgroups are active, attempt to calculate the number of CPUs available.
- * If cgroups are unavailable or if cpu.max is set to "max", return 0.
- */
-unsigned int getMaxCPU();
-
-/**
- * Automatic cleanup of resources.
- */
-
-
-class AutoDelete
-{
-    Path path;
-    bool del;
-    bool recursive;
-public:
-    AutoDelete();
-    AutoDelete(const Path & p, bool recursive = true);
-    ~AutoDelete();
-    void cancel();
-    void reset(const Path & p, bool recursive = true);
-    operator Path() const { return path; }
-    operator PathView() const { return path; }
-};
-
-
-class AutoCloseFD
-{
-    int fd;
-public:
-    AutoCloseFD();
-    AutoCloseFD(int fd);
-    AutoCloseFD(const AutoCloseFD & fd) = delete;
-    AutoCloseFD(AutoCloseFD&& fd);
-    ~AutoCloseFD();
-    AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
-    AutoCloseFD& operator =(AutoCloseFD&& fd);
-    int get() const;
-    explicit operator bool() const;
-    int release();
-    void close();
-    void fsync();
-};
-
-
-/**
- * Create a temporary directory.
- */
-Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
-    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
-
-/**
- * Create a temporary file, returning a file handle and its path.
- */
-std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
-
-
-class Pipe
-{
-public:
-    AutoCloseFD readSide, writeSide;
-    void create();
-    void close();
-};
-
-
-struct DIRDeleter
-{
-    void operator()(DIR * dir) const {
-        closedir(dir);
-    }
-};
-
-typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir;
-
-
-class Pid
-{
-    pid_t pid = -1;
-    bool separatePG = false;
-    int killSignal = SIGKILL;
-public:
-    Pid();
-    Pid(pid_t pid);
-    ~Pid();
-    void operator =(pid_t pid);
-    operator pid_t();
-    int kill();
-    int wait();
-
-    void setSeparatePG(bool separatePG);
-    void setKillSignal(int signal);
-    pid_t release();
-};
-
-
-/**
- * Kill all processes running under the specified uid by sending them
- * a SIGKILL.
- */
-void killUser(uid_t uid);
-
-
-/**
- * Fork a process that runs the given function, and return the child
- * pid to the caller.
- */
-struct ProcessOptions
-{
-    std::string errorPrefix = "";
-    bool dieWithParent = true;
-    bool runExitHandlers = false;
-    bool allowVfork = false;
-    /**
-     * use clone() with the specified flags (Linux only)
-     */
-    int cloneFlags = 0;
-};
-
-pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
-
-
-/**
- * Run a program and return its stdout in a string (i.e., like the
- * shell backtick operator).
- */
-std::string runProgram(Path program, bool searchPath = false,
-    const Strings & args = Strings(),
-    const std::optional<std::string> & input = {}, bool isInteractive = false);
-
-struct RunOptions
-{
-    Path program;
-    bool searchPath = true;
-    Strings args;
-    std::optional<uid_t> uid;
-    std::optional<uid_t> gid;
-    std::optional<Path> chdir;
-    std::optional<std::map<std::string, std::string>> environment;
-    std::optional<std::string> input;
-    Source * standardIn = nullptr;
-    Sink * standardOut = nullptr;
-    bool mergeStderrToStdout = false;
-    bool isInteractive = false;
-};
-
-std::pair<int, std::string> runProgram(RunOptions && options);
-
-void runProgram2(const RunOptions & options);
-
-
-/**
- * Change the stack size.
- */
-void setStackSize(size_t stackSize);
-
-
-/**
- * Restore the original inherited Unix process context (such as signal
- * masks, stack size).
-
- * See startSignalHandlerThread(), saveSignalMask().
- */
-void restoreProcessContext(bool restoreMounts = true);
-
-/**
- * Save the current mount namespace. Ignored if called more than
- * once.
- */
-void saveMountNamespace();
-
-/**
- * Restore the mount namespace saved by saveMountNamespace(). Ignored
- * if saveMountNamespace() was never called.
- */
-void restoreMountNamespace();
-
-/**
- * Cause this thread to not share any FS attributes with the main
- * thread, because this causes setns() in restoreMountNamespace() to
- * fail.
- */
-void unshareFilesystem();
-
-
-class ExecError : public Error
-{
-public:
-    int status;
-
-    template<typename... Args>
-    ExecError(int status, const Args & ... args)
-        : Error(args...), status(status)
-    { }
-};
-
 /**
  * Convert a list of strings to a null-terminated vector of `char
  * *`s. The result must not be accessed beyond the lifetime of the
@@ -496,36 +23,6 @@ public:
  */
 std::vector<char *> stringsToCharPtrs(const Strings & ss);
 
-/**
- * Close all file descriptors except those listed in the given set.
- * Good practice in child processes.
- */
-void closeMostFDs(const std::set<int> & exceptions);
-
-/**
- * Set the close-on-exec flag for the given file descriptor.
- */
-void closeOnExec(int fd);
-
-
-/* User interruption. */
-
-extern std::atomic<bool> _isInterrupted;
-
-extern thread_local std::function<bool()> interruptCheck;
-
-void setInterruptThrown();
-
-void _interrupted();
-
-void inline checkInterrupt()
-{
-    if (_isInterrupted || (interruptCheck && interruptCheck()))
-        _interrupted();
-}
-
-MakeError(Interrupted, BaseError);
-
 
 MakeError(FormatError, Error);
 
@@ -601,15 +98,6 @@ std::string replaceStrings(
 std::string rewriteStrings(std::string s, const StringMap & rewrites);
 
 
-/**
- * Convert the exit status of a child as returned by wait() into an
- * error string.
- */
-std::string statusToString(int status);
-
-bool statusOk(int status);
-
-
 /**
  * Parse a string into an integer.
  */
@@ -701,10 +189,8 @@ std::string toLower(const std::string & s);
 std::string shellEscape(const std::string_view s);
 
 
-/**
- * Exception handling in destructors: print an error message, then
- * ignore the exception.
- */
+/* Exception handling in destructors: print an error message, then
+   ignore the exception. */
 void ignoreException(Verbosity lvl = lvlError);
 
 
@@ -717,23 +203,6 @@ constexpr char treeLast[] = "└───";
 constexpr char treeLine[] = "│   ";
 constexpr char treeNull[] = "    ";
 
-/**
- * Determine whether ANSI escape sequences are appropriate for the
- * present output.
- */
-bool shouldANSI();
-
-/**
- * Truncate a string to 'width' printable characters. If 'filterAll'
- * is true, all ANSI escape sequences are filtered out. Otherwise,
- * some escape sequences (such as colour setting) are copied but not
- * included in the character count. Also, tabs are expanded to
- * spaces.
- */
-std::string filterANSIEscapes(std::string_view s,
-    bool filterAll = false,
-    unsigned int width = std::numeric_limits<unsigned int>::max());
-
 
 /**
  * Base64 encoding/decoding.
@@ -821,61 +290,6 @@ template<typename T>
 class Callback;
 
 
-/**
- * Start a thread that handles various signals. Also block those signals
- * on the current thread (and thus any threads created by it).
- * Saves the signal mask before changing the mask to block those signals.
- * See saveSignalMask().
- */
-void startSignalHandlerThread();
-
-/**
- * Saves the signal mask, which is the signal mask that nix will restore
- * before creating child processes.
- * See setChildSignalMask() to set an arbitrary signal mask instead of the
- * current mask.
- */
-void saveSignalMask();
-
-/**
- * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
- * necessarily match the current thread's mask.
- * See saveSignalMask() to set the saved mask to the current mask.
- */
-void setChildSignalMask(sigset_t *sigs);
-
-struct InterruptCallback
-{
-    virtual ~InterruptCallback() { };
-};
-
-/**
- * Register a function that gets called on SIGINT (in a non-signal
- * context).
- */
-std::unique_ptr<InterruptCallback> createInterruptCallback(
-    std::function<void()> callback);
-
-void triggerInterrupt();
-
-/**
- * A RAII class that causes the current thread to receive SIGUSR1 when
- * the signal handler thread receives SIGINT. That is, this allows
- * SIGINT to be multiplexed to multiple threads.
- */
-struct ReceiveInterrupts
-{
-    pthread_t target;
-    std::unique_ptr<InterruptCallback> callback;
-
-    ReceiveInterrupts()
-        : target(pthread_self())
-        , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); }))
-    { }
-};
-
-
-
 /**
  * A RAII helper that increments a counter on construction and
  * decrements it on destruction.
@@ -890,45 +304,6 @@ struct MaintainCount
 };
 
 
-/**
- * @return the number of rows and columns of the terminal.
- */
-std::pair<unsigned short, unsigned short> getWindowSize();
-
-
-/**
- * Used in various places.
- */
-typedef std::function<bool(const Path & path)> PathFilter;
-
-extern PathFilter defaultPathFilter;
-
-/**
- * Common initialisation performed in child processes.
- */
-void commonChildInit();
-
-/**
- * Create a Unix domain socket.
- */
-AutoCloseFD createUnixDomainSocket();
-
-/**
- * Create a Unix domain socket in listen mode.
- */
-AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
-
-/**
- * Bind a Unix domain socket to a path.
- */
-void bind(int fd, const std::string & path);
-
-/**
- * Connect to a Unix domain socket.
- */
-void connect(int fd, const std::string & path);
-
-
 /**
  * A Rust/Python-like enumerate() iterator adapter.
  *