Merge remote-tracking branch 'origin/master' into flip-coroutines

src/libutil/checked-arithmetic.hh

@@ -0,0 +1,182 @@
+#pragma once
+/**
+ * @file Checked arithmetic with classes that make it hard to accidentally make something an unchecked operation.
+ */
+
+#include <compare>
+#include <concepts> // IWYU pragma: keep
+#include <exception>
+#include <ostream>
+#include <limits>
+#include <optional>
+#include <type_traits>
+
+namespace nix::checked {
+
+class DivideByZero : std::exception
+{};
+
+/**
+ * Numeric value enforcing checked arithmetic. Performing mathematical operations on such values will return a Result
+ * type which needs to be checked.
+ */
+template<std::integral T>
+struct Checked
+{
+    using Inner = T;
+
+    // TODO: this must be a "trivial default constructor", which means it
+    // cannot set the value to NOT DO UB on uninit.
+    T value;
+
+    Checked() = default;
+    explicit Checked(T const value)
+        : value{value}
+    {
+    }
+    Checked(Checked<T> const & other) = default;
+    Checked(Checked<T> && other) = default;
+    Checked<T> & operator=(Checked<T> const & other) = default;
+
+    std::strong_ordering operator<=>(Checked<T> const & other) const = default;
+    std::strong_ordering operator<=>(T const & other) const
+    {
+        return value <=> other;
+    }
+
+    explicit operator T() const
+    {
+        return value;
+    }
+
+    enum class OverflowKind {
+        NoOverflow,
+        Overflow,
+        DivByZero,
+    };
+
+    class Result
+    {
+        T value;
+        OverflowKind overflowed_;
+
+    public:
+        Result(T value, bool overflowed)
+            : value{value}
+            , overflowed_{overflowed ? OverflowKind::Overflow : OverflowKind::NoOverflow}
+        {
+        }
+        Result(T value, OverflowKind overflowed)
+            : value{value}
+            , overflowed_{overflowed}
+        {
+        }
+
+        bool operator==(Result other) const
+        {
+            return value == other.value && overflowed_ == other.overflowed_;
+        }
+
+        std::optional<T> valueChecked() const
+        {
+            if (overflowed_ != OverflowKind::NoOverflow) {
+                return std::nullopt;
+            } else {
+                return value;
+            }
+        }
+
+        /**
+         * Returns the result as if the arithmetic were performed as wrapping arithmetic.
+         *
+         * \throws DivideByZero if the operation was a divide by zero.
+         */
+        T valueWrapping() const
+        {
+            if (overflowed_ == OverflowKind::DivByZero) {
+                throw DivideByZero{};
+            }
+            return value;
+        }
+
+        bool overflowed() const
+        {
+            return overflowed_ == OverflowKind::Overflow;
+        }
+
+        bool divideByZero() const
+        {
+            return overflowed_ == OverflowKind::DivByZero;
+        }
+    };
+
+    Result operator+(Checked<T> const other) const
+    {
+        return (*this) + other.value;
+    }
+    Result operator+(T const other) const
+    {
+        T result;
+        bool overflowed = __builtin_add_overflow(value, other, &result);
+        return Result{result, overflowed};
+    }
+
+    Result operator-(Checked<T> const other) const
+    {
+        return (*this) - other.value;
+    }
+    Result operator-(T const other) const
+    {
+        T result;
+        bool overflowed = __builtin_sub_overflow(value, other, &result);
+        return Result{result, overflowed};
+    }
+
+    Result operator*(Checked<T> const other) const
+    {
+        return (*this) * other.value;
+    }
+    Result operator*(T const other) const
+    {
+        T result;
+        bool overflowed = __builtin_mul_overflow(value, other, &result);
+        return Result{result, overflowed};
+    }
+
+    Result operator/(Checked<T> const other) const
+    {
+        return (*this) / other.value;
+    }
+    /**
+     * Performs a checked division.
+     *
+     * If the right hand side is zero, the result is marked as a DivByZero and
+     * valueWrapping will throw.
+     */
+    Result operator/(T const other) const
+    {
+        constexpr T const minV = std::numeric_limits<T>::min();
+
+        // It's only possible to overflow with signed division since doing so
+        // requires crossing the two's complement limits by MIN / -1 (since
+        // two's complement has one more in range in the negative direction
+        // than in the positive one).
+        if (std::is_signed<T>() && (value == minV && other == -1)) {
+            return Result{minV, true};
+        } else if (other == 0) {
+            return Result{0, OverflowKind::DivByZero};
+        } else {
+            T result = value / other;
+            return Result{result, false};
+        }
+    }
+};
+
+template<std::integral T>
+std::ostream & operator<<(std::ostream & ios, Checked<T> v)
+{
+    ios << v.value;
+    return ios;
+}
+
+}

src/libutil/chunked-vector.hh

@@ -6,6 +6,8 @@
 #include <vector>
 #include <limits>
 
+#include "error.hh"
+
 namespace nix {
 
 /**
@@ -30,7 +32,7 @@ private:
     auto & addChunk()
     {
         if (size_ >= std::numeric_limits<uint32_t>::max() - ChunkSize)
-            abort();
+            unreachable();
         chunks.emplace_back();
         chunks.back().reserve(ChunkSize);
         return chunks.back();

src/libutil/config.cc

@@ -1,6 +1,7 @@
 #include "config.hh"
 #include "args.hh"
 #include "abstract-setting-to-json.hh"
+#include "environment-variables.hh"
 #include "experimental-features.hh"
 #include "util.hh"
 #include "file-system.hh"
@@ -170,9 +171,18 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             set(name, value);
 
     // Then apply other settings
-    for (const auto & [name, value] : parsedContents)
-        if (name != "experimental-features" && name != "extra-experimental-features")
+    // XXX: NIX_PATH must override the regular setting! This is done in `initGC()`
+    // Environment variables overriding settings should probably be part of the Config mechanism,
+    // but at the time of writing it's not worth building that for just one thing
+    for (const auto & [name, value] : parsedContents) {
+        if (name != "experimental-features" && name != "extra-experimental-features") {
+            if ((name == "nix-path" || name == "extra-nix-path")
+                && getEnv("NIX_PATH").has_value()) {
+                continue;
+            }
             set(name, value);
+        }
+    }
 }
 
 void Config::resetOverridden()

src/libutil/config.hh

@@ -393,7 +393,7 @@ struct ExperimentalFeatureSettings : Config {
 
           {{#include experimental-features-shortlist.md}}
 
-          Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md).
+          Experimental features are [further documented in the manual](@docroot@/development/experimental-features.md).
         )"};
 
     /**

src/libutil/current-process.cc

@@ -15,13 +15,12 @@
 
 #if __linux__
 # include <mutex>
-# include <sys/resource.h>
 # include "cgroup.hh"
 # include "namespaces.hh"
 #endif
 
 #ifndef _WIN32
-# include <sys/mount.h>
+# include <sys/resource.h>
 #endif
 
 namespace nix {
@@ -138,7 +137,7 @@ std::optional<Path> getSelfExe()
 {
     static auto cached = []() -> std::optional<Path>
     {
-        #if __linux__
+        #if __linux__ || __GNU__
         return readLink("/proc/self/exe");
         #elif __APPLE__
         char buf[1024];

src/libutil/environment-variables.cc

@@ -1,20 +1,23 @@
 #include "util.hh"
 #include "environment-variables.hh"
 
-extern char * * environ __attribute__((weak));
+extern char ** environ __attribute__((weak));
 
 namespace nix {
 
 std::optional<std::string> getEnv(const std::string & key)
 {
     char * value = getenv(key.c_str());
-    if (!value) return {};
+    if (!value)
+        return {};
     return std::string(value);
 }
 
-std::optional<std::string> getEnvNonEmpty(const std::string & key) {
+std::optional<std::string> getEnvNonEmpty(const std::string & key)
+{
     auto value = getEnv(key);
-    if (value == "") return {};
+    if (value == "")
+        return {};
     return value;
 }
 

src/libutil/environment-variables.hh

@@ -9,6 +9,7 @@
 #include <optional>
 
 #include "types.hh"
+#include "file-path.hh"
 
 namespace nix {
 
@@ -17,6 +18,11 @@ namespace nix {
  */
 std::optional<std::string> getEnv(const std::string & key);
 
+/**
+ * Like `getEnv`, but using `OsString` to avoid coercions.
+ */
+std::optional<OsString> getEnvOs(const OsString & key);
+
 /**
  * @return a non empty environment variable. Returns nullopt if the env
  * variable is set to ""
@@ -43,6 +49,11 @@ int unsetenv(const char * name);
  */
 int setEnv(const char * name, const char * value);
 
+/**
+ * Like `setEnv`, but using `OsString` to avoid coercions.
+ */
+int setEnvOs(const OsString & name, const OsString & value);
+
 /**
  * Clear the environment.
  */

src/libutil/error.cc

@@ -1,3 +1,5 @@
+#include <algorithm>
+
 #include "error.hh"
 #include "environment-variables.hh"
 #include "signals.hh"
@@ -430,4 +432,36 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
     return out;
 }
 
+/** Write to stderr in a robust and minimal way, considering that the process
+ * may be in a bad state.
+ */
+static void writeErr(std::string_view buf)
+{
+    while (!buf.empty()) {
+        auto n = write(STDERR_FILENO, buf.data(), buf.size());
+        if (n < 0) {
+            if (errno == EINTR) continue;
+            abort();
+        }
+        buf = buf.substr(n);
+    }
+}
+
+void panic(std::string_view msg)
+{
+    writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL );
+    writeErr(msg);
+    writeErr("\n");
+    abort();
+}
+
+void panic(const char * file, int line, const char * func)
+{
+    char buf[512];
+    int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line);
+    if (n < 0)
+        panic("Unexpected condition and could not format error message");
+    panic(std::string_view(buf, std::min(static_cast<int>(sizeof(buf)), n)));
+}
+
 }

src/libutil/error.hh

@@ -273,4 +273,24 @@ using NativeSysError =
  */
 void throwExceptionSelfCheck();
 
+/**
+ * Print a message and abort().
+ */
+[[noreturn]]
+void panic(std::string_view msg);
+
+/**
+ * Print a basic error message with source position and abort().
+ * Use the unreachable() macro to call this.
+ */
+[[noreturn]]
+void panic(const char * file, int line, const char * func);
+
+/**
+ * Print a basic error message with source position and abort().
+ *
+ * @note: This assumes that the logger is operational
+ */
+#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__))
+
 }

src/libutil/executable-path.cc

@@ -0,0 +1,95 @@
+#include "environment-variables.hh"
+#include "executable-path.hh"
+#include "strings-inline.hh"
+#include "util.hh"
+#include "file-path-impl.hh"
+
+namespace nix {
+
+namespace fs = std::filesystem;
+
+constexpr static const OsStringView path_var_separator{
+    &ExecutablePath::separator,
+    1,
+};
+
+ExecutablePath ExecutablePath::load()
+{
+    // "If PATH is unset or is set to null, the path search is
+    // implementation-defined."
+    // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03
+    return ExecutablePath::parse(getEnvOs(OS_STR("PATH")).value_or(OS_STR("")));
+}
+
+ExecutablePath ExecutablePath::parse(const OsString & path)
+{
+    auto strings = path.empty() ? (std::list<OsString>{})
+                                : basicSplitString<std::list<OsString>, OsString::value_type>(path, path_var_separator);
+
+    std::vector<fs::path> ret;
+    ret.reserve(strings.size());
+
+    std::transform(
+        std::make_move_iterator(strings.begin()),
+        std::make_move_iterator(strings.end()),
+        std::back_inserter(ret),
+        [](auto && str) {
+            return fs::path{
+                str.empty()
+                    // "A zero-length prefix is a legacy feature that
+                    // indicates the current working directory. It
+                    // appears as two adjacent <colon> characters
+                    // ("::"), as an initial <colon> preceding the rest
+                    // of the list, or as a trailing <colon> following
+                    // the rest of the list."
+                    // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03
+                    ? OS_STR(".")
+                    : std::move(str),
+            };
+        });
+
+    return {ret};
+}
+
+OsString ExecutablePath::render() const
+{
+    std::vector<PathViewNG> path2;
+    for (auto & p : directories)
+        path2.push_back(p.native());
+    return basicConcatStringsSep(path_var_separator, path2);
+}
+
+std::optional<fs::path>
+ExecutablePath::findName(const OsString & exe, std::function<bool(const fs::path &)> isExecutable) const
+{
+    // "If the pathname being sought contains a <slash>, the search
+    // through the path prefixes shall not be performed."
+    // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03
+    assert(OsPathTrait<fs::path::value_type>::rfindPathSep(exe) == exe.npos);
+
+    for (auto & dir : directories) {
+        auto candidate = dir / exe;
+        if (isExecutable(candidate))
+            return std::filesystem::canonical(candidate);
+    }
+
+    return std::nullopt;
+}
+
+fs::path ExecutablePath::findPath(const fs::path & exe, std::function<bool(const fs::path &)> isExecutable) const
+{
+    // "If the pathname being sought contains a <slash>, the search
+    // through the path prefixes shall not be performed."
+    // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03
+    if (exe.filename() == exe) {
+        auto resOpt = findName(exe, isExecutable);
+        if (resOpt)
+            return *resOpt;
+        else
+            throw ExecutableLookupError("Could not find executable '%s'", exe.native());
+    } else {
+        return exe;
+    }
+}
+
+} // namespace nix

src/libutil/executable-path.hh

@@ -0,0 +1,77 @@
+#pragma once
+///@file
+
+#include "file-system.hh"
+
+namespace nix {
+
+MakeError(ExecutableLookupError, Error);
+
+struct ExecutablePath
+{
+    std::vector<std::filesystem::path> directories;
+
+    constexpr static const OsString::value_type separator =
+#ifdef WIN32
+        L';'
+#else
+        ':'
+#endif
+        ;
+
+    /**
+     * Parse `path` into a list of paths.
+     *
+     * On Unix we split on `:`, on Windows we split on `;`.
+     *
+     * For Unix, this is according to the POSIX spec for `PATH`.
+     * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03
+     */
+    static ExecutablePath parse(const OsString & path);
+
+    /**
+     * Load the `PATH` environment variable and `parse` it.
+     */
+    static ExecutablePath load();
+
+    /**
+     * Opposite of `parse`
+     */
+    OsString render() const;
+
+    /**
+     * Search for an executable.
+     *
+     * For Unix, this is according to the POSIX spec for `PATH`.
+     * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03
+     *
+     * @param exe This must just be a name, and not contain any `/` (or
+     * `\` on Windows). in case it does, per the spec no lookup should
+     * be perfomed, and the path (it is not just a file name) as is.
+     * This is the caller's respsonsibility.
+     *
+     * This is a pure function, except for the default `isExecutable`
+     * argument, which uses the ambient file system to check if a file is
+     * executable (and exists).
+     *
+     * @return path to a resolved executable
+     */
+    std::optional<std::filesystem::path> findName(
+        const OsString & exe,
+        std::function<bool(const std::filesystem::path &)> isExecutableFile = isExecutableFileAmbient) const;
+
+    /**
+     * Like the `findName` but also allows a file path as input.
+     *
+     * This implements the full POSIX spec: if the path is just a name,
+     * it searches like the above. Otherwise, it returns the path as is.
+     * If (in the name case) the search fails, an exception is thrown.
+     */
+    std::filesystem::path findPath(
+        const std::filesystem::path & exe,
+        std::function<bool(const std::filesystem::path &)> isExecutable = isExecutableFileAmbient) const;
+
+    bool operator==(const ExecutablePath &) const = default;
+};
+
+} // namespace nix

src/libutil/experimental-features.cc

@@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails
  * feature, we either have no issue at all if few features are not added
  * at the end of the list, or a proper merge conflict if they are.
  */
-constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::VerifiedFetches);
+constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::PipeOperators);
 
 constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails = {{
     {
@@ -294,6 +294,14 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         )",
         .trackingUrl = "https://github.com/NixOS/nix/milestone/48",
     },
+    {
+        .tag = Xp::PipeOperators,
+        .name = "pipe-operators",
+        .description = R"(
+            Add `|>` and `<|` operators to the Nix language.
+        )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/55",
+    },
 }};
 
 static_assert(

src/libutil/experimental-features.hh

@@ -35,6 +35,7 @@ enum struct ExperimentalFeature
     ConfigurableImpureEnv,
     MountedSSHStore,
     VerifiedFetches,
+    PipeOperators,
 };
 
 /**

src/libutil/file-content-address.cc

@@ -63,7 +63,7 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method)
     case FileIngestionMethod::Git:
         return "git";
     default:
-        abort();
+        unreachable();
     }
 }
 

src/libutil/file-path-impl.hh

@@ -91,13 +91,10 @@ struct WindowsPathTrait
 };
 
 
-/**
- * @todo Revisit choice of `char` or `wchar_t` for `WindowsPathTrait`
- * argument.
- */
-using NativePathTrait =
+template<typename CharT>
+using OsPathTrait =
 #ifdef _WIN32
-    WindowsPathTrait<char>
+    WindowsPathTrait<CharT>
 #else
     UnixPathTrait
 #endif

src/libutil/file-path.hh

@@ -1,10 +1,10 @@
 #pragma once
 ///@file
 
-#include <optional>
 #include <filesystem>
 
 #include "types.hh"
+#include "os-string.hh"
 
 namespace nix {
 
@@ -22,39 +22,26 @@ typedef std::set<std::filesystem::path> PathSetNG;
  *
  * @todo drop `NG` suffix and replace the one in `types.hh`.
  */
-struct PathViewNG : std::basic_string_view<std::filesystem::path::value_type>
+struct PathViewNG : OsStringView
 {
-    using string_view = std::basic_string_view<std::filesystem::path::value_type>;
+    using string_view = OsStringView;
 
     using string_view::string_view;
 
     PathViewNG(const std::filesystem::path & path)
-        : std::basic_string_view<std::filesystem::path::value_type>(path.native())
+        : OsStringView{path.native()}
     { }
 
-    PathViewNG(const std::filesystem::path::string_type & path)
-        : std::basic_string_view<std::filesystem::path::value_type>(path)
+    PathViewNG(const OsString & path)
+        : OsStringView{path}
     { }
 
     const string_view & native() const { return *this; }
     string_view & native() { return *this; }
 };
 
-std::string os_string_to_string(PathViewNG::string_view path);
-
-std::filesystem::path::string_type string_to_os_string(std::string_view s);
-
 std::optional<std::filesystem::path> maybePath(PathView path);
 
 std::filesystem::path pathNG(PathView path);
 
-/**
- * Create string literals with the native character width of paths
- */
-#ifndef _WIN32
-# define PATHNG_LITERAL(s) s
-#else
-# define PATHNG_LITERAL(s) L ## s
-#endif
-
 }

src/libutil/file-system.cc

@@ -92,7 +92,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
        arbitrary (but high) limit to prevent infinite loops. */
     unsigned int followCount = 0, maxFollow = 1024;
 
-    auto ret = canonPathInner<NativePathTrait>(
+    auto ret = canonPathInner<OsPathTrait<char>>(
         path,
         [&followCount, &temp, maxFollow, resolveSymlinks]
         (std::string & result, std::string_view & remaining) {
@@ -122,7 +122,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
 
 Path dirOf(const PathView path)
 {
-    Path::size_type pos = NativePathTrait::rfindPathSep(path);
+    Path::size_type pos = OsPathTrait<char>::rfindPathSep(path);
     if (pos == path.npos)
         return ".";
     return fs::path{path}.parent_path().string();
@@ -135,10 +135,10 @@ std::string_view baseNameOf(std::string_view path)
         return "";
 
     auto last = path.size() - 1;
-    while (last > 0 && NativePathTrait::isPathSep(path[last]))
+    while (last > 0 && OsPathTrait<char>::isPathSep(path[last]))
         last -= 1;
 
-    auto pos = NativePathTrait::rfindPathSep(path, last);
+    auto pos = OsPathTrait<char>::rfindPathSep(path, last);
     if (pos == path.npos)
         pos = 0;
     else
@@ -569,7 +569,7 @@ void replaceSymlink(const Path & target, const Path & link)
 }
 
 void setWriteTime(
-    const std::filesystem::path & path,
+    const fs::path & path,
     time_t accessedTime,
     time_t modificationTime,
     std::optional<bool> optIsSymlink)
@@ -685,4 +685,18 @@ void moveFile(const Path & oldName, const Path & newName)
 
 //////////////////////////////////////////////////////////////////////
 
+bool isExecutableFileAmbient(const fs::path & exe) {
+    // Check file type, because directory being executable means
+    // something completely different.
+    // `is_regular_file` follows symlinks before checking.
+    return std::filesystem::is_regular_file(exe)
+        && access(exe.string().c_str(),
+#ifdef WIN32
+        0 // TODO do better
+#else
+        X_OK
+#endif
+        ) == 0;
+}
+
 }

src/libutil/file-system.hh

@@ -263,6 +263,12 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
  */
 Path defaultTempDir();
 
+/**
+ * Interpret `exe` as a location in the ambient file system and return
+ * whether it resolves to a file that is executable.
+ */
+bool isExecutableFileAmbient(const std::filesystem::path & exe);
+
 /**
  * Used in various places.
  */

src/libutil/fs-sink.cc

@@ -53,7 +53,7 @@ void copyRecursive(
         throw Error("file '%1%' has an unsupported type", from);
 
     default:
-        abort();
+        unreachable();
     }
 }
 

src/libutil/git.cc

@@ -201,7 +201,7 @@ std::optional<Mode> convertMode(SourceAccessor::Type type)
     case SourceAccessor::tRegular:   return Mode::Regular;
     case SourceAccessor::tDirectory: return Mode::Directory;
     case SourceAccessor::tMisc:      return std::nullopt;
-    default: abort();
+    default: unreachable();
     }
 }
 

src/libutil/hash.cc

@@ -25,7 +25,7 @@ static size_t regularHashSize(HashAlgorithm type) {
     case HashAlgorithm::SHA256: return sha256HashSize;
     case HashAlgorithm::SHA512: return sha512HashSize;
     }
-    abort();
+    unreachable();
 }
 
 

src/libutil/logging.cc

@@ -189,7 +189,7 @@ struct JSONLogger : Logger {
             else if (f.type == Logger::Field::tString)
                 arr.push_back(f.s);
             else
-                abort();
+                unreachable();
     }
 
     void write(const nlohmann::json & json)

src/libutil/lru-cache.hh

@@ -89,7 +89,7 @@ public:
         return i->second.second;
     }
 
-    size_t size()
+    size_t size() const
     {
         return data.size();
     }

src/libutil/meson.build

@@ -62,6 +62,7 @@ endif
 boost = dependency(
   'boost',
   modules : ['context', 'coroutine'],
+  include_type: 'system',
 )
 # boost is a public dependency, but not a pkg-config dependency unfortunately, so we
 # put in `deps_other`.
@@ -128,6 +129,7 @@ sources = files(
   'english.cc',
   'environment-variables.cc',
   'error.cc',
+  'executable-path.cc',
   'exit.cc',
   'experimental-features.cc',
   'file-content-address.cc',
@@ -170,6 +172,7 @@ headers = [config_h] + files(
   'args/root.hh',
   'callback.hh',
   'canon-path.hh',
+  'checked-arithmetic.hh',
   'chunked-vector.hh',
   'closure.hh',
   'comparator.hh',
@@ -182,6 +185,7 @@ headers = [config_h] + files(
   'english.hh',
   'environment-variables.hh',
   'error.hh',
+  'executable-path.hh',
   'exit.hh',
   'experimental-features.hh',
   'file-content-address.hh',
@@ -201,6 +205,7 @@ headers = [config_h] + files(
   'lru-cache.hh',
   'memory-source-accessor.hh',
   'muxable-pipe.hh',
+  'os-string.hh',
   'pool.hh',
   'position.hh',
   'posix-source-accessor.hh',

src/libutil/os-string.hh

@@ -0,0 +1,43 @@
+#pragma once
+///@file
+
+#include <optional>
+#include <string>
+#include <string_view>
+
+namespace nix {
+
+/**
+ * Named because it is similar to the Rust type, except it is in the
+ * native encoding not WTF-8.
+ *
+ * Same as `std::filesystem::path::string_type`, but manually defined to
+ * avoid including a much more complex header.
+ */
+using OsString = std::basic_string<
+#if defined(_WIN32) && !defined(__CYGWIN__)
+    wchar_t
+#else
+    char
+#endif
+    >;
+
+/**
+ * `std::string_view` counterpart for `OsString`.
+ */
+using OsStringView = std::basic_string_view<OsString::value_type>;
+
+std::string os_string_to_string(OsStringView path);
+
+OsString string_to_os_string(std::string_view s);
+
+/**
+ * Create string literals with the native character width of paths
+ */
+#ifndef _WIN32
+#  define OS_STR(s) s
+#else
+#  define OS_STR(s) L##s
+#endif
+
+}

src/libutil/package.nix

@@ -88,12 +88,8 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  enableParallelBuilding = true;
-
   separateDebugInfo = !stdenv.hostPlatform.isStatic;
 
-  strictDeps = true;
-
   hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
 
   meta = {

src/libutil/posix-source-accessor.cc

@@ -97,7 +97,7 @@ std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & pa
     Path absPath = makeAbsPath(path).string();
 
     {
-        auto cache(_cache.read());
+        auto cache(_cache.readLock());
         auto i = cache->find(absPath);
         if (i != cache->end()) return i->second;
     }

src/libutil/ref.hh

@@ -23,14 +23,14 @@ public:
         : p(r.p)
     { }
 
-    explicit ref<T>(const std::shared_ptr<T> & p)
+    explicit ref(const std::shared_ptr<T> & p)
         : p(p)
     {
         if (!p)
             throw std::invalid_argument("null pointer cast to ref");
     }
 
-    explicit ref<T>(T * p)
+    explicit ref(T * p)
         : p(p)
     {
         if (!p)

src/libutil/serialise.cc

@@ -209,7 +209,7 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
                 });
             }
 
-            if (!*coro) { abort(); }
+            if (!*coro) { unreachable(); }
 
             if (!cur.empty()) {
                 (*coro)(false);
@@ -258,7 +258,7 @@ std::unique_ptr<Source> sinkToSource(
                 });
             }
 
-            if (!*coro) { eof(); abort(); }
+            if (!*coro) { eof(); unreachable(); }
 
             if (pos == cur.size()) {
                 if (!cur.empty()) {

src/libutil/serialise.hh

@@ -204,7 +204,7 @@ struct TeeSink : Sink
 {
     Sink & sink1, & sink2;
     TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
-    virtual void operator () (std::string_view data)
+    virtual void operator () (std::string_view data) override
     {
         sink1(data);
         sink2(data);
@@ -221,7 +221,7 @@ struct TeeSource : Source
     Sink & sink;
     TeeSource(Source & orig, Sink & sink)
         : orig(orig), sink(sink) { }
-    size_t read(char * data, size_t len)
+    size_t read(char * data, size_t len) override
     {
         size_t n = orig.read(data, len);
         sink({data, n});
@@ -238,7 +238,7 @@ struct SizedSource : Source
     size_t remain;
     SizedSource(Source & orig, size_t size)
         : orig(orig), remain(size) { }
-    size_t read(char * data, size_t len)
+    size_t read(char * data, size_t len) override
     {
         if (this->remain <= 0) {
             throw EndOfFile("sized: unexpected end-of-file");

src/libutil/strings-inline.hh

@@ -4,8 +4,51 @@
 
 namespace nix {
 
+template<class C, class CharT>
+C basicTokenizeString(std::basic_string_view<CharT> s, std::basic_string_view<CharT> separators)
+{
+    C result;
+    auto pos = s.find_first_not_of(separators, 0);
+    while (pos != s.npos) {
+        auto end = s.find_first_of(separators, pos + 1);
+        if (end == s.npos)
+            end = s.size();
+        result.insert(result.end(), std::basic_string<CharT>(s, pos, end - pos));
+        pos = s.find_first_not_of(separators, end);
+    }
+    return result;
+}
+
 template<class C>
-std::string concatStringsSep(const std::string_view sep, const C & ss)
+C tokenizeString(std::string_view s, std::string_view separators)
+{
+    return basicTokenizeString<C, char>(s, separators);
+}
+
+template<class C, class CharT>
+C basicSplitString(std::basic_string_view<CharT> s, std::basic_string_view<CharT> separators)
+{
+    C result;
+    size_t pos = 0;
+    while (pos <= s.size()) {
+        auto end = s.find_first_of(separators, pos);
+        if (end == s.npos)
+            end = s.size();
+        result.insert(result.end(), std::basic_string<CharT>(s, pos, end - pos));
+        pos = end + 1;
+    }
+
+    return result;
+}
+
+template<class C>
+C splitString(std::string_view s, std::string_view separators)
+{
+    return basicSplitString<C, char>(s, separators);
+}
+
+template<class CharT, class C>
+std::basic_string<CharT> basicConcatStringsSep(const std::basic_string_view<CharT> sep, const C & ss)
 {
     size_t size = 0;
     bool tail = false;
@@ -13,10 +56,10 @@ std::string concatStringsSep(const std::string_view sep, const C & ss)
     for (const auto & s : ss) {
         if (tail)
             size += sep.size();
-        size += std::string_view(s).size();
+        size += std::basic_string_view<CharT>{s}.size();
         tail = true;
     }
-    std::string s;
+    std::basic_string<CharT> s;
     s.reserve(size);
     tail = false;
     for (auto & i : ss) {
@@ -28,4 +71,36 @@ std::string concatStringsSep(const std::string_view sep, const C & ss)
     return s;
 }
 
+template<class C>
+std::string concatStringsSep(const std::string_view sep, const C & ss)
+{
+    return basicConcatStringsSep<char, C>(sep, ss);
+}
+
+template<class C>
+std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss)
+{
+    size_t size = 0;
+
+    // TODO? remove to make sure we don't rely on the empty item ignoring behavior,
+    //       or just get rid of this function by understanding the remaining calls.
+    // for (auto & i : ss) {
+    //     // Make sure we don't rely on the empty item ignoring behavior
+    //     assert(!i.empty());
+    //     break;
+    // }
+
+    // need a cast to string_view since this is also called with Symbols
+    for (const auto & s : ss)
+        size += sep.size() + std::string_view(s).size();
+    std::string s;
+    s.reserve(size);
+    for (auto & i : ss) {
+        if (s.size() != 0)
+            s += sep;
+        s += i;
+    }
+    return s;
+}
+
 } // namespace nix

src/libutil/strings.cc

@@ -1,12 +1,24 @@
+#include <filesystem>
 #include <string>
 
 #include "strings-inline.hh"
-#include "util.hh"
+#include "os-string.hh"
 
 namespace nix {
 
-template std::string concatStringsSep(std::string_view, const Strings &);
-template std::string concatStringsSep(std::string_view, const StringSet &);
+template std::list<std::string> tokenizeString(std::string_view s, std::string_view separators);
+template std::set<std::string> tokenizeString(std::string_view s, std::string_view separators);
+template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);
+
+template std::list<std::string> splitString(std::string_view s, std::string_view separators);
+template std::set<std::string> splitString(std::string_view s, std::string_view separators);
+template std::vector<std::string> splitString(std::string_view s, std::string_view separators);
+
+template std::list<OsString> basicSplitString(
+    std::basic_string_view<OsString::value_type> s, std::basic_string_view<OsString::value_type> separators);
+
+template std::string concatStringsSep(std::string_view, const std::list<std::string> &);
+template std::string concatStringsSep(std::string_view, const std::set<std::string> &);
 template std::string concatStringsSep(std::string_view, const std::vector<std::string> &);
 
 typedef std::string_view strings_2[2];
@@ -16,4 +28,8 @@ template std::string concatStringsSep(std::string_view, const strings_3 &);
 typedef std::string_view strings_4[4];
 template std::string concatStringsSep(std::string_view, const strings_4 &);
 
+template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list<std::string> &);
+template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set<std::string> &);
+template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector<std::string> &);
+
 } // namespace nix

src/libutil/strings.hh

@@ -8,6 +8,38 @@
 
 namespace nix {
 
+/**
+ * String tokenizer.
+ *
+ * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end.
+ */
+template<class C, class CharT = char>
+C basicTokenizeString(std::basic_string_view<CharT> s, std::basic_string_view<CharT> separators);
+
+/**
+ * Like `basicTokenizeString` but specialized to the default `char`
+ */
+template<class C>
+C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r");
+
+extern template std::list<std::string> tokenizeString(std::string_view s, std::string_view separators);
+extern template std::set<std::string> tokenizeString(std::string_view s, std::string_view separators);
+extern template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);
+
+/**
+ * Split a string, preserving empty strings between separators, as well as at the start and end.
+ *
+ * Returns a non-empty collection of strings.
+ */
+template<class C, class CharT = char>
+C basicSplitString(std::basic_string_view<CharT> s, std::basic_string_view<CharT> separators);
+template<typename C>
+C splitString(std::string_view s, std::string_view separators);
+
+extern template std::list<std::string> splitString(std::string_view s, std::string_view separators);
+extern template std::set<std::string> splitString(std::string_view s, std::string_view separators);
+extern template std::vector<std::string> splitString(std::string_view s, std::string_view separators);
+
 /**
  * Concatenate the given strings with a separator between the elements.
  */
@@ -18,4 +50,20 @@ extern template std::string concatStringsSep(std::string_view, const std::list<s
 extern template std::string concatStringsSep(std::string_view, const std::set<std::string> &);
 extern template std::string concatStringsSep(std::string_view, const std::vector<std::string> &);
 
+/**
+ * Ignore any empty strings at the start of the list, and then concatenate the
+ * given strings with a separator between the elements.
+ *
+ * @deprecated This function exists for historical reasons. You probably just
+ *             want to use `concatStringsSep`.
+ */
+template<class C>
+[[deprecated(
+    "Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] std::string
+dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss);
+
+extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list<std::string> &);
+extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set<std::string> &);
+extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector<std::string> &);
+
 }

src/libutil/sync.hh

@@ -7,6 +7,8 @@
 #include <condition_variable>
 #include <cassert>
 
+#include "error.hh"
+
 namespace nix {
 
 /**
@@ -47,7 +49,7 @@ public:
         friend SyncBase;
         Lock(SyncBase * s) : s(s), lk(s->mutex) { }
     public:
-        Lock(Lock && l) : s(l.s) { abort(); }
+        Lock(Lock && l) : s(l.s) { unreachable(); }
         Lock(const Lock & l) = delete;
         ~Lock() { }
 
@@ -104,7 +106,7 @@ public:
      * Acquire read access to the inner value. When using
      * `std::shared_mutex`, this will use a shared lock.
      */
-    ReadLock read() const { return ReadLock(const_cast<SyncBase *>(this)); }
+    ReadLock readLock() const { return ReadLock(const_cast<SyncBase *>(this)); }
 };
 
 template<class T>

src/libutil/unix/environment-variables.cc

@@ -9,4 +9,14 @@ int setEnv(const char * name, const char * value)
     return ::setenv(name, value, 1);
 }
 
+std::optional<std::string> getEnvOs(const std::string & key)
+{
+    return getEnv(key);
+}
+
+int setEnvOs(const OsString & name, const OsString & value)
+{
+    return setEnv(name.c_str(), value.c_str());
+}
+
 }

src/libutil/unix/file-path.cc

@@ -8,16 +8,6 @@
 
 namespace nix {
 
-std::string os_string_to_string(PathViewNG::string_view path)
-{
-    return std::string { path };
-}
-
-std::filesystem::path::string_type string_to_os_string(std::string_view s)
-{
-    return std::string { s };
-}
-
 std::optional<std::filesystem::path> maybePath(PathView path)
 {
     return { path };

src/libutil/unix/meson.build

@@ -4,6 +4,7 @@ sources += files(
   'file-path.cc',
   'file-system.cc',
   'muxable-pipe.cc',
+  'os-string.cc',
   'processes.cc',
   'signals.cc',
   'users.cc',

src/libutil/unix/monitor-fd.hh

@@ -40,7 +40,9 @@ public:
                     #endif
                     ;
                 auto count = poll(fds, 1, -1);
-                if (count == -1) abort(); // can't happen
+                if (count == -1)
+                    unreachable();
+
                 /* This shouldn't happen, but can on macOS due to a bug.
                    See rdar://37550628.
 

src/libutil/unix/os-string.cc

@@ -0,0 +1,21 @@
+#include <algorithm>
+#include <codecvt>
+#include <iostream>
+#include <locale>
+
+#include "file-path.hh"
+#include "util.hh"
+
+namespace nix {
+
+std::string os_string_to_string(PathViewNG::string_view path)
+{
+    return std::string{path};
+}
+
+std::filesystem::path::string_type string_to_os_string(std::string_view s)
+{
+    return std::string{s};
+}
+
+}

src/libutil/unix/processes.cc

@@ -182,7 +182,7 @@ static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun)
 #endif
     if (pid != 0) return pid;
     fun();
-    abort();
+    unreachable();
 }
 
 

src/libutil/util.cc

@@ -1,5 +1,6 @@
 #include "util.hh"
 #include "fmt.hh"
+#include "file-path.hh"
 
 #include <array>
 #include <cctype>
@@ -53,24 +54,6 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 //////////////////////////////////////////////////////////////////////
 
 
-template<class C> C tokenizeString(std::string_view s, std::string_view separators)
-{
-    C result;
-    auto pos = s.find_first_not_of(separators, 0);
-    while (pos != s.npos) {
-        auto end = s.find_first_of(separators, pos + 1);
-        if (end == s.npos) end = s.size();
-        result.insert(result.end(), std::string(s, pos, end - pos));
-        pos = s.find_first_not_of(separators, end);
-    }
-    return result;
-}
-
-template Strings tokenizeString(std::string_view s, std::string_view separators);
-template StringSet tokenizeString(std::string_view s, std::string_view separators);
-template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);
-
-
 std::string chomp(std::string_view s)
 {
     size_t i = s.find_last_not_of(" \n\r\t");

src/libutil/util.hh

@@ -28,49 +28,11 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss);
 MakeError(FormatError, Error);
 
 
-/**
- * String tokenizer.
- */
-template<class C> C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r");
-
-
-/**
- * Ignore any empty strings at the start of the list, and then concatenate the
- * given strings with a separator between the elements.
- *
- * @deprecated This function exists for historical reasons. You probably just
- *             want to use `concatStringsSep`.
- */
-template<class C>
-[[deprecated("Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]]
-std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss)
-{
-    size_t size = 0;
-
-    // TODO? remove to make sure we don't rely on the empty item ignoring behavior,
-    //       or just get rid of this function by understanding the remaining calls.
-    // for (auto & i : ss) {
-    //     // Make sure we don't rely on the empty item ignoring behavior
-    //     assert(!i.empty());
-    //     break;
-    // }
-
-    // need a cast to string_view since this is also called with Symbols
-    for (const auto & s : ss) size += sep.size() + std::string_view(s).size();
-    std::string s;
-    s.reserve(size);
-    for (auto & i : ss) {
-        if (s.size() != 0) s += sep;
-        s += i;
-    }
-    return s;
-}
-
-template<class ... Parts>
-auto concatStrings(Parts && ... parts)
+template<class... Parts>
+auto concatStrings(Parts &&... parts)
     -> std::enable_if_t<(... && std::is_convertible_v<Parts, std::string_view>), std::string>
 {
-    std::string_view views[sizeof...(parts)] = { parts... };
+    std::string_view views[sizeof...(parts)] = {parts...};
     return concatStringsSep({}, views);
 }
 

src/libutil/windows/environment-variables.cc

@@ -4,7 +4,30 @@
 
 namespace nix {
 
-int unsetenv(const char *name)
+std::optional<OsString> getEnvOs(const OsString & key)
+{
+    // Determine the required buffer size for the environment variable value
+    DWORD bufferSize = GetEnvironmentVariableW(key.c_str(), nullptr, 0);
+    if (bufferSize == 0) {
+        return std::nullopt;
+    }
+
+    // Allocate a buffer to hold the environment variable value
+    std::wstring value{L'\0', bufferSize};
+
+    // Retrieve the environment variable value
+    DWORD resultSize = GetEnvironmentVariableW(key.c_str(), &value[0], bufferSize);
+    if (resultSize == 0) {
+        return std::nullopt;
+    }
+
+    // Resize the string to remove the extra null characters
+    value.resize(resultSize);
+
+    return value;
+}
+
+int unsetenv(const char * name)
 {
     return -SetEnvironmentVariableA(name, nullptr);
 }
@@ -14,4 +37,9 @@ int setEnv(const char * name, const char * value)
     return -SetEnvironmentVariableA(name, value);
 }
 
+int setEnvOs(const OsString & name, const OsString & value)
+{
+    return -SetEnvironmentVariableW(name.c_str(), value.c_str());
+}
+
 }

src/libutil/windows/file-path.cc

@@ -9,18 +9,6 @@
 
 namespace nix {
 
-std::string os_string_to_string(PathViewNG::string_view path)
-{
-    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
-    return converter.to_bytes(std::filesystem::path::string_type { path });
-}
-
-std::filesystem::path::string_type string_to_os_string(std::string_view s)
-{
-    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
-    return converter.from_bytes(std::string { s });
-}
-
 std::optional<std::filesystem::path> maybePath(PathView path)
 {
     if (path.length() >= 3 && (('A' <= path[0] && path[0] <= 'Z') || ('a' <= path[0] && path[0] <= 'z')) && path[1] == ':' && WindowsPathTrait<char>::isPathSep(path[2])) {

src/libutil/windows/meson.build

@@ -4,6 +4,7 @@ sources += files(
   'file-path.cc',
   'file-system.cc',
   'muxable-pipe.cc',
+  'os-string.cc',
   'processes.cc',
   'users.cc',
   'windows-async-pipe.cc',

src/libutil/windows/os-string.cc

@@ -0,0 +1,24 @@
+#include <algorithm>
+#include <codecvt>
+#include <iostream>
+#include <locale>
+
+#include "file-path.hh"
+#include "file-path-impl.hh"
+#include "util.hh"
+
+namespace nix {
+
+std::string os_string_to_string(PathViewNG::string_view path)
+{
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
+    return converter.to_bytes(std::filesystem::path::string_type{path});
+}
+
+std::filesystem::path::string_type string_to_os_string(std::string_view s)
+{
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
+    return converter.from_bytes(std::string{s});
+}
+
+}