Merge pull request #9177 from edolstra/input-accessors

Backport FSInputAccessor and MemoryInputAccessor from lazy-trees

src/libutil/archive.cc

@@ -14,6 +14,7 @@
 #include "archive.hh"
 #include "util.hh"
 #include "config.hh"
+#include "posix-source-accessor.hh"
 
 namespace nix {
 
@@ -36,91 +37,87 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
 PathFilter defaultPathFilter = [](const Path &) { return true; };
 
 
-static void dumpContents(const Path & path, off_t size,
-    Sink & sink)
+void SourceAccessor::dumpPath(
+    const CanonPath & path,
+    Sink & sink,
+    PathFilter & filter)
 {
-    sink << "contents" << size;
+    auto dumpContents = [&](const CanonPath & path)
+    {
+        sink << "contents";
+        std::optional<uint64_t> size;
+        readFile(path, sink, [&](uint64_t _size)
+        {
+            size = _size;
+            sink << _size;
+        });
+        assert(size);
+        writePadding(*size, sink);
+    };
 
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError("opening file '%1%'", path);
+    std::function<void(const CanonPath & path)> dump;
 
-    std::vector<char> buf(65536);
-    size_t left = size;
+    dump = [&](const CanonPath & path) {
+        checkInterrupt();
 
-    while (left > 0) {
-        auto n = std::min(left, buf.size());
-        readFull(fd.get(), buf.data(), n);
-        left -= n;
-        sink({buf.data(), n});
-    }
+        auto st = lstat(path);
 
-    writePadding(size, sink);
-}
+        sink << "(";
 
+        if (st.type == tRegular) {
+            sink << "type" << "regular";
+            if (st.isExecutable)
+                sink << "executable" << "";
+            dumpContents(path);
+        }
 
-static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
-{
-    checkInterrupt();
+        else if (st.type == tDirectory) {
+            sink << "type" << "directory";
 
-    auto st = lstat(path);
-    time_t result = st.st_mtime;
+            /* If we're on a case-insensitive system like macOS, undo
+               the case hack applied by restorePath(). */
+            std::map<std::string, std::string> unhacked;
+            for (auto & i : readDirectory(path))
+                if (archiveSettings.useCaseHack) {
+                    std::string name(i.first);
+                    size_t pos = i.first.find(caseHackSuffix);
+                    if (pos != std::string::npos) {
+                        debug("removing case hack suffix from '%s'", path + i.first);
+                        name.erase(pos);
+                    }
+                    if (!unhacked.emplace(name, i.first).second)
+                        throw Error("file name collision in between '%s' and '%s'",
+                            (path + unhacked[name]),
+                            (path + i.first));
+                } else
+                    unhacked.emplace(i.first, i.first);
 
-    sink << "(";
-
-    if (S_ISREG(st.st_mode)) {
-        sink << "type" << "regular";
-        if (st.st_mode & S_IXUSR)
-            sink << "executable" << "";
-        dumpContents(path, st.st_size, sink);
-    }
-
-    else if (S_ISDIR(st.st_mode)) {
-        sink << "type" << "directory";
-
-        /* If we're on a case-insensitive system like macOS, undo
-           the case hack applied by restorePath(). */
-        std::map<std::string, std::string> unhacked;
-        for (auto & i : readDirectory(path))
-            if (archiveSettings.useCaseHack) {
-                std::string name(i.name);
-                size_t pos = i.name.find(caseHackSuffix);
-                if (pos != std::string::npos) {
-                    debug("removing case hack suffix from '%1%'", path + "/" + i.name);
-                    name.erase(pos);
+            for (auto & i : unhacked)
+                if (filter((path + i.first).abs())) {
+                    sink << "entry" << "(" << "name" << i.first << "node";
+                    dump(path + i.second);
+                    sink << ")";
                 }
-                if (!unhacked.emplace(name, i.name).second)
-                    throw Error("file name collision in between '%1%' and '%2%'",
-                       (path + "/" + unhacked[name]),
-                       (path + "/" + i.name));
-            } else
-                unhacked.emplace(i.name, i.name);
+        }
 
-        for (auto & i : unhacked)
-            if (filter(path + "/" + i.first)) {
-                sink << "entry" << "(" << "name" << i.first << "node";
-                auto tmp_mtime = dump(path + "/" + i.second, sink, filter);
-                if (tmp_mtime > result) {
-                    result = tmp_mtime;
-                }
-                sink << ")";
-            }
-    }
+        else if (st.type == tSymlink)
+            sink << "type" << "symlink" << "target" << readLink(path);
 
-    else if (S_ISLNK(st.st_mode))
-        sink << "type" << "symlink" << "target" << readLink(path);
+        else throw Error("file '%s' has an unsupported type", path);
 
-    else throw Error("file '%1%' has an unsupported type", path);
+        sink << ")";
+    };
 
-    sink << ")";
-
-    return result;
+    sink << narVersionMagic1;
+    dump(path);
 }
 
 
 time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
 {
-    sink << narVersionMagic1;
-    return dump(path, sink, filter);
+    PosixSourceAccessor accessor;
+    accessor.dumpPath(CanonPath::fromCwd(path), sink, filter);
+    return accessor.mtime;
 }
 
 void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
@@ -141,17 +138,6 @@ static SerialisationError badArchive(const std::string & s)
 }
 
 
-#if 0
-static void skipGeneric(Source & source)
-{
-    if (readString(source) == "(") {
-        while (readString(source) != ")")
-            skipGeneric(source);
-    }
-}
-#endif
-
-
 static void parseContents(ParseSink & sink, Source & source, const Path & path)
 {
     uint64_t size = readLongLong(source);

src/libutil/posix-source-accessor.cc

@@ -0,0 +1,86 @@
+#include "posix-source-accessor.hh"
+
+namespace nix {
+
+void PosixSourceAccessor::readFile(
+    const CanonPath & path,
+    Sink & sink,
+    std::function<void(uint64_t)> sizeCallback)
+{
+    // FIXME: add O_NOFOLLOW since symlinks should be resolved by the
+    // caller?
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+
+    struct stat st;
+    if (fstat(fd.get(), &st) == -1)
+        throw SysError("statting file");
+
+    sizeCallback(st.st_size);
+
+    off_t left = st.st_size;
+
+    std::vector<unsigned char> buf(64 * 1024);
+    while (left) {
+        checkInterrupt();
+        ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size()));
+        if (rd == -1) {
+            if (errno != EINTR)
+                throw SysError("reading from file '%s'", showPath(path));
+        }
+        else if (rd == 0)
+            throw SysError("unexpected end-of-file reading '%s'", showPath(path));
+        else {
+            assert(rd <= left);
+            sink({(char *) buf.data(), (size_t) rd});
+            left -= rd;
+        }
+    }
+}
+
+bool PosixSourceAccessor::pathExists(const CanonPath & path)
+{
+    return nix::pathExists(path.abs());
+}
+
+SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path)
+{
+    auto st = nix::lstat(path.abs());
+    mtime = std::max(mtime, st.st_mtime);
+    return Stat {
+        .type =
+            S_ISREG(st.st_mode) ? tRegular :
+            S_ISDIR(st.st_mode) ? tDirectory :
+            S_ISLNK(st.st_mode) ? tSymlink :
+            tMisc,
+        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
+    };
+}
+
+SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path)
+{
+    DirEntries res;
+    for (auto & entry : nix::readDirectory(path.abs())) {
+        std::optional<Type> type;
+        switch (entry.type) {
+        case DT_REG: type = Type::tRegular; break;
+        case DT_LNK: type = Type::tSymlink; break;
+        case DT_DIR: type = Type::tDirectory; break;
+        }
+        res.emplace(entry.name, type);
+    }
+    return res;
+}
+
+std::string PosixSourceAccessor::readLink(const CanonPath & path)
+{
+    return nix::readLink(path.abs());
+}
+
+std::optional<CanonPath> PosixSourceAccessor::getPhysicalPath(const CanonPath & path)
+{
+    return path;
+}
+
+}

src/libutil/posix-source-accessor.hh

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "source-accessor.hh"
+
+namespace nix {
+
+/**
+ * A source accessor that uses the Unix filesystem.
+ */
+struct PosixSourceAccessor : SourceAccessor
+{
+    /**
+     * The most recent mtime seen by lstat(). This is a hack to
+     * support dumpPathAndGetMtime(). Should remove this eventually.
+     */
+    time_t mtime = 0;
+
+    void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback) override;
+
+    bool pathExists(const CanonPath & path) override;
+
+    Stat lstat(const CanonPath & path) override;
+
+    DirEntries readDirectory(const CanonPath & path) override;
+
+    std::string readLink(const CanonPath & path) override;
+
+    std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override;
+};
+
+}

src/libutil/source-accessor.cc

@@ -0,0 +1,58 @@
+#include "source-accessor.hh"
+#include "archive.hh"
+
+namespace nix {
+
+static std::atomic<size_t> nextNumber{0};
+
+SourceAccessor::SourceAccessor()
+    : number(++nextNumber)
+{
+}
+
+std::string SourceAccessor::readFile(const CanonPath & path)
+{
+    StringSink sink;
+    std::optional<uint64_t> size;
+    readFile(path, sink, [&](uint64_t _size)
+    {
+        size = _size;
+    });
+    assert(size && *size == sink.s.size());
+    return std::move(sink.s);
+}
+
+void SourceAccessor::readFile(
+    const CanonPath & path,
+    Sink & sink,
+    std::function<void(uint64_t)> sizeCallback)
+{
+    auto s = readFile(path);
+    sizeCallback(s.size());
+    sink(s);
+}
+
+Hash SourceAccessor::hashPath(
+    const CanonPath & path,
+    PathFilter & filter,
+    HashType ht)
+{
+    HashSink sink(ht);
+    dumpPath(path, sink, filter);
+    return sink.finish().first;
+}
+
+std::optional<SourceAccessor::Stat> SourceAccessor::maybeLstat(const CanonPath & path)
+{
+    // FIXME: merge these into one operation.
+    if (!pathExists(path))
+        return {};
+    return lstat(path);
+}
+
+std::string SourceAccessor::showPath(const CanonPath & path)
+{
+    return path.abs();
+}
+
+}

src/libutil/source-accessor.hh

@@ -0,0 +1,107 @@
+#pragma once
+
+#include "canon-path.hh"
+#include "hash.hh"
+
+namespace nix {
+
+struct Sink;
+
+/**
+ * A read-only filesystem abstraction. This is used by the Nix
+ * evaluator and elsewhere for accessing sources in various
+ * filesystem-like entities (such as the real filesystem, tarballs or
+ * Git repositories).
+ */
+struct SourceAccessor
+{
+    const size_t number;
+
+    SourceAccessor();
+
+    virtual ~SourceAccessor()
+    { }
+
+    /**
+     * Return the contents of a file as a string.
+     */
+    virtual std::string readFile(const CanonPath & path);
+
+    /**
+     * Write the contents of a file as a sink. `sizeCallback` must be
+     * called with the size of the file before any data is written to
+     * the sink.
+     *
+     * Note: subclasses of `SourceAccessor` need to implement at least
+     * one of the `readFile()` variants.
+     */
+    virtual void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback = [](uint64_t size){});
+
+    virtual bool pathExists(const CanonPath & path) = 0;
+
+    enum Type {
+      tRegular, tSymlink, tDirectory,
+      /**
+        Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things.
+
+        Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`.
+
+        Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types.
+      */
+      tMisc
+    };
+
+    struct Stat
+    {
+        Type type = tMisc;
+        //uint64_t fileSize = 0; // regular files only
+        bool isExecutable = false; // regular files only
+    };
+
+    virtual Stat lstat(const CanonPath & path) = 0;
+
+    std::optional<Stat> maybeLstat(const CanonPath & path);
+
+    typedef std::optional<Type> DirEntry;
+
+    typedef std::map<std::string, DirEntry> DirEntries;
+
+    virtual DirEntries readDirectory(const CanonPath & path) = 0;
+
+    virtual std::string readLink(const CanonPath & path) = 0;
+
+    virtual void dumpPath(
+        const CanonPath & path,
+        Sink & sink,
+        PathFilter & filter = defaultPathFilter);
+
+    Hash hashPath(
+        const CanonPath & path,
+        PathFilter & filter = defaultPathFilter,
+        HashType ht = htSHA256);
+
+    /**
+     * Return a corresponding path in the root filesystem, if
+     * possible. This is only possible for filesystems that are
+     * materialized in the root filesystem.
+     */
+    virtual std::optional<CanonPath> getPhysicalPath(const CanonPath & path)
+    { return std::nullopt; }
+
+    bool operator == (const SourceAccessor & x) const
+    {
+        return number == x.number;
+    }
+
+    bool operator < (const SourceAccessor & x) const
+    {
+        return number < x.number;
+    }
+
+    virtual std::string showPath(const CanonPath & path);
+};
+
+}