Merge remote-tracking branch 'origin/master' into lfs

src/libfetchers/git-utils.cc

@@ -14,13 +14,17 @@
 #include <git2/describe.h>
 #include <git2/errors.h>
 #include <git2/global.h>
+#include <git2/indexer.h>
 #include <git2/object.h>
+#include <git2/odb.h>
 #include <git2/refs.h>
 #include <git2/remote.h>
 #include <git2/repository.h>
 #include <git2/revparse.h>
 #include <git2/status.h>
 #include <git2/submodule.h>
+#include <git2/sys/odb_backend.h>
+#include <git2/sys/mempack.h>
 #include <git2/tree.h>
 
 #include <iostream>
@@ -77,6 +81,9 @@ typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusLi
 typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
 typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
 typedef std::unique_ptr<git_config_iterator, Deleter<git_config_iterator_free>> ConfigIterator;
+typedef std::unique_ptr<git_odb, Deleter<git_odb_free>> ObjectDb;
+typedef std::unique_ptr<git_packbuilder, Deleter<git_packbuilder_free>> PackBuilder;
+typedef std::unique_ptr<git_indexer, Deleter<git_indexer_free>> Indexer;
 
 // A helper to ensure that we don't leak objects returned by libgit2.
 template<typename T>
@@ -160,24 +167,101 @@ static Object peelToTreeOrBlob(git_object * obj)
         return peelObject<Object>(obj, GIT_OBJECT_TREE);
 }
 
+struct PackBuilderContext {
+    std::exception_ptr exception;
+
+    void handleException(const char * activity, int errCode)
+    {
+        switch (errCode) {
+            case GIT_OK:
+                break;
+            case GIT_EUSER:
+                if (!exception)
+                    panic("PackBuilderContext::handleException: user error, but exception was not set");
+
+                std::rethrow_exception(exception);
+            default:
+                throw Error("%s: %i, %s", Uncolored(activity), errCode, git_error_last()->message);
+        }
+    }
+};
+
+extern "C" {
+
+/**
+ * A `git_packbuilder_progress` implementation that aborts the pack building if needed.
+ */
+static int packBuilderProgressCheckInterrupt(int stage, uint32_t current, uint32_t total, void *payload)
+{
+    PackBuilderContext & args = * (PackBuilderContext *) payload;
+    try {
+        checkInterrupt();
+        return GIT_OK;
+    } catch (const std::exception & e) {
+        args.exception = std::current_exception();
+        return GIT_EUSER;
+    }
+};
+static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuilderProgressCheckInterrupt;
+
+} // extern "C"
+
+static void initRepoAtomically(std::filesystem::path &path, bool bare) {
+    if (pathExists(path.string())) return;
+
+    Path tmpDir = createTempDir(os_string_to_string(PathViewNG { std::filesystem::path(path).parent_path() }));
+    AutoDelete delTmpDir(tmpDir, true);
+    Repository tmpRepo;
+
+    if (git_repository_init(Setter(tmpRepo), tmpDir.c_str(), bare))
+        throw Error("creating Git repository %s: %s", path, git_error_last()->message);
+    try {
+        std::filesystem::rename(tmpDir, path);
+    } catch (std::filesystem::filesystem_error & e) {
+        if (e.code() == std::errc::file_exists) // Someone might race us to create the repository.
+            return;
+        else
+            throw SysError("moving temporary git repository from %s to %s", tmpDir, path);
+    }
+    // we successfully moved the repository, so the temporary directory no longer exists.
+    delTmpDir.cancel();
+}
+
 struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 {
     /** Location of the repository on disk. */
     std::filesystem::path path;
+    /**
+     * libgit2 repository. Note that new objects are not written to disk,
+     * because we are using a mempack backend. For writing to disk, see
+     * `flush()`, which is also called by `GitFileSystemObjectSink::sync()`.
+     */
     Repository repo;
+    /**
+     * In-memory object store for efficient batched writing to packfiles.
+     * Owned by `repo`.
+     */
+    git_odb_backend * mempack_backend;
 
     GitRepoImpl(std::filesystem::path _path, bool create, bool bare)
         : path(std::move(_path))
     {
         initLibGit2();
 
-        if (pathExists(path.string())) {
-            if (git_repository_open(Setter(repo), path.string().c_str()))
-                throw Error("opening Git repository '%s': %s", path, git_error_last()->message);
-        } else {
-            if (git_repository_init(Setter(repo), path.string().c_str(), bare))
-                throw Error("creating Git repository '%s': %s", path, git_error_last()->message);
-        }
+        initRepoAtomically(path, bare);
+        if (git_repository_open(Setter(repo), path.string().c_str()))
+            throw Error("opening Git repository %s: %s", path, git_error_last()->message);
+
+        ObjectDb odb;
+        if (git_repository_odb(Setter(odb), repo.get()))
+            throw Error("getting Git object database: %s", git_error_last()->message);
+
+        // mempack_backend will be owned by the repository, so we are not expected to free it ourselves.
+        if (git_mempack_new(&mempack_backend))
+            throw Error("creating mempack backend: %s", git_error_last()->message);
+
+        if (git_odb_add_backend(odb.get(), mempack_backend, 999))
+            throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
     }
 
     operator git_repository * ()
@@ -185,6 +269,62 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return repo.get();
     }
 
+    void flush() override {
+        checkInterrupt();
+
+        git_buf buf = GIT_BUF_INIT;
+        Finally _disposeBuf { [&] { git_buf_dispose(&buf); } };
+        PackBuilder packBuilder;
+        PackBuilderContext packBuilderContext;
+        git_packbuilder_new(Setter(packBuilder), *this);
+        git_packbuilder_set_callbacks(packBuilder.get(), PACKBUILDER_PROGRESS_CHECK_INTERRUPT, &packBuilderContext);
+        git_packbuilder_set_threads(packBuilder.get(), 0 /* autodetect */);
+
+        packBuilderContext.handleException(
+            "preparing packfile",
+            git_mempack_write_thin_pack(mempack_backend, packBuilder.get())
+        );
+        checkInterrupt();
+        packBuilderContext.handleException(
+            "writing packfile",
+            git_packbuilder_write_buf(&buf, packBuilder.get())
+        );
+        checkInterrupt();
+
+        std::string repo_path = std::string(git_repository_path(repo.get()));
+        while (!repo_path.empty() && repo_path.back() == '/')
+            repo_path.pop_back();
+        std::string pack_dir_path = repo_path + "/objects/pack";
+
+        // TODO (performance): could the indexing be done in a separate thread?
+        //                     we'd need a more streaming variation of
+        //                     git_packbuilder_write_buf, or incur the cost of
+        //                     copying parts of the buffer to a separate thread.
+        //                     (synchronously on the git_packbuilder_write_buf thread)
+        Indexer indexer;
+        git_indexer_progress stats;
+        if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr))
+            throw Error("creating git packfile indexer: %s", git_error_last()->message);
+
+        // TODO: provide index callback for checkInterrupt() termination
+        //       though this is about an order of magnitude faster than the packbuilder
+        //       expect up to 1 sec latency due to uninterruptible git_indexer_append.
+        constexpr size_t chunkSize = 128 * 1024;
+        for (size_t offset = 0; offset < buf.size; offset += chunkSize) {
+            if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats))
+                throw Error("appending to git packfile index: %s", git_error_last()->message);
+            checkInterrupt();
+        }
+
+        if (git_indexer_commit(indexer.get(), &stats))
+            throw Error("committing git packfile index: %s", git_error_last()->message);
+
+        if (git_mempack_reset(mempack_backend))
+            throw Error("resetting git mempack backend: %s", git_error_last()->message);
+
+        checkInterrupt();
+    }
+
     uint64_t getRevCount(const Hash & rev) override
     {
         std::unordered_set<git_oid> done;
@@ -461,7 +601,13 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
         for (const fetchers::PublicKey & k : publicKeys){
             // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
-            auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
+            std::string keyDecoded;
+            try {
+                keyDecoded = base64Decode(k.key);
+            } catch (Error & e) {
+                e.addTrace({}, "while decoding public key '%s' used for git signature", k.key);
+            }
+            auto fingerprint = trim(hashString(HashAlgorithm::SHA256, keyDecoded).to_string(nix::HashFormat::Base64, false), "=");
             auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
             re += "(" + escaped_fingerprint + ")";
         }
@@ -641,12 +787,16 @@ struct GitSourceAccessor : SourceAccessor
         return readBlob(path, true);
     }
 
-    Hash getSubmoduleRev(const CanonPath & path)
+    /**
+     * If `path` exists and is a submodule, return its
+     * revision. Otherwise return nothing.
+     */
+    std::optional<Hash> getSubmoduleRev(const CanonPath & path)
     {
-        auto entry = need(path);
+        auto entry = lookup(path);
 
-        if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
-            throw Error("'%s' is not a submodule", showPath(path));
+        if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
+            return std::nullopt;
 
         return toHash(*git_tree_entry_id(entry));
     }
@@ -870,8 +1020,24 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
     void pushBuilder(std::string name)
     {
+        const git_tree_entry * entry;
+        Tree prevTree = nullptr;
+
+        if (!pendingDirs.empty() &&
+            (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str())))
+        {
+            /* Clone a tree that we've already finished. This happens
+               if a tarball has directory entries that are not
+               contiguous. */
+            if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+                throw Error("parent of '%s' is not a directory", name);
+
+            if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(prevTree), *repo, entry))
+                throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
+        }
+
         git_treebuilder * b;
-        if (git_treebuilder_new(&b, *repo, nullptr))
+        if (git_treebuilder_new(&b, *repo, prevTree.get()))
             throw Error("creating a tree builder: %s", git_error_last()->message);
         pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) });
     };
@@ -1045,12 +1211,14 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
             git_tree_entry_filemode(entry));
     }
 
-    Hash sync() override
+    Hash flush() override
     {
         updateBuilders({});
 
         auto [oid, _name] = popBuilder();
 
+        repo->flush();
+
         return toHash(oid);
     }
 };
@@ -1123,8 +1291,10 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     auto rawAccessor = getRawAccessor(rev, false);
 
     for (auto & submodule : parseSubmodules(pathTemp)) {
-        auto rev = rawAccessor->getSubmoduleRev(submodule.path);
-        result.push_back({std::move(submodule), rev});
+        /* Filter out .gitmodules entries that don't exist or are not
+           submodules. */
+        if (auto rev = rawAccessor->getSubmoduleRev(submodule.path))
+            result.push_back({std::move(submodule), *rev});
     }
 
     return result;
@@ -1132,7 +1302,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
 
 ref<GitRepo> getTarballCache()
 {
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "nix" / "tarball-cache";
+    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
 
     return GitRepo::openRepo(repoDir, true, true);
 }