Merge pull request #11330 from NixOS/packfile

libgit2, GitRepo: Write (thin) packfiles

src/libfetchers/git-utils.cc

@@ -13,13 +13,17 @@
 #include <git2/describe.h>
 #include <git2/errors.h>
 #include <git2/global.h>
+#include <git2/indexer.h>
 #include <git2/object.h>
+#include <git2/odb.h>
 #include <git2/refs.h>
 #include <git2/remote.h>
 #include <git2/repository.h>
 #include <git2/revparse.h>
 #include <git2/status.h>
 #include <git2/submodule.h>
+#include <git2/sys/odb_backend.h>
+#include <git2/sys/mempack.h>
 #include <git2/tree.h>
 
 #include <iostream>
@@ -76,6 +80,9 @@ typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusLi
 typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
 typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
 typedef std::unique_ptr<git_config_iterator, Deleter<git_config_iterator_free>> ConfigIterator;
+typedef std::unique_ptr<git_odb, Deleter<git_odb_free>> ObjectDb;
+typedef std::unique_ptr<git_packbuilder, Deleter<git_packbuilder_free>> PackBuilder;
+typedef std::unique_ptr<git_indexer, Deleter<git_indexer_free>> Indexer;
 
 // A helper to ensure that we don't leak objects returned by libgit2.
 template<typename T>
@@ -159,11 +166,60 @@ static Object peelToTreeOrBlob(git_object * obj)
         return peelObject<Object>(obj, GIT_OBJECT_TREE);
 }
 
+struct PackBuilderContext {
+    std::exception_ptr exception;
+
+    void handleException(const char * activity, int errCode)
+    {
+        switch (errCode) {
+            case GIT_OK:
+                break;
+            case GIT_EUSER:
+                if (!exception)
+                    panic("PackBuilderContext::handleException: user error, but exception was not set");
+
+                std::rethrow_exception(exception);
+            default:
+                throw Error("%s: %i, %s", Uncolored(activity), errCode, git_error_last()->message);
+        }
+    }
+};
+
+extern "C" {
+
+/**
+ * A `git_packbuilder_progress` implementation that aborts the pack building if needed.
+ */
+static int packBuilderProgressCheckInterrupt(int stage, uint32_t current, uint32_t total, void *payload)
+{
+    PackBuilderContext & args = * (PackBuilderContext *) payload;
+    try {
+        checkInterrupt();
+        return GIT_OK;
+    } catch (const std::exception & e) {
+        args.exception = std::current_exception();
+        return GIT_EUSER;
+    }
+};
+static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuilderProgressCheckInterrupt;
+
+} // extern "C"
+
 struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 {
     /** Location of the repository on disk. */
     std::filesystem::path path;
+    /**
+     * libgit2 repository. Note that new objects are not written to disk,
+     * because we are using a mempack backend. For writing to disk, see
+     * `flush()`, which is also called by `GitFileSystemObjectSink::sync()`.
+     */
     Repository repo;
+    /**
+     * In-memory object store for efficient batched writing to packfiles.
+     * Owned by `repo`.
+     */
+    git_odb_backend * mempack_backend;
 
     GitRepoImpl(std::filesystem::path _path, bool create, bool bare)
         : path(std::move(_path))
@@ -177,6 +233,17 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
             if (git_repository_init(Setter(repo), path.string().c_str(), bare))
                 throw Error("creating Git repository '%s': %s", path, git_error_last()->message);
         }
+
+        ObjectDb odb;
+        if (git_repository_odb(Setter(odb), repo.get()))
+            throw Error("getting Git object database: %s", git_error_last()->message);
+
+        // mempack_backend will be owned by the repository, so we are not expected to free it ourselves.
+        if (git_mempack_new(&mempack_backend))
+            throw Error("creating mempack backend: %s", git_error_last()->message);
+
+        if (git_odb_add_backend(odb.get(), mempack_backend, 999))
+            throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
     }
 
     operator git_repository * ()
@@ -184,6 +251,62 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return repo.get();
     }
 
+    void flush() override {
+        checkInterrupt();
+
+        git_buf buf = GIT_BUF_INIT;
+        Finally _disposeBuf { [&] { git_buf_dispose(&buf); } };
+        PackBuilder packBuilder;
+        PackBuilderContext packBuilderContext;
+        git_packbuilder_new(Setter(packBuilder), *this);
+        git_packbuilder_set_callbacks(packBuilder.get(), PACKBUILDER_PROGRESS_CHECK_INTERRUPT, &packBuilderContext);
+        git_packbuilder_set_threads(packBuilder.get(), 0 /* autodetect */);
+
+        packBuilderContext.handleException(
+            "preparing packfile",
+            git_mempack_write_thin_pack(mempack_backend, packBuilder.get())
+        );
+        checkInterrupt();
+        packBuilderContext.handleException(
+            "writing packfile",
+            git_packbuilder_write_buf(&buf, packBuilder.get())
+        );
+        checkInterrupt();
+
+        std::string repo_path = std::string(git_repository_path(repo.get()));
+        while (!repo_path.empty() && repo_path.back() == '/')
+            repo_path.pop_back();
+        std::string pack_dir_path = repo_path + "/objects/pack";
+
+        // TODO (performance): could the indexing be done in a separate thread?
+        //                     we'd need a more streaming variation of
+        //                     git_packbuilder_write_buf, or incur the cost of
+        //                     copying parts of the buffer to a separate thread.
+        //                     (synchronously on the git_packbuilder_write_buf thread)
+        Indexer indexer;
+        git_indexer_progress stats;
+        if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr))
+            throw Error("creating git packfile indexer: %s", git_error_last()->message);
+
+        // TODO: provide index callback for checkInterrupt() termination
+        //       though this is about an order of magnitude faster than the packbuilder
+        //       expect up to 1 sec latency due to uninterruptible git_indexer_append.
+        constexpr size_t chunkSize = 128 * 1024;
+        for (size_t offset = 0; offset < buf.size; offset += chunkSize) {
+            if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats))
+                throw Error("appending to git packfile index: %s", git_error_last()->message);
+            checkInterrupt();
+        }
+
+        if (git_indexer_commit(indexer.get(), &stats))
+            throw Error("committing git packfile index: %s", git_error_last()->message);
+
+        if (git_mempack_reset(mempack_backend))
+            throw Error("resetting git mempack backend: %s", git_error_last()->message);
+
+        checkInterrupt();
+    }
+
     uint64_t getRevCount(const Hash & rev) override
     {
         std::unordered_set<git_oid> done;
@@ -1006,12 +1129,14 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
             git_tree_entry_filemode(entry));
     }
 
-    Hash sync() override
+    Hash flush() override
     {
         updateBuilders({});
 
         auto [oid, _name] = popBuilder();
 
+        repo->flush();
+
         return toHash(oid);
     }
 };