From dc3ccf02bfd4d359228b54f5c24ae2b6caf6428e Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Mon, 31 Jul 2023 18:40:45 +0100 Subject: [PATCH 01/37] base64Decode: clearer error message when an invalid character is detected Output the offending string in its entirety to provide context. Closes #8479 --- src/libutil/util.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 26f9dc8a8..952015b0c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1618,8 +1618,9 @@ std::string base64Decode(std::string_view s) if (c == '\n') continue; char digit = base64DecodeChars[(unsigned char) c]; - if (digit == npos) - throw Error("invalid character in Base64 string: '%c'", c); + if (digit == npos) { + throw Error("invalid character in Base64 string: '%c' in '%s'", c, s.data()); + } bits += 6; d = d << 6 | digit; From 5dd6c4f062ec392b965a5fcfc18fd3f75e2a79cb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 19 Aug 2024 13:21:44 +0200 Subject: [PATCH 02/37] libgit2, GitRepo: Write thin packfiles libgit2 didn't write thin ones, hence the patch. This should improve performance on systems with weak I/O in ~/.cache, especially in terms of operations per second, or where system calls are slower. (macOS, VMs?) --- packaging/dependencies.nix | 2 + .../libgit2-mempack-thin-packfile.patch | 282 ++++++++++++++++++ src/libfetchers/git-utils.cc | 57 ++++ src/libfetchers/git-utils.hh | 2 + 4 files changed, 343 insertions(+) create mode 100644 packaging/patches/libgit2-mempack-thin-packfile.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 21c48e5cc..74b5cbc05 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -115,6 +115,8 @@ scope: { version = inputs.libgit2.lastModifiedDate; cmakeFlags = attrs.cmakeFlags or [] ++ [ "-DUSE_SSH=exec" ]; + patches = attrs.patches or [] + ++ [ ./patches/libgit2-mempack-thin-packfile.patch ]; }); busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { diff --git a/packaging/patches/libgit2-mempack-thin-packfile.patch b/packaging/patches/libgit2-mempack-thin-packfile.patch new file mode 100644 index 000000000..fb74b1683 --- /dev/null +++ b/packaging/patches/libgit2-mempack-thin-packfile.patch @@ -0,0 +1,282 @@ +commit 9bacade4a3ef4b6b26e2c02f549eef0e9eb9eaa2 +Author: Robert Hensing +Date: Sun Aug 18 20:20:36 2024 +0200 + + Add unoptimized git_mempack_write_thin_pack + +diff --git a/include/git2/sys/mempack.h b/include/git2/sys/mempack.h +index 17da590a3..3688bdd50 100644 +--- a/include/git2/sys/mempack.h ++++ b/include/git2/sys/mempack.h +@@ -44,6 +44,29 @@ GIT_BEGIN_DECL + */ + GIT_EXTERN(int) git_mempack_new(git_odb_backend **out); + ++/** ++ * Write a thin packfile with the objects in the memory store. ++ * ++ * A thin packfile is a packfile that does not contain its transitive closure of ++ * references. This is useful for efficiently distributing additions to a ++ * repository over the network, but also finds use in the efficient bulk ++ * addition of objects to a repository, locally. ++ * ++ * This operation performs the (shallow) insert operations into the ++ * `git_packbuilder`, but does not write the packfile to disk; ++ * see `git_packbuilder_write_buf`. ++ * ++ * It also does not reset the memory store; see `git_mempack_reset`. ++ * ++ * @note This function may or may not write trees and blobs that are not ++ * referenced by commits. Currently everything is written, but this ++ * behavior may change in the future as the packer is optimized. ++ * ++ * @param backend The mempack backend ++ * @param pb The packbuilder to use to write the packfile ++ */ ++GIT_EXTERN(int) git_mempack_write_thin_pack(git_odb_backend *backend, git_packbuilder *pb); ++ + /** + * Dump all the queued in-memory writes to a packfile. + * +diff --git a/src/libgit2/odb_mempack.c b/src/libgit2/odb_mempack.c +index 6f27f45f8..0b61e2b66 100644 +--- a/src/libgit2/odb_mempack.c ++++ b/src/libgit2/odb_mempack.c +@@ -132,6 +132,35 @@ cleanup: + return err; + } + ++int git_mempack_write_thin_pack(git_odb_backend *backend, git_packbuilder *pb) ++{ ++ struct memory_packer_db *db = (struct memory_packer_db *)backend; ++ const git_oid *oid; ++ size_t iter = 0; ++ int err = -1; ++ ++ /* TODO: Implement the recency heuristics. ++ For this it probably makes sense to only write what's referenced ++ through commits, an option I've carved out for you in the docs. ++ wrt heuristics: ask your favorite LLM to translate https://git-scm.com/docs/pack-heuristics/en ++ to actual normal reference documentation. */ ++ while (true) { ++ err = git_oidmap_iterate(NULL, db->objects, &iter, &oid); ++ if (err == GIT_ITEROVER) { ++ err = 0; ++ break; ++ } ++ if (err != 0) ++ return err; ++ ++ err = git_packbuilder_insert(pb, oid, NULL); ++ if (err != 0) ++ return err; ++ } ++ ++ return 0; ++} ++ + int git_mempack_dump( + git_buf *pack, + git_repository *repo, +diff --git a/tests/libgit2/mempack/thinpack.c b/tests/libgit2/mempack/thinpack.c +new file mode 100644 +index 000000000..604a4dda2 +--- /dev/null ++++ b/tests/libgit2/mempack/thinpack.c +@@ -0,0 +1,196 @@ ++#include "clar_libgit2.h" ++#include "git2/indexer.h" ++#include "git2/odb_backend.h" ++#include "git2/tree.h" ++#include "git2/types.h" ++#include "git2/sys/mempack.h" ++#include "git2/sys/odb_backend.h" ++#include "util.h" ++ ++static git_repository *_repo; ++static git_odb_backend * _mempack_backend; ++ ++void test_mempack_thinpack__initialize(void) ++{ ++ git_odb *odb; ++ ++ _repo = cl_git_sandbox_init_new("mempack_thinpack_repo"); ++ ++ cl_git_pass(git_mempack_new(&_mempack_backend)); ++ cl_git_pass(git_repository_odb(&odb, _repo)); ++ cl_git_pass(git_odb_add_backend(odb, _mempack_backend, 999)); ++ git_odb_free(odb); ++} ++ ++void _mempack_thinpack__cleanup(void) ++{ ++ cl_git_sandbox_cleanup(); ++} ++ ++/* ++ Generating a packfile for an unchanged repo works and produces an empty packfile. ++ Even if we allow this scenario to be detected, it shouldn't misbehave if the ++ application is unaware of it. ++*/ ++void test_mempack_thinpack__empty(void) ++{ ++ git_packbuilder *pb; ++ int version; ++ int n; ++ git_buf buf = GIT_BUF_INIT; ++ ++ git_packbuilder_new(&pb, _repo); ++ ++ cl_git_pass(git_mempack_write_thin_pack(_mempack_backend, pb)); ++ cl_git_pass(git_packbuilder_write_buf(&buf, pb)); ++ cl_assert_in_range(12, buf.size, 1024 /* empty packfile is >0 bytes, but certainly not that big */); ++ cl_assert(buf.ptr[0] == 'P'); ++ cl_assert(buf.ptr[1] == 'A'); ++ cl_assert(buf.ptr[2] == 'C'); ++ cl_assert(buf.ptr[3] == 'K'); ++ version = (buf.ptr[4] << 24) | (buf.ptr[5] << 16) | (buf.ptr[6] << 8) | buf.ptr[7]; ++ /* Subject to change. https://git-scm.com/docs/pack-format: Git currently accepts version number 2 or 3 but generates version 2 only.*/ ++ cl_assert_equal_i(2, version); ++ n = (buf.ptr[8] << 24) | (buf.ptr[9] << 16) | (buf.ptr[10] << 8) | buf.ptr[11]; ++ cl_assert_equal_i(0, n); ++ git_buf_dispose(&buf); ++ ++ git_packbuilder_free(pb); ++} ++ ++#define LIT_LEN(x) x, sizeof(x) - 1 ++ ++/* ++ Check that git_mempack_write_thin_pack produces a thin packfile. ++*/ ++void test_mempack_thinpack__thin(void) ++{ ++ /* Outline: ++ - Create tree 1 ++ - Flush to packfile A ++ - Create tree 2 ++ - Flush to packfile B ++ ++ Tree 2 has a new blob and a reference to a blob from tree 1. ++ ++ Expectation: ++ - Packfile B is thin and does not contain the objects from packfile A ++ */ ++ ++ ++ git_oid oid_blob_1; ++ git_oid oid_blob_2; ++ git_oid oid_blob_3; ++ git_oid oid_tree_1; ++ git_oid oid_tree_2; ++ git_treebuilder *tb; ++ ++ git_packbuilder *pb; ++ git_buf buf = GIT_BUF_INIT; ++ git_indexer *indexer; ++ git_indexer_progress stats; ++ char pack_dir_path[1024]; ++ ++ char sbuf[1024]; ++ const char * repo_path; ++ const char * pack_name_1; ++ const char * pack_name_2; ++ git_str pack_path_1 = GIT_STR_INIT; ++ git_str pack_path_2 = GIT_STR_INIT; ++ git_odb_backend * pack_odb_backend_1; ++ git_odb_backend * pack_odb_backend_2; ++ ++ ++ cl_assert_in_range(0, snprintf(pack_dir_path, sizeof(pack_dir_path), "%s/objects/pack", git_repository_path(_repo)), sizeof(pack_dir_path)); ++ ++ /* Create tree 1 */ ++ ++ cl_git_pass(git_blob_create_from_buffer(&oid_blob_1, _repo, LIT_LEN("thinpack blob 1"))); ++ cl_git_pass(git_blob_create_from_buffer(&oid_blob_2, _repo, LIT_LEN("thinpack blob 2"))); ++ ++ ++ cl_git_pass(git_treebuilder_new(&tb, _repo, NULL)); ++ cl_git_pass(git_treebuilder_insert(NULL, tb, "blob1", &oid_blob_1, GIT_FILEMODE_BLOB)); ++ cl_git_pass(git_treebuilder_insert(NULL, tb, "blob2", &oid_blob_2, GIT_FILEMODE_BLOB)); ++ cl_git_pass(git_treebuilder_write(&oid_tree_1, tb)); ++ ++ /* Flush */ ++ ++ cl_git_pass(git_packbuilder_new(&pb, _repo)); ++ cl_git_pass(git_mempack_write_thin_pack(_mempack_backend, pb)); ++ cl_git_pass(git_packbuilder_write_buf(&buf, pb)); ++ cl_git_pass(git_indexer_new(&indexer, pack_dir_path, 0, NULL, NULL)); ++ cl_git_pass(git_indexer_append(indexer, buf.ptr, buf.size, &stats)); ++ cl_git_pass(git_indexer_commit(indexer, &stats)); ++ pack_name_1 = strdup(git_indexer_name(indexer)); ++ cl_assert(pack_name_1); ++ git_buf_dispose(&buf); ++ git_mempack_reset(_mempack_backend); ++ git_indexer_free(indexer); ++ git_packbuilder_free(pb); ++ ++ /* Create tree 2 */ ++ ++ cl_git_pass(git_treebuilder_clear(tb)); ++ /* blob 1 won't be used, but we add it anyway to test that just "declaring" an object doesn't ++ necessarily cause its inclusion in the next thin packfile. It must only be included if new. */ ++ cl_git_pass(git_blob_create_from_buffer(&oid_blob_1, _repo, LIT_LEN("thinpack blob 1"))); ++ cl_git_pass(git_blob_create_from_buffer(&oid_blob_3, _repo, LIT_LEN("thinpack blob 3"))); ++ cl_git_pass(git_treebuilder_insert(NULL, tb, "blob1", &oid_blob_1, GIT_FILEMODE_BLOB)); ++ cl_git_pass(git_treebuilder_insert(NULL, tb, "blob3", &oid_blob_3, GIT_FILEMODE_BLOB)); ++ cl_git_pass(git_treebuilder_write(&oid_tree_2, tb)); ++ ++ /* Flush */ ++ ++ cl_git_pass(git_packbuilder_new(&pb, _repo)); ++ cl_git_pass(git_mempack_write_thin_pack(_mempack_backend, pb)); ++ cl_git_pass(git_packbuilder_write_buf(&buf, pb)); ++ cl_git_pass(git_indexer_new(&indexer, pack_dir_path, 0, NULL, NULL)); ++ cl_git_pass(git_indexer_append(indexer, buf.ptr, buf.size, &stats)); ++ cl_git_pass(git_indexer_commit(indexer, &stats)); ++ pack_name_2 = strdup(git_indexer_name(indexer)); ++ cl_assert(pack_name_2); ++ git_buf_dispose(&buf); ++ git_mempack_reset(_mempack_backend); ++ git_indexer_free(indexer); ++ git_packbuilder_free(pb); ++ git_treebuilder_free(tb); ++ ++ /* Assertions */ ++ ++ assert(pack_name_1); ++ assert(pack_name_2); ++ ++ repo_path = git_repository_path(_repo); ++ ++ snprintf(sbuf, sizeof(sbuf), "objects/pack/pack-%s.pack", pack_name_1); ++ git_str_joinpath(&pack_path_1, repo_path, sbuf); ++ snprintf(sbuf, sizeof(sbuf), "objects/pack/pack-%s.pack", pack_name_2); ++ git_str_joinpath(&pack_path_2, repo_path, sbuf); ++ ++ /* If they're the same, something definitely went wrong. */ ++ cl_assert(strcmp(pack_name_1, pack_name_2) != 0); ++ ++ cl_git_pass(git_odb_backend_one_pack(&pack_odb_backend_1, pack_path_1.ptr)); ++ cl_assert(pack_odb_backend_1->exists(pack_odb_backend_1, &oid_blob_1)); ++ cl_assert(pack_odb_backend_1->exists(pack_odb_backend_1, &oid_blob_2)); ++ cl_assert(!pack_odb_backend_1->exists(pack_odb_backend_1, &oid_blob_3)); ++ cl_assert(pack_odb_backend_1->exists(pack_odb_backend_1, &oid_tree_1)); ++ cl_assert(!pack_odb_backend_1->exists(pack_odb_backend_1, &oid_tree_2)); ++ ++ cl_git_pass(git_odb_backend_one_pack(&pack_odb_backend_2, pack_path_2.ptr)); ++ /* blob 1 is already in the packfile 1, so packfile 2 must not include it, in order to be _thin_. */ ++ cl_assert(!pack_odb_backend_2->exists(pack_odb_backend_2, &oid_blob_1)); ++ cl_assert(!pack_odb_backend_2->exists(pack_odb_backend_2, &oid_blob_2)); ++ cl_assert(pack_odb_backend_2->exists(pack_odb_backend_2, &oid_blob_3)); ++ cl_assert(!pack_odb_backend_2->exists(pack_odb_backend_2, &oid_tree_1)); ++ cl_assert(pack_odb_backend_2->exists(pack_odb_backend_2, &oid_tree_2)); ++ ++ pack_odb_backend_1->free(pack_odb_backend_1); ++ pack_odb_backend_2->free(pack_odb_backend_2); ++ free((void *)pack_name_1); ++ free((void *)pack_name_2); ++ git_str_dispose(&pack_path_1); ++ git_str_dispose(&pack_path_2); ++ ++} diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 114aa4ec0..5306d8780 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -13,13 +13,17 @@ #include #include #include +#include #include +#include #include #include #include #include #include #include +#include +#include #include #include @@ -76,6 +80,9 @@ typedef std::unique_ptr> StatusLi typedef std::unique_ptr> Remote; typedef std::unique_ptr> GitConfig; typedef std::unique_ptr> ConfigIterator; +typedef std::unique_ptr> ObjectDb; +typedef std::unique_ptr> PackBuilder; +typedef std::unique_ptr> Indexer; // A helper to ensure that we don't leak objects returned by libgit2. template @@ -164,6 +171,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this /** Location of the repository on disk. */ std::filesystem::path path; Repository repo; + /** + * In-memory object store for efficient batched writing to packfiles. + * Owned by `repo`. + */ + git_odb_backend * mempack_backend; GitRepoImpl(std::filesystem::path _path, bool create, bool bare) : path(std::move(_path)) @@ -177,6 +189,16 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_repository_init(Setter(repo), path.string().c_str(), bare)) throw Error("creating Git repository '%s': %s", path, git_error_last()->message); } + git_odb * odb; + if (git_repository_odb(&odb, repo.get())) + throw Error("getting Git object database: %s", git_error_last()->message); + + // TODO: release mempack_backend? + if (git_mempack_new(&mempack_backend)) + throw Error("creating mempack backend: %s", git_error_last()->message); + + if (git_odb_add_backend(odb, mempack_backend, 999)) + throw Error("adding mempack backend to Git object database: %s", git_error_last()->message); } operator git_repository * () @@ -184,6 +206,39 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return repo.get(); } + void flush() override { + git_buf buf = GIT_BUF_INIT; + try { + PackBuilder packBuilder; + git_packbuilder_new(Setter(packBuilder), *this); + git_mempack_write_thin_pack(mempack_backend, packBuilder.get()); + git_packbuilder_write_buf(&buf, packBuilder.get()); + + std::string repo_path = std::string(git_repository_path(repo.get())); + while (!repo_path.empty() && repo_path.back() == '/') + repo_path.pop_back(); + std::string pack_dir_path = repo_path + "/objects/pack"; + + // TODO: could the indexing be done in a separate thread? + Indexer indexer; + git_indexer_progress stats; + if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) + throw Error("creating git packfile indexer: %s", git_error_last()->message); + if (git_indexer_append(indexer.get(), buf.ptr, buf.size, &stats)) + throw Error("appending to git packfile index: %s", git_error_last()->message); + if (git_indexer_commit(indexer.get(), &stats)) + throw Error("committing git packfile index: %s", git_error_last()->message); + + if (git_mempack_reset(mempack_backend)) + throw Error("resetting git mempack backend: %s", git_error_last()->message); + + git_buf_dispose(&buf); + } catch (...) { + git_buf_dispose(&buf); + throw; + } + } + uint64_t getRevCount(const Hash & rev) override { std::unordered_set done; @@ -1008,6 +1063,8 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink auto [oid, _name] = popBuilder(); + repo->flush(); + return toHash(oid); } }; diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 915252868..65a598ce5 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -80,6 +80,8 @@ struct GitRepo virtual ref getFileSystemObjectSink() = 0; + virtual void flush() = 0; + virtual void fetch( const std::string & url, const std::string & refspec, From d0f8a9236392ab41414dddbe7400da98fad9f62d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 26 Aug 2024 11:38:40 +0200 Subject: [PATCH 03/37] Make tarball cache more interruptible --- src/libfetchers/git-utils.cc | 7 +++++++ src/libutil/unix/signals-impl.hh | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 5306d8780..fb5341599 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -211,8 +211,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this try { PackBuilder packBuilder; git_packbuilder_new(Setter(packBuilder), *this); + checkInterrupt(); git_mempack_write_thin_pack(mempack_backend, packBuilder.get()); + checkInterrupt(); + // TODO make git_packbuilder_write_buf() interruptible git_packbuilder_write_buf(&buf, packBuilder.get()); + checkInterrupt(); std::string repo_path = std::string(git_repository_path(repo.get())); while (!repo_path.empty() && repo_path.back() == '/') @@ -224,8 +228,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this git_indexer_progress stats; if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) throw Error("creating git packfile indexer: %s", git_error_last()->message); + // TODO: feed buf in (fairly large) chunk to make this interruptible if (git_indexer_append(indexer.get(), buf.ptr, buf.size, &stats)) throw Error("appending to git packfile index: %s", git_error_last()->message); + checkInterrupt(); if (git_indexer_commit(indexer.get(), &stats)) throw Error("committing git packfile index: %s", git_error_last()->message); @@ -237,6 +243,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this git_buf_dispose(&buf); throw; } + checkInterrupt(); } uint64_t getRevCount(const Hash & rev) override diff --git a/src/libutil/unix/signals-impl.hh b/src/libutil/unix/signals-impl.hh index 7ac8c914d..2193922be 100644 --- a/src/libutil/unix/signals-impl.hh +++ b/src/libutil/unix/signals-impl.hh @@ -84,6 +84,12 @@ static inline bool getInterrupted() return unix::_isInterrupted; } +/** + * Throw `Interrupted` exception if the process has been interrupted. + * + * Call this in long-running loops and between slow operations to terminate + * them as needed. + */ void inline checkInterrupt() { using namespace unix; From 97ff2ed4555ce0761368046270fc4466550bfb0c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 26 Aug 2024 11:39:14 +0200 Subject: [PATCH 04/37] Sync tarball cache within tarball cache Activity --- src/libfetchers/github.cc | 3 ++- src/libfetchers/tarball.cc | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2e914164a..ecfd035fc 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -261,11 +261,12 @@ struct GitArchiveInputScheme : InputScheme auto tarballCache = getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + auto tree = parseSink->sync(); act.reset(); TarballInfo tarballInfo { - .treeHash = tarballCache->dereferenceSingletonDirectory(parseSink->sync()), + .treeHash = tarballCache->dereferenceSingletonDirectory(tree), .lastModified = lastModified }; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index dd4f3b780..a082b0078 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -170,6 +170,7 @@ static DownloadTarballResult downloadTarball_( auto tarballCache = getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + auto tree = parseSink->sync(); act.reset(); @@ -184,7 +185,7 @@ static DownloadTarballResult downloadTarball_( } else { infoAttrs.insert_or_assign("etag", res->etag); infoAttrs.insert_or_assign("treeHash", - tarballCache->dereferenceSingletonDirectory(parseSink->sync()).gitRev()); + tarballCache->dereferenceSingletonDirectory(tree).gitRev()); infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); if (res->immutableUrl) infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); From fb8d3ed1506d2c0e3065abaad9587e16dccfd1b9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 26 Aug 2024 15:04:47 +0200 Subject: [PATCH 05/37] fixup: sync -> flush The latter is not used for memory synchronization things. --- src/libfetchers/git-utils.cc | 7 ++++++- src/libfetchers/git-utils.hh | 6 +++++- src/libfetchers/github.cc | 2 +- src/libfetchers/tarball.cc | 2 +- tests/unit/libfetchers/git-utils.cc | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index fb5341599..59b4d424a 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -170,6 +170,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ std::filesystem::path path; + /** + * libgit2 repository. Note that new objects are not written to disk, + * because we are using a mempack backend. For writing to disk, see + * `flush()`, which is also called by `GitFileSystemObjectSink::sync()`. + */ Repository repo; /** * In-memory object store for efficient batched writing to packfiles. @@ -1064,7 +1069,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink git_tree_entry_filemode(entry)); } - Hash sync() override + Hash flush() override { updateBuilders({}); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 65a598ce5..f45b5a504 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -7,12 +7,16 @@ namespace nix { namespace fetchers { struct PublicKey; } +/** + * A sink that writes into a Git repository. Note that nothing may be written + * until `flush()` is called. + */ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink { /** * Flush builder and return a final Git hash. */ - virtual Hash sync() = 0; + virtual Hash flush() = 0; }; struct GitRepo diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ecfd035fc..308cff33a 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -261,7 +261,7 @@ struct GitArchiveInputScheme : InputScheme auto tarballCache = getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); - auto tree = parseSink->sync(); + auto tree = parseSink->flush(); act.reset(); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index a082b0078..aa5d61bc5 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -170,7 +170,7 @@ static DownloadTarballResult downloadTarball_( auto tarballCache = getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); - auto tree = parseSink->sync(); + auto tree = parseSink->flush(); act.reset(); diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index de5110cc3..0bf3076dc 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -77,7 +77,7 @@ TEST_F(GitUtilsTest, sink_basic) // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); - auto result = repo->dereferenceSingletonDirectory(sink->sync()); + auto result = repo->dereferenceSingletonDirectory(sink->flush()); auto accessor = repo->getAccessor(result, false); auto entries = accessor->readDirectory(CanonPath::root); ASSERT_EQ(entries.size(), 5); From 57c48304bba7d1f92fb9acc8abc9e8db4450fc0c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 26 Aug 2024 15:34:35 +0200 Subject: [PATCH 06/37] fixup: Release odb --- src/libfetchers/git-utils.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 59b4d424a..70391a287 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -194,15 +194,16 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_repository_init(Setter(repo), path.string().c_str(), bare)) throw Error("creating Git repository '%s': %s", path, git_error_last()->message); } - git_odb * odb; - if (git_repository_odb(&odb, repo.get())) + + ObjectDb odb; + if (git_repository_odb(Setter(odb), repo.get())) throw Error("getting Git object database: %s", git_error_last()->message); - // TODO: release mempack_backend? + // mempack_backend will be owned by the repository, so we are not expected to free it ourselves. if (git_mempack_new(&mempack_backend)) throw Error("creating mempack backend: %s", git_error_last()->message); - if (git_odb_add_backend(odb, mempack_backend, 999)) + if (git_odb_add_backend(odb.get(), mempack_backend, 999)) throw Error("adding mempack backend to Git object database: %s", git_error_last()->message); } From c1fe3546ed67babeecfb376315063f0642ce4bbd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 28 Aug 2024 01:48:25 +0200 Subject: [PATCH 07/37] libgit2: Add libgit2-packbuilder-callback-interruptible.patch --- packaging/dependencies.nix | 18 +- ...2-packbuilder-callback-interruptible.patch | 930 ++++++++++++++++++ 2 files changed, 947 insertions(+), 1 deletion(-) create mode 100644 packaging/patches/libgit2-packbuilder-callback-interruptible.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 74b5cbc05..0182f29c0 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -115,8 +115,24 @@ scope: { version = inputs.libgit2.lastModifiedDate; cmakeFlags = attrs.cmakeFlags or [] ++ [ "-DUSE_SSH=exec" ]; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ [ + # Needed for `git apply`; see `prePatch` + pkgs.buildPackages.gitMinimal + ]; + # Only `git apply` can handle git binary patches + prePatch = '' + patch() { + git apply + } + ''; patches = attrs.patches or [] - ++ [ ./patches/libgit2-mempack-thin-packfile.patch ]; + ++ [ + ./patches/libgit2-mempack-thin-packfile.patch + + # binary patch; see `prePatch` + ./patches/libgit2-packbuilder-callback-interruptible.patch + ]; }); busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { diff --git a/packaging/patches/libgit2-packbuilder-callback-interruptible.patch b/packaging/patches/libgit2-packbuilder-callback-interruptible.patch new file mode 100644 index 000000000..c67822ff7 --- /dev/null +++ b/packaging/patches/libgit2-packbuilder-callback-interruptible.patch @@ -0,0 +1,930 @@ +commit e9823c5da4fa977c46bcb97167fbdd0d70adb5ff +Author: Robert Hensing +Date: Mon Aug 26 20:07:04 2024 +0200 + + Make packbuilder interruptible using progress callback + + Forward errors from packbuilder->progress_cb + + This allows the callback to terminate long-running operations when + the application is interrupted. + +diff --git a/include/git2/pack.h b/include/git2/pack.h +index 0f6bd2ab9..bee72a6c0 100644 +--- a/include/git2/pack.h ++++ b/include/git2/pack.h +@@ -247,6 +247,9 @@ typedef int GIT_CALLBACK(git_packbuilder_progress)( + * @param progress_cb Function to call with progress information during + * pack building. Be aware that this is called inline with pack building + * operations, so performance may be affected. ++ * When progress_cb returns an error, the pack building process will be ++ * aborted and the error will be returned from the invoked function. ++ * `pb` must then be freed. + * @param progress_cb_payload Payload for progress callback. + * @return 0 or an error code + */ +diff --git a/src/libgit2/pack-objects.c b/src/libgit2/pack-objects.c +index b2d80cba9..7c331c2d5 100644 +--- a/src/libgit2/pack-objects.c ++++ b/src/libgit2/pack-objects.c +@@ -932,6 +932,9 @@ static int report_delta_progress( + { + int ret; + ++ if (pb->failure) ++ return pb->failure; ++ + if (pb->progress_cb) { + uint64_t current_time = git_time_monotonic(); + uint64_t elapsed = current_time - pb->last_progress_report_time; +@@ -943,8 +946,10 @@ static int report_delta_progress( + GIT_PACKBUILDER_DELTAFICATION, + count, pb->nr_objects, pb->progress_cb_payload); + +- if (ret) ++ if (ret) { ++ pb->failure = ret; + return git_error_set_after_callback(ret); ++ } + } + } + +@@ -976,7 +981,10 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list, + } + + pb->nr_deltified += 1; +- report_delta_progress(pb, pb->nr_deltified, false); ++ if ((error = report_delta_progress(pb, pb->nr_deltified, false)) < 0) { ++ GIT_ASSERT(git_packbuilder__progress_unlock(pb) == 0); ++ goto on_error; ++ } + + po = *list++; + (*list_size)--; +@@ -1124,6 +1132,10 @@ struct thread_params { + size_t depth; + size_t working; + size_t data_ready; ++ ++ /* A pb->progress_cb can stop the packing process by returning an error. ++ When that happens, all threads observe the error and stop voluntarily. */ ++ bool stopped; + }; + + static void *threaded_find_deltas(void *arg) +@@ -1133,7 +1145,12 @@ static void *threaded_find_deltas(void *arg) + while (me->remaining) { + if (find_deltas(me->pb, me->list, &me->remaining, + me->window, me->depth) < 0) { +- ; /* TODO */ ++ me->stopped = true; ++ GIT_ASSERT_WITH_RETVAL(git_packbuilder__progress_lock(me->pb) == 0, NULL); ++ me->working = false; ++ git_cond_signal(&me->pb->progress_cond); ++ GIT_ASSERT_WITH_RETVAL(git_packbuilder__progress_unlock(me->pb) == 0, NULL); ++ return NULL; + } + + GIT_ASSERT_WITH_RETVAL(git_packbuilder__progress_lock(me->pb) == 0, NULL); +@@ -1175,8 +1192,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + pb->nr_threads = git__online_cpus(); + + if (pb->nr_threads <= 1) { +- find_deltas(pb, list, &list_size, window, depth); +- return 0; ++ return find_deltas(pb, list, &list_size, window, depth); + } + + p = git__mallocarray(pb->nr_threads, sizeof(*p)); +@@ -1195,6 +1211,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + p[i].depth = depth; + p[i].working = 1; + p[i].data_ready = 0; ++ p[i].stopped = 0; + + /* try to split chunks on "path" boundaries */ + while (sub_size && sub_size < list_size && +@@ -1262,7 +1279,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + (!victim || victim->remaining < p[i].remaining)) + victim = &p[i]; + +- if (victim) { ++ if (victim && !target->stopped) { + sub_size = victim->remaining / 2; + list = victim->list + victim->list_size - sub_size; + while (sub_size && list[0]->hash && +@@ -1286,7 +1303,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + } + target->list_size = sub_size; + target->remaining = sub_size; +- target->working = 1; ++ target->working = 1; /* even when target->stopped, so that we don't process this thread again */ + GIT_ASSERT(git_packbuilder__progress_unlock(pb) == 0); + + if (git_mutex_lock(&target->mutex)) { +@@ -1299,7 +1316,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + git_cond_signal(&target->cond); + git_mutex_unlock(&target->mutex); + +- if (!sub_size) { ++ if (target->stopped || !sub_size) { + git_thread_join(&target->thread, NULL); + git_cond_free(&target->cond); + git_mutex_free(&target->mutex); +@@ -1308,7 +1325,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + } + + git__free(p); +- return 0; ++ return pb->failure; + } + + #else +@@ -1319,6 +1336,7 @@ int git_packbuilder__prepare(git_packbuilder *pb) + { + git_pobject **delta_list; + size_t i, n = 0; ++ int error; + + if (pb->nr_objects == 0 || pb->done) + return 0; /* nothing to do */ +@@ -1327,8 +1345,10 @@ int git_packbuilder__prepare(git_packbuilder *pb) + * Although we do not report progress during deltafication, we + * at least report that we are in the deltafication stage + */ +- if (pb->progress_cb) +- pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload); ++ if (pb->progress_cb) { ++ if ((error = pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload)) < 0) ++ return git_error_set_after_callback(error); ++ } + + delta_list = git__mallocarray(pb->nr_objects, sizeof(*delta_list)); + GIT_ERROR_CHECK_ALLOC(delta_list); +@@ -1345,31 +1365,33 @@ int git_packbuilder__prepare(git_packbuilder *pb) + + if (n > 1) { + git__tsort((void **)delta_list, n, type_size_sort); +- if (ll_find_deltas(pb, delta_list, n, ++ if ((error = ll_find_deltas(pb, delta_list, n, + GIT_PACK_WINDOW + 1, +- GIT_PACK_DEPTH) < 0) { ++ GIT_PACK_DEPTH)) < 0) { + git__free(delta_list); +- return -1; ++ return error; + } + } + +- report_delta_progress(pb, pb->nr_objects, true); ++ error = report_delta_progress(pb, pb->nr_objects, true); + + pb->done = true; + git__free(delta_list); +- return 0; ++ return error; + } + +-#define PREPARE_PACK if (git_packbuilder__prepare(pb) < 0) { return -1; } ++#define PREPARE_PACK error = git_packbuilder__prepare(pb); if (error < 0) { return error; } + + int git_packbuilder_foreach(git_packbuilder *pb, int (*cb)(void *buf, size_t size, void *payload), void *payload) + { ++ int error; + PREPARE_PACK; + return write_pack(pb, cb, payload); + } + + int git_packbuilder__write_buf(git_str *buf, git_packbuilder *pb) + { ++ int error; + PREPARE_PACK; + + return write_pack(pb, &write_pack_buf, buf); +diff --git a/src/libgit2/pack-objects.h b/src/libgit2/pack-objects.h +index bbc8b9430..380a28ebe 100644 +--- a/src/libgit2/pack-objects.h ++++ b/src/libgit2/pack-objects.h +@@ -100,6 +100,10 @@ struct git_packbuilder { + uint64_t last_progress_report_time; + + bool done; ++ ++ /* A non-zero error code in failure causes all threads to shut themselves ++ down. Some functions will return this error code. */ ++ volatile int failure; + }; + + int git_packbuilder__write_buf(git_str *buf, git_packbuilder *pb); +diff --git a/tests/libgit2/pack/cancel.c b/tests/libgit2/pack/cancel.c +new file mode 100644 +index 000000000..a0aa9716a +--- /dev/null ++++ b/tests/libgit2/pack/cancel.c +@@ -0,0 +1,240 @@ ++#include "clar_libgit2.h" ++#include "futils.h" ++#include "pack.h" ++#include "hash.h" ++#include "iterator.h" ++#include "vector.h" ++#include "posix.h" ++#include "hash.h" ++#include "pack-objects.h" ++ ++static git_repository *_repo; ++static git_revwalk *_revwalker; ++static git_packbuilder *_packbuilder; ++static git_indexer *_indexer; ++static git_vector _commits; ++static int _commits_is_initialized; ++static git_indexer_progress _stats; ++ ++extern bool git_disable_pack_keep_file_checks; ++ ++static void pack_packbuilder_init(const char *sandbox) { ++ _repo = cl_git_sandbox_init(sandbox); ++ /* cl_git_pass(p_chdir(sandbox)); */ ++ cl_git_pass(git_revwalk_new(&_revwalker, _repo)); ++ cl_git_pass(git_packbuilder_new(&_packbuilder, _repo)); ++ cl_git_pass(git_vector_init(&_commits, 0, NULL)); ++ _commits_is_initialized = 1; ++ memset(&_stats, 0, sizeof(_stats)); ++ p_fsync__cnt = 0; ++} ++ ++void test_pack_cancel__initialize(void) ++{ ++ pack_packbuilder_init("small.git"); ++} ++ ++void test_pack_cancel__cleanup(void) ++{ ++ git_oid *o; ++ unsigned int i; ++ ++ cl_git_pass(git_libgit2_opts(GIT_OPT_ENABLE_FSYNC_GITDIR, 0)); ++ cl_git_pass(git_libgit2_opts(GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, false)); ++ ++ if (_commits_is_initialized) { ++ _commits_is_initialized = 0; ++ git_vector_foreach(&_commits, i, o) { ++ git__free(o); ++ } ++ git_vector_free(&_commits); ++ } ++ ++ git_packbuilder_free(_packbuilder); ++ _packbuilder = NULL; ++ ++ git_revwalk_free(_revwalker); ++ _revwalker = NULL; ++ ++ git_indexer_free(_indexer); ++ _indexer = NULL; ++ ++ /* cl_git_pass(p_chdir("..")); */ ++ cl_git_sandbox_cleanup(); ++ _repo = NULL; ++} ++ ++static int seed_packbuilder(void) ++{ ++ int error; ++ git_oid oid, *o; ++ unsigned int i; ++ ++ git_revwalk_sorting(_revwalker, GIT_SORT_TIME); ++ cl_git_pass(git_revwalk_push_ref(_revwalker, "HEAD")); ++ ++ while (git_revwalk_next(&oid, _revwalker) == 0) { ++ o = git__malloc(sizeof(git_oid)); ++ cl_assert(o != NULL); ++ git_oid_cpy(o, &oid); ++ cl_git_pass(git_vector_insert(&_commits, o)); ++ } ++ ++ git_vector_foreach(&_commits, i, o) { ++ if((error = git_packbuilder_insert(_packbuilder, o, NULL)) < 0) ++ return error; ++ } ++ ++ git_vector_foreach(&_commits, i, o) { ++ git_object *obj; ++ cl_git_pass(git_object_lookup(&obj, _repo, o, GIT_OBJECT_COMMIT)); ++ error = git_packbuilder_insert_tree(_packbuilder, ++ git_commit_tree_id((git_commit *)obj)); ++ git_object_free(obj); ++ if (error < 0) ++ return error; ++ } ++ ++ return 0; ++} ++ ++static int fail_stage; ++ ++static int packbuilder_cancel_after_n_calls_cb(int stage, uint32_t current, uint32_t total, void *payload) ++{ ++ ++ /* Force the callback to run again on the next opportunity regardless ++ of how fast we're running. */ ++ _packbuilder->last_progress_report_time = 0; ++ ++ if (stage == fail_stage) { ++ int *calls = (int *)payload; ++ int n = *calls; ++ /* Always decrement, including past zero. This way the error is only ++ triggered once, making sure it is picked up immediately. */ ++ --*calls; ++ if (n == 0) ++ return GIT_EUSER; ++ } ++ ++ return 0; ++} ++ ++static void test_cancel(int n) ++{ ++ ++ int calls_remaining = n; ++ int err; ++ git_buf buf = GIT_BUF_INIT; ++ ++ /* Switch to a small repository, so that `packbuilder_cancel_after_n_calls_cb` ++ can hack the time to call the callback on every opportunity. */ ++ ++ cl_git_pass(git_packbuilder_set_callbacks(_packbuilder, &packbuilder_cancel_after_n_calls_cb, &calls_remaining)); ++ err = seed_packbuilder(); ++ if (!err) ++ err = git_packbuilder_write_buf(&buf, _packbuilder); ++ ++ cl_assert_equal_i(GIT_EUSER, err); ++} ++void test_pack_cancel__cancel_after_add_0(void) ++{ ++ fail_stage = GIT_PACKBUILDER_ADDING_OBJECTS; ++ test_cancel(0); ++} ++ ++void test_pack_cancel__cancel_after_add_1(void) ++{ ++ cl_skip(); ++ fail_stage = GIT_PACKBUILDER_ADDING_OBJECTS; ++ test_cancel(1); ++} ++ ++void test_pack_cancel__cancel_after_delta_0(void) ++{ ++ fail_stage = GIT_PACKBUILDER_DELTAFICATION; ++ test_cancel(0); ++} ++ ++void test_pack_cancel__cancel_after_delta_1(void) ++{ ++ fail_stage = GIT_PACKBUILDER_DELTAFICATION; ++ test_cancel(1); ++} ++ ++void test_pack_cancel__cancel_after_delta_0_threaded(void) ++{ ++#ifdef GIT_THREADS ++ git_packbuilder_set_threads(_packbuilder, 8); ++ fail_stage = GIT_PACKBUILDER_DELTAFICATION; ++ test_cancel(0); ++#else ++ cl_skip(); ++#endif ++} ++ ++void test_pack_cancel__cancel_after_delta_1_threaded(void) ++{ ++#ifdef GIT_THREADS ++ git_packbuilder_set_threads(_packbuilder, 8); ++ fail_stage = GIT_PACKBUILDER_DELTAFICATION; ++ test_cancel(1); ++#else ++ cl_skip(); ++#endif ++} ++ ++static int foreach_cb(void *buf, size_t len, void *payload) ++{ ++ git_indexer *idx = (git_indexer *) payload; ++ cl_git_pass(git_indexer_append(idx, buf, len, &_stats)); ++ return 0; ++} ++ ++void test_pack_cancel__foreach(void) ++{ ++ git_indexer *idx; ++ ++ seed_packbuilder(); ++ ++#ifdef GIT_EXPERIMENTAL_SHA256 ++ cl_git_pass(git_indexer_new(&idx, ".", GIT_OID_SHA1, NULL)); ++#else ++ cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL)); ++#endif ++ ++ cl_git_pass(git_packbuilder_foreach(_packbuilder, foreach_cb, idx)); ++ cl_git_pass(git_indexer_commit(idx, &_stats)); ++ git_indexer_free(idx); ++} ++ ++static int foreach_cancel_cb(void *buf, size_t len, void *payload) ++{ ++ git_indexer *idx = (git_indexer *)payload; ++ cl_git_pass(git_indexer_append(idx, buf, len, &_stats)); ++ return (_stats.total_objects > 2) ? -1111 : 0; ++} ++ ++void test_pack_cancel__foreach_with_cancel(void) ++{ ++ git_indexer *idx; ++ ++ seed_packbuilder(); ++ ++#ifdef GIT_EXPERIMENTAL_SHA256 ++ cl_git_pass(git_indexer_new(&idx, ".", GIT_OID_SHA1, NULL)); ++#else ++ cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL)); ++#endif ++ ++ cl_git_fail_with( ++ git_packbuilder_foreach(_packbuilder, foreach_cancel_cb, idx), -1111); ++ git_indexer_free(idx); ++} ++ ++void test_pack_cancel__keep_file_check(void) ++{ ++ assert(!git_disable_pack_keep_file_checks); ++ cl_git_pass(git_libgit2_opts(GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, true)); ++ assert(git_disable_pack_keep_file_checks); ++} +diff --git a/tests/resources/small.git/HEAD b/tests/resources/small.git/HEAD +new file mode 100644 +index 0000000000000000000000000000000000000000..cb089cd89a7d7686d284d8761201649346b5aa1c +GIT binary patch +literal 23 +ecmXR)O|w!cN=+-)&qz&7Db~+TEG|hc;sO9;xClW2 + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/config b/tests/resources/small.git/config +new file mode 100644 +index 0000000000000000000000000000000000000000..07d359d07cf1ed0c0074fdad71ffff5942f0adfa +GIT binary patch +literal 66 +zcmaz}&M!)h<>D+#Eyypk5{uv*03B5png9R* + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/description b/tests/resources/small.git/description +new file mode 100644 +index 0000000000000000000000000000000000000000..498b267a8c7812490d6479839c5577eaaec79d62 +GIT binary patch +literal 73 +zcmWH|%S+5nO;IRHEyyp$t+PQ$;d2LNXyJgRZve!Elw`VEGWs$&r??@ +Q$yWgB0LrH#Y0~2Y0PnOK(EtDd + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/applypatch-msg.sample b/tests/resources/small.git/hooks/applypatch-msg.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..dcbf8167fa503f96ff6a39c68409007eadc9b1f3 +GIT binary patch +literal 535 +zcmY+AX;Q;542A#a6e8^~FyI8r&I~hf2QJ{GO6(?HuvEG*+#R{4EI%zhfA8r{j%sh$ +zHE~E-UtQd8{bq4@*S%jq3@bmxwQDXGv#o!N`o3AHMw3xD)hy0#>&E&zzl%vRffomqo=v6>_2NRa#TwDdYvTVQyueO*15Nlo%=#DXgC0bhF3vTa`LQGaO9;jeD$OP?~ +za$G4Q{z+Q_{5V?5h;a-noM$P{<>Q~j4o7u%#P6^o^16{y*jU=-K8GYD_dUtdj4FSx +zSC0C!DvAnv%S!4dgk +XB^)11aoGMJPCqWs%IS0YSv(eBT&%T6 + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/commit-msg.sample b/tests/resources/small.git/hooks/commit-msg.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..f3780f92349638ebe32f6baf24c7c3027675d7c9 +GIT binary patch +literal 953 +zcmaJy@-{3h^^Cx;#d0zEA@DDc$nY4ez&|=%jTg@_HU*ub=!!y$xW09TSjlj +z(`I@QCsM`!9&80$I98wsQ8yK#)Orb<8re8FjkKh630D$QUDwi~(gkX=RunYm$rDjk +zlp%RUSnzA#6yjdG5?T?2DcYKp+v_lts0ljn&bh3J0bD5@N@1UKZ190O6ZeWr-BuZ^ +zWRebCX%(%=Xoj#(xYk1Cjtr!=tyBesf@m6}8zY6Ijbz9i9ziI_jG9MvR +zDH*e>^ga9IR?2wrSrAVm;eButj4Y>7(E2?b~jsu>& +zRKCJ7bp#19sqYh627wD%D9R$8=Ml$TNlumDypl~$jBu*G>5fIR^FB0h0Ex&TGZNr> +zL5hs1_K>taRb!|ThN9ns7^@4MXKP+6aGI_UK)T-M#rcP$;kN(Vcf#P)+5GzWa{l@J +z>-E{`$1iiNVYxq27}j;uo%;)r3kJI2xCFF~Ux;$Q%) +wjbk6JlDCM`jU&P+UVOvg`|iYl<7~9k>HHB4I;pdlQ=I-^$DrHaN$@lH1?P!0U;qFB + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/fsmonitor-watchman.sample b/tests/resources/small.git/hooks/fsmonitor-watchman.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..41184ebc318c159f51cd1ebe2290559805df89d8 +GIT binary patch +literal 4777 +zcmbtYYi}F368$Xwipg4lq(BeHMvzvH-4;n7DGJBPqq#tw3aed8+IU5-m)yvL>;Cqh +z8FFRGj$`9CA8aoJ?j^$%==FV``-=rhLcPW`McSytRm~mEO7_&_cAVZrf1fFy*ha@8oe%*-aBYE +zcjzZg>LOkgxuUr-XJnHyD;zmPnRaSc#!k_P*d_BttRdc+J6G7za5#+^Y1nkc2Oowk`ya47uUR3Feu?B(w;S{(VYzxh}q-=#zP@uxSx{wbyPUMFU;K(06)$o{07&3yI?q{GqMcQ1c_^M<0< +zF4acAV)Il-V(rCTC1(;bsZ*}bl8dmejAk~yb`B}!^0;g^(o9kGUfZfDOvyp@x4OQt +zSgWh6T|3eq;9MFs8-#z+FDM1h(IjRUP|``PxupgJ7CUHOH90gbgl^2~97`?_X{P)) +zB*$r1cDlF-%azKND}?Gv`2K8-9v5e`gQoft=j?T<&a13c^!wY_$D`5z-X1g?ty&6- +zQN50{8?bUk9AI->^W@~~nkOghHIC2YN+AXkLQG_2-{Pq3%{`3KUMeG$iIn%%^6*NYb +zn|_BdV#C)n4565VccX;uT8&z3vSi!HXGbUj2B!R +zdz~&#fk#L-&k$fLwo$4?>12g@AXOKFekuo#6EHB%gmpD?1eyh%N8s{2wGoTu +z*@6cEZ^ZW!FAF_|JL`NkV7k}0ow|-2jHwbgH0;c@Dq*o?@&c*HnGdyx6^su8Qk%2{ +z*ye(dxO*6-&>qn1+zw}tc6;=sOX{4WB=VqjTS^))y1jlX2Q;=e!qMmFA5lC$#;BxC +z=Y%tRpWxb+_uQAvAw7Q{HGV#R$xb&udLCzZ+HN?kTyB};1EJ8UlQ5!>5eGW@)RX0n +zkjj>EF!3=0Gl^8dzv$B^NMGRxJoqN4A`xq-@wCbrx*u2NmIJ1xZ%H +zh;{|4T3(!E9sY#Ni(wUJYs1MmIc9bl)(4Nl3_wD_BWB>i<1S(LX7m*{Q7PU$muMS* +zM!%0EZx-Vw=Zey;erC?SNxF;pY@^A%-krqzfLV2meBp1vWdyArFYn`DD19T)Hw(?n +z)}{NP(Lk(o*?gl#B@pP7^*r|=;PIDT4|F#{2Hzh-AL0Rv$6uT;n|WzE4=slK?on@(fZeGhRgQCu56qB +z{+n81Az96qnQjMY*-*r-KV*7;Z#4QuJRJJV$M^KdldiMhj?ImK6~FvwJ*L5a){QoM=L5TYHkGO1$UrO3`a>{?Opw|b +zG(#59NQ#jFL9v~vgOVkM@^^(^A}onOE))yWEwhIlk&{ZyseZ^O0b=w8&O=BK{k<5B +k^Q-B@eG}LeHrquz%(SVEp_N)VhYZikCW__82JXfD17`J9Qvd(} + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/pre-applypatch.sample b/tests/resources/small.git/hooks/pre-applypatch.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..625837e25f91421b8809a097f4a3103dd387ef31 +GIT binary patch +literal 481 +zcmY+ATTa6;5Jms9iouO45IBJXEg&Jm9@v1LPHMM_ZR|;#6tQh$71hSXq*MxP;V& +zj0cY7SCL=x4`a46sF)C>94Gk%=3q$W2s;j6iHtB2$R0%gix4oK@&T~=ALd_o*CKxt +I-`Pv{1Bpzc>;M1& + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/pre-commit.sample b/tests/resources/small.git/hooks/pre-commit.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..10b39b2e26981b8f87ea424e735ef87359066dbb +GIT binary patch +literal 1706 +zcmZuxU2ohr5PY_N#pZ0-F<{-v&v-X^RA+u>k}E$4d&uD7=g_fA8+pNNV=4s0|iD3p<=DTXClTS +zXV23tJ;ECmN@M0j@zUAKEYW@3bv!SeYZ8ZH`YQNTApFVNc;F|9r5p4TqGs=>8E?6y +zi|gY{iM#PG1nL?UE9YCnWTk72kgZPG*Usqw!~Qd3c?~@w2?%eg@~)+VlSs6N5Yf2^ +zz;owF#K#r^&KMq1A`oqVGFpD&-!Pv|Rc +zO3KSqA@h9nSc%bm`0)Amk6*J}@14J*1-219l%%7D!Pl}UK>|lVi0Dfgu2jN3WC!uL +z0ej??b2iSehVgdnWHmZV4kUo*QL#aiIp}U=9x)IXk}JJ7VQ;CI9Rtn5e0VcjbYcVt+`x5D+svCGD;Z5hm*E$jSEQZ%SQ(}oLgslTvrKK@9Qf#b!hajVFnp9@oIix;NcI9Wk +xjnh0ya!AWet{I7YpD;y6HXyzI*lfSvH=o6*7mJZPkuaYpm>vzZ`wyGEBtOQPo|pgt + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/pre-push.sample b/tests/resources/small.git/hooks/pre-push.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..02cbd80c287f959fe33975bb66c56293e3f5b396 +GIT binary patch +literal 1431 +zcmaJ>U60!~5PUX&#a1@z9B{IIZkjLT0t5kq9#8~D(I5{+8&J~9;#ndUk~-ZT`r|uG +z$#K$$J{TsKs*LP1}9!GoZ@4I4myMMG_di|of +z%?llx{O8TS-#^;(OioEmPy%kwWQBA1OMzV{hsQ8XFzS1k!~YQoLa5 +zhtP1fA$q6VmMbbAC_9)4I628k*O5J$NR19uHe4QYDK<==I~SQk)Nu%xQ~KH +z53w=!ke(FGb_PpnZfd*+hnXDTn;2*`u^~;?+5C~cn?bRka7NR%06%e6O91{MAgN6J +zmlO8{Biw4&wr&&(z4p3eln`E}XR9m9bNYZ7Ibrg(4yZIXrfgD7N*AFD7L3YSM#j}% +zo__rOS5fr;@8UM<6cl+cv_$YB$PQ&9dv($eM*))g!_cu!QcSh-mqE9i#QDZT)=o#` +z?8!RtE?w6p?GkGZ-6yt_p~5~4ecu|Sf^)6096%h*q-eNiEA1;Xwg)p~Q&iGSG7-IQ +z9aII&`ps$WOojFA`*bjGkFk|E@sHHuD}W^d`7YJ3YE^zrQnqR +zGoq?;YGKe)93o|_=^f%3U1KYZGPOXRRxK7w`UUbMMa3<86OmVH!EKP$8RCrn9mWX+ +zC?9yF!fRVLmud3hF<}x;;sR}f(*r}6Gap3fR6zLHR~kbMgD{98N`L+r&?3p~*0+FX +zcAL%j=(SO}xTJUTvA`&Lf`2mv4koPG9&|;2+68$XxiXKL@ma;l5d2^5Ba_rPh_DHI-u1#&_upttZXp;no03$20|NFiM +zK#D#xQ>!Z3JkX8T-LDVm!B5j7y_{;JDmmTTef+K1oIiPzeEr+Ai*<2PUgnG4^ZB>p +z_fkAvoR1emuf~ri^K$-px=4#D-vY9w& +z`bCv#2zVn=YnJyeNey(Y +zRh`9vtLw~A+5zsjp|W0Nsa|29Rm!B>OoG5a+vi;ari8O>KkU!KAWg_fa3btK2x*_@ +z0bEc7J;Ubghm}n9bOi(Sv_B66nQ7U)J7f0fO}8Wuf*uorcIgEG +zOHc|-V6+HlRhOP}?Cn?@5iwSl43abmBA^2lyL$+cpabCGVES+v^j^FO_}?FIp%En%Ll?Z*7*}TwrZyg5OSZ9rY-`aU~Mc-jjv{Ll)FLMgtB4ujktfQ`Xhqrka +zT=P!A;9w^;Z?PqpLwOLu=cj3L>TdUKw2;DMu)`oVkj}#bcDx4tYg=j%D`+i{W~fVM +zVmZ>W9VMyin9c-0KzI_;iZ-g|OyzuG`Yq%(%dvl;ifnVr0;jWE&S`z|rQu=!yHBBO +zx`OJ;oOQ(KKM<$(bC38o>pD0%|HA(E0TRw7qj$fJ_pRN+7Nm>dSC(gLg{(`t+5Z=?o+}wXU4tHy+&%F&aRhFebeEhR2R5|$#Ycbp^w@t +zTl%=f1t=w+WpJzF<|CE@?SCNAz)%9?w33lQ8vrHJqPfH9@}qs*QXOG71W=ylx;wOB +zcx!Bj^)Yy6WX$a^vBkBJ5CobqlaDx_B0c<3b+8)f84LCrt;e;qxc+7>VbwVK{skNv!wvBiTa^9Iu +zkwP;VK)jH$WJ{`MRwAA9fal!y0dtV;FWg8PTkWU>CwnqD>1ZX2B@;$DlX%C5MI+}{ +z9xQVnffR*~v2KAUj*hCdgul~`bk#mk`o>zk9)<2Uc8?hUZAEvd!`9em)~$Z)zev>w^8 +zyAgCP_$&Y)7HSQ84`xG}OeTavaEswwF|8Xpi5iZzZa@hCiv(J-%bfFC&)HLlO+Rhw +zG6g?9eL5&A!SuJnQ6}LxG%tU+@vZ`i+!+Rz6iYvsTdhnPo7lW{m-}{hya@viX4)XZ +zngaw+j;gloB#|UwI@8sOmQpc`h+bicQJnQIB5eifIMQNgD2+oai33m!34~xU|0Azj +zhu$8z+T5^;Pxx@d{N)pzOJLSa^e;aDf$W%N5XcOf!mGC9l9j$Ev2h6N+6ZQC+CJzl +zaM7?S!SrFLS2DASjj(h6y1WN3N?|bmqmyzm!&nLoE|`rKBOc_yDF$a#FsUn!IQf(t +zdC&Us(kQz*7mvH^j*^MC@>wTDb}g%~sx*ng#>{@lR=XG-Z5_ +z#<9*Oh0joMzt;nS)ObAp)347`D=}r-;nV!TbIq&xrGRGsF6fZg+!VkfUei@_&l-M& +zPqQ+Dw)RV}+)I8RuqAxa`Pv8e&!_gXS=e2-un>=Ktn}-;%lLZxaVn?Q>yZCb2R3Wk +z77zr%;Rq&h|2ncqyKYmFI0148JVY7Q$V5p=dWj+Qqpu%i|xp2C=WaOb2Wudn^h0EcD%$p9YVU1fnoRV9`(cy(vv6K>FXS!2jY>1GnU--7)4usH&K +zao*&P^@9~YmUe|ZdLW@C>H;!*Vt3>Nw4M*;=?j(TBD#O@XCv0|MEhA;z}kTFRv@`tPHhp=&Yh +zg%Zhg4i7o_k{a5i&f5;tZ==%}^Sn4aD_6%qs_XAuJt&EumdH4Yu`UjT<-+XHTuHss+b +YOmM2;hq8Egm*4=7_P9T{21QBYH*F=mfB*mh + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/prepare-commit-msg.sample b/tests/resources/small.git/hooks/prepare-commit-msg.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..b1970da1b6d3f42f00069fd17c325de72cda812e +GIT binary patch +literal 1702 +zcmb_cTW{Mo6n>t6#i?x6xmZ$SFLf{QfG*3r0L?Pg?px55l8$UTGO3bO;spKi{V3XX +z))weX0X>M9bNMcZ-6yG%>(n}JI2|25dr}WZBP@ih?JX^+@ +zu#5O48P>yRX(mfDIhYP)doc1&TADZa@ZGpusJ$6G+e$ZMcmC +zoOosDQPS}l{H?YPsq(4;0SGkATa9eeqAaDcjq8n2wALbFwU@2i@FAaRV!=uw-nwx1gKn2SvY +z>Ff>;2sg!+Hxfkwv1lsiii=p6WenF=5)6LZcQaZ=aS_}+-4Y&?!@HWh|<^gJ21!|T@+%On#w6azxPHV}XsRbe*w +zR_TZ2XEsQa1lPK~biYqg@0-RW@5J1@=<87cFzEUABdCoFH2CZo?}l(Z*!OFqUxo>K +z_d`l#4d9|H6;VPT{X?^{VJ>oL|D7K{BJwwqB>`YcPoGk+9hbvHnoQ{EM|kPgD_`wk +zKm4#2xu;-y`RAm!=L_BnLvJ8$AZm8@?)v<%vwvsw8AF2x6!mTT;c72A_~U9nIq0ST +zv)N0!I!^1p=g8-RQfx5)E_Mb_4I2vtQpI30XZ&t-9h5!Hn + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/push-to-checkout.sample b/tests/resources/small.git/hooks/push-to-checkout.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..a80611e18896f212c390d845e49a3f6d5693b41d +GIT binary patch +literal 2840 +zcmai0U31$u5PXh)#YOS7cE^-rw@uolNhe9&aUS|HtvhX>G$45tVUYj>fRdF?|9kfU +zNR~aG=E)WbEbeyq7JTw}ZuHIE2kUtL<AoeCNptd-NM1aZLhESzC;I`+Ns +zfmNNjdAp^W8#Q*}l>CT7RB9F5(BbI8ly2l~+E};JW|>&d1)=epZ-8vm8ppkbEVn#R +zt30a5A-c(YQR8eM5%;|UAnO>rt!&@x@G@yp+92%w-}%(5P_+P&Wf_zb$f-Qrl5(7z +z2ah(bkE;!DK(&aAMuQ%1TS>ai?wSXCOCSj=_}8x4IbCx^$}9q)whwv)SBt| +zg#MX4;;Oau`m=MI9(^&zPbueY@~>3*ixX%mvR5m_1&nAg@ZKvY1E$O}&EtLiG;mhV +z1xhMIm~fGjmf_#{62f`y;09?I7M1W2tWQvz<}i9lR>OpQyUJi45_&*pQus&EkwY<> +zI|ZAx=*3i9a-)g)hXkvO7>UJ5MNgL(Z+-wpXVcgbSgpmFmbf1~DPA(OVGI&FNLeIE +zNH!_aiH$vsif$_j7=T2{cS(!DOI`~bn@)vSd-0d7xL=DF;UNP|tW}4ih>DvHtu9tY_pbJ6x(6E*hxgC +zzNDao%qlr-IE%YGbS4hF!n!on7#W3$bX-_hbZAaws^nHu#)Dx=WzdbJ>AKzAy@T$x +zSWE^x9+|TEHVEPyaPYa0DOChp?AeHSBBDbZNokQpAY{lE!7geZI=jV)G^2@l)&91Zb1+`T+oq9wWF +zRV~kGTGce0O~p^6mj{kT5kL(pv>r;Lvd7VDX*P>A^Th`$3cWO0L81p4Ysdo3ZP1(SrR-peEdTo;-@bkB((G +zPHYQXUL!@Q$e(OQ;R9r%@Afz+50I7>*^^c&&|E*r-jN)LH=pM4AqMwWxSv|nqjddE +Z4{_hwv8!W(T +zYw`X3V>TCdnSD1ru8&`j=2DIPbCT@SnIgUw>$+lEYP}+x8(BMYnr=iT3*ndq)xzaV +z>I+qjv}vC#8_9M+b1p#uNS0M0)q

8!3p_LRQ0MA3M`!2foxzRUjbFY@}O~(ki=S +zqscnq8cU*dY)D$$cqE}n)V0yIk>CNKHCrndOtSP*HbOb;nbwAHSb;R+gs^?^Dve%) +zoW}t(*D}$>O3ab0TS^-;J|u&sb-PkZzo#kn*#xYt(;FGuwzSb^g&RDiGcOz9TB;Hu`nJh)$W=C=XCSm2AY=$w3G3P-V#Oo+N*;#2 +z4ijJ-pBZ=;T(RTgp_HYrD!uW-dTMfkuqY5jwOy)~gM;#=P^i{!l7`pXTS^s(&^{RU +zydaw}OpS#^D1cXM8?FW+fh`t7D(g;yr6|}fdaNtZBx3hlK~IpkTu3!Qq%R+zAo#t}Bs8^3$vHD+-TGT@`F>H1Cc#WAVW;&$S6%fE2d6@kLS0g&ihIM{}0z +z8#XhD>b>3{(BH|Px7}&lJ4%y1v(CihZJx@8MPoGdl*BJGD;usf*iS7%;{Joe; +zNFuBa>*~o&qETDPo~u&~$FxE1xb^x&(CbE`Y3GfsibL2rl+L;>P6j&Y3U>K$mkp*6 +zd`Q{<^+^&;GskGjwD-%!boR&i-TCA9UOR|@=GYb5x#+dhd7fkaVIR^pol`Mv+rUbmZ43dVL6^S7g3{NsPiG$iy$5EDB% +z6KIgnb$H(n&t3e4E6d4V7w^B?JS}JkG)PM6+X3Co`SQs($O*AA+MG~{S7RJ=cy-l& +z>~%3y`tjfx2>uOutB_^s +ziwG=e=ch|FQ0IkN91US7rhdQkXhwwt$gU0WEVDjo=IPb+?6PC=s8}J*ua(Ms))`UL +fi$|vMHn?H_tSE3ettp-hLlsZCxaLX8(nU;bVRB;Ce6@s#eu2|WvLz>- +zvy(&>Gyfp@+BtKnpqWkKi^+v{4jn_pNw_zeuxETifiGO|)w}OANj2n2D^K=o3j6P6uOL70#cbA{uzWXDlk1wr9GV1X(2W{RuTvjXV +zCmd8u +zH%V`94=q3)Dk)PHNrnFC(T1)Om6f{Usj;u1R->&XoCYVK2V3ZlgZuF?N}1+33OER*x +z*9Z=L=zI8CN>A_^jYjt0F$psO$sL=38q5q|SG)qCN6{^>RFh5E&l5GZ$pEahnF&d+ +z5c>64t}uJPkf~_!VUj#&N%nC-gUMj%=@B=!V>&}xtj2%@-mOm#rQUSJ3(ccmc+fza +znZ#uxF>N?QN5UrIEd!5RgHEfW#;(nKYF+D<*rdshJ$X-z2OZ2X;)nn@KSVdVhaA?}@3;6gZxb4v +zozoWSr{{+!h}zGpumG3H`=AvWpm^9kW;J$Jp^Xl*?8ckr`fqN%c|Z;VC0|cM4vSrk +zH_O8Yvh85nvJp^;``wo8=z0f`FWg?`>gO#y1hjX1{}rTlg9rwIKia8eyGexA3GnuR +z`Rg~XZoW;0pA)vI8=p5!+6sIn#C^FCvR>ffv39h6SCNi9v);%WD;WZ`of_MgwyRWy +z-yY%n*Y>X89W-v4`Ff%bx$Vkn}$!Ay}rnY6F$m-Kg*KD_+;Lx#g4|^&N +I02NaX#p`nv=Kufz + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/objects/af/5626b4a114abcb82d63db7c8082c3c4756e51b b/tests/resources/small.git/objects/af/5626b4a114abcb82d63db7c8082c3c4756e51b +new file mode 100644 +index 0000000000000000000000000000000000000000..822bc151862ec3763cf2d3fa2372b93bbd3a4b65 +GIT binary patch +literal 30 +mcmb>0i}&W3IZ_@1U=^!a~EV1casc=c+{&un1qQN*i9hD|0|m(2n|iwp*q%W +z%N;b$hu%cM`$TMo*~EnC1BFP&Pfj~;jZVKXQ96s_PhV<-XAROi+@-v8dBLUa`!;GB +k^iXlEv8$>R)1G>9th&t3j;s7J{?^9n|7U^`%mXoWC24Q^m!3%@{ + +literal 0 +HcmV?d00001 + From 976f539f7dd2ee6345b605def105883d16b32c60 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 28 Aug 2024 02:35:46 +0200 Subject: [PATCH 08/37] Make Repo::flush interruptible --- src/libfetchers/git-utils.cc | 115 ++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 30 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 70391a287..265eb35d2 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -166,6 +166,45 @@ static Object peelToTreeOrBlob(git_object * obj) return peelObject(obj, GIT_OBJECT_TREE); } +struct PackBuilderContext { + std::exception_ptr exception; + + void handleException(const char * activity, int errCode) + { + switch (errCode) { + case GIT_OK: + break; + case GIT_EUSER: + if (!exception) + panic("PackBuilderContext::handleException: user error, but exception was not set"); + + std::rethrow_exception(exception); + default: + throw Error("%s: %i, %s", Uncolored(activity), errCode, git_error_last()->message); + } + } +}; + +extern "C" { + +/** + * A `git_packbuilder_progress` implementation that aborts the pack building if needed. + */ +static int packBuilderProgressCheckInterrupt(int stage, uint32_t current, uint32_t total, void *payload) +{ + PackBuilderContext & args = * (PackBuilderContext *) payload; + try { + checkInterrupt(); + return GIT_OK; + } catch (const std::exception & e) { + args.exception = std::current_exception(); + return GIT_EUSER; + } +}; +static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuilderProgressCheckInterrupt; + +} // extern "C" + struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ @@ -213,42 +252,58 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this } void flush() override { + checkInterrupt(); + git_buf buf = GIT_BUF_INIT; - try { - PackBuilder packBuilder; - git_packbuilder_new(Setter(packBuilder), *this); - checkInterrupt(); - git_mempack_write_thin_pack(mempack_backend, packBuilder.get()); - checkInterrupt(); - // TODO make git_packbuilder_write_buf() interruptible - git_packbuilder_write_buf(&buf, packBuilder.get()); - checkInterrupt(); + Finally _disposeBuf { [&] { git_buf_dispose(&buf); } }; + PackBuilder packBuilder; + PackBuilderContext packBuilderContext; + git_packbuilder_new(Setter(packBuilder), *this); + git_packbuilder_set_callbacks(packBuilder.get(), PACKBUILDER_PROGRESS_CHECK_INTERRUPT, &packBuilderContext); + git_packbuilder_set_threads(packBuilder.get(), 0 /* autodetect */); - std::string repo_path = std::string(git_repository_path(repo.get())); - while (!repo_path.empty() && repo_path.back() == '/') - repo_path.pop_back(); - std::string pack_dir_path = repo_path + "/objects/pack"; + packBuilderContext.handleException( + "preparing packfile", + git_mempack_write_thin_pack(mempack_backend, packBuilder.get()) + ); + checkInterrupt(); + packBuilderContext.handleException( + "writing packfile", + git_packbuilder_write_buf(&buf, packBuilder.get()) + ); + checkInterrupt(); - // TODO: could the indexing be done in a separate thread? - Indexer indexer; - git_indexer_progress stats; - if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) - throw Error("creating git packfile indexer: %s", git_error_last()->message); - // TODO: feed buf in (fairly large) chunk to make this interruptible - if (git_indexer_append(indexer.get(), buf.ptr, buf.size, &stats)) + std::string repo_path = std::string(git_repository_path(repo.get())); + while (!repo_path.empty() && repo_path.back() == '/') + repo_path.pop_back(); + std::string pack_dir_path = repo_path + "/objects/pack"; + + // TODO (performance): could the indexing be done in a separate thread? + // we'd need a more streaming variation of + // git_packbuilder_write_buf, or incur the cost of + // copying parts of the buffer to a separate thread. + // (synchronously on the git_packbuilder_write_buf thread) + Indexer indexer; + git_indexer_progress stats; + if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) + throw Error("creating git packfile indexer: %s", git_error_last()->message); + + // TODO: provide index callback for checkInterrupt() termination + // though this is about an order of magnitude faster than the packbuilder + // expect up to 1 sec latency due to uninterruptible git_indexer_append. + constexpr size_t chunkSize = 128 * 1024; + for (size_t offset = 0; offset < buf.size; offset += chunkSize) { + if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats)) throw Error("appending to git packfile index: %s", git_error_last()->message); checkInterrupt(); - if (git_indexer_commit(indexer.get(), &stats)) - throw Error("committing git packfile index: %s", git_error_last()->message); - - if (git_mempack_reset(mempack_backend)) - throw Error("resetting git mempack backend: %s", git_error_last()->message); - - git_buf_dispose(&buf); - } catch (...) { - git_buf_dispose(&buf); - throw; } + + if (git_indexer_commit(indexer.get(), &stats)) + throw Error("committing git packfile index: %s", git_error_last()->message); + + if (git_mempack_reset(mempack_backend)) + throw Error("resetting git mempack backend: %s", git_error_last()->message); + checkInterrupt(); } From 459d02672c5ea9c6b909c0ace6b11141af79421e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 28 Aug 2024 11:13:03 +0200 Subject: [PATCH 09/37] fix Windows build --- packaging/dependencies.nix | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 0182f29c0..e5f4c0f91 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -116,20 +116,25 @@ scope: { cmakeFlags = attrs.cmakeFlags or [] ++ [ "-DUSE_SSH=exec" ]; nativeBuildInputs = attrs.nativeBuildInputs or [] - ++ [ + # gitMinimal does not build on Windows. See packbuilder patch. + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ # Needed for `git apply`; see `prePatch` pkgs.buildPackages.gitMinimal ]; # Only `git apply` can handle git binary patches - prePatch = '' - patch() { - git apply - } - ''; + prePatch = attrs.prePatch or "" + + lib.optionalString (!stdenv.hostPlatform.isWindows) '' + patch() { + git apply + } + ''; patches = attrs.patches or [] ++ [ ./patches/libgit2-mempack-thin-packfile.patch - + ] + # gitMinimal does not build on Windows, but fortunately this patch only + # impacts interruptibility + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ # binary patch; see `prePatch` ./patches/libgit2-packbuilder-callback-interruptible.patch ]; From afa6cc72714161042f05cacf9b21d328a49ccd78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 22:59:12 +0000 Subject: [PATCH 10/37] Bump cachix/install-nix-action from V27 to 28 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from V27 to 28. This release includes the previously tagged commit. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/V27...V28) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84e5ab998..30272ecf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@V27 + - uses: cachix/install-nix-action@V28 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -89,7 +89,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@V27 + - uses: cachix/install-nix-action@V28 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - uses: cachix/cachix-action@v15 @@ -112,7 +112,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@V27 + - uses: cachix/install-nix-action@V28 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -142,7 +142,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@V27 + - uses: cachix/install-nix-action@V28 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From 65f138f6693eca7edfa0aeacc780597520868201 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Sep 2024 12:37:33 +0200 Subject: [PATCH 11/37] nix derivation add: Remove reference to "installable" It doesn't operate on an installable, so don't mention it. --- src/nix/derivation-add.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md index 331cbdd88..35507d9ad 100644 --- a/src/nix/derivation-add.md +++ b/src/nix/derivation-add.md @@ -3,7 +3,7 @@ R""( # Description This command reads from standard input a JSON representation of a -[store derivation] to which an [*installable*](./nix.md#installables) evaluates. +[store derivation]. Store derivations are used internally by Nix. They are store paths with extension `.drv` that represent the build-time dependency graph to which From c7c3a7f667d9ed85ef8255205c54c15b6b99bcd9 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 17 Sep 2024 16:15:39 +0200 Subject: [PATCH 12/37] Fix meson build on macOS in sandbox Workaround at src/libstore/meson.build#L429-L434 by @Ericson2314 from https://github.com/NixOS/nix/pull/11302 erroneously used `macos` instead of `darwin` to distinguish macOS, while meson docs list only `darwin`: https://mesonbuild.com/Reference-tables.html#operating-system-names. Original thread: https://github.com/NixOS/nix/issues/2503#issuecomment-2353184049 --- src/libstore/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 2adc9b3e4..6a6aabf97 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -427,7 +427,7 @@ extra_pkg_config_variables = { } # Working around https://github.com/mesonbuild/meson/issues/13584 -if host_machine.system() != 'macos' +if host_machine.system() != 'darwin' extra_pkg_config_variables += { 'localstatedir' : get_option('localstatedir'), } From ee3f0b7a8b40becb2c03d944a0f02169bb205cda Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Sep 2024 16:55:44 +0200 Subject: [PATCH 13/37] Rename import-derivation -> import-from-derivation --- maintainers/flake-module.nix | 2 +- .../ca/{import-derivation.sh => import-from-derivation.sh} | 2 +- tests/functional/ca/local.mk | 2 +- tests/functional/ca/meson.build | 2 +- .../{import-derivation.nix => import-from-derivation.nix} | 0 .../{import-derivation.sh => import-from-derivation.sh} | 4 ++-- tests/functional/local.mk | 2 +- tests/functional/meson.build | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename tests/functional/ca/{import-derivation.sh => import-from-derivation.sh} (55%) rename tests/functional/{import-derivation.nix => import-from-derivation.nix} (100%) rename tests/functional/{import-derivation.sh => import-from-derivation.sh} (58%) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 0b83e5696..fb286208d 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -508,7 +508,7 @@ ''^tests/functional/ca/concurrent-builds\.sh$'' ''^tests/functional/ca/eval-store\.sh$'' ''^tests/functional/ca/gc\.sh$'' - ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/import-from-derivation\.sh$'' ''^tests/functional/ca/new-build-cmd\.sh$'' ''^tests/functional/ca/nix-shell\.sh$'' ''^tests/functional/ca/post-hook\.sh$'' diff --git a/tests/functional/ca/import-derivation.sh b/tests/functional/ca/import-from-derivation.sh similarity index 55% rename from tests/functional/ca/import-derivation.sh rename to tests/functional/ca/import-from-derivation.sh index e98e0fbd0..0713619a6 100644 --- a/tests/functional/ca/import-derivation.sh +++ b/tests/functional/ca/import-from-derivation.sh @@ -2,5 +2,5 @@ source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 -cd .. && source import-derivation.sh +cd .. && source import-from-derivation.sh diff --git a/tests/functional/ca/local.mk b/tests/functional/ca/local.mk index 4f86b268f..7c2fcc451 100644 --- a/tests/functional/ca/local.mk +++ b/tests/functional/ca/local.mk @@ -7,7 +7,7 @@ ca-tests := \ $(d)/duplicate-realisation-in-closure.sh \ $(d)/eval-store.sh \ $(d)/gc.sh \ - $(d)/import-derivation.sh \ + $(d)/import-from-derivation.sh \ $(d)/new-build-cmd.sh \ $(d)/nix-copy.sh \ $(d)/nix-run.sh \ diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build index f682ab28f..00cf8b35f 100644 --- a/tests/functional/ca/meson.build +++ b/tests/functional/ca/meson.build @@ -16,7 +16,7 @@ suites += { 'duplicate-realisation-in-closure.sh', 'eval-store.sh', 'gc.sh', - 'import-derivation.sh', + 'import-from-derivation.sh', 'new-build-cmd.sh', 'nix-copy.sh', 'nix-run.sh', diff --git a/tests/functional/import-derivation.nix b/tests/functional/import-from-derivation.nix similarity index 100% rename from tests/functional/import-derivation.nix rename to tests/functional/import-from-derivation.nix diff --git a/tests/functional/import-derivation.sh b/tests/functional/import-from-derivation.sh similarity index 58% rename from tests/functional/import-derivation.sh rename to tests/functional/import-from-derivation.sh index 68ddcfa4a..dc6e80ed2 100755 --- a/tests/functional/import-derivation.sh +++ b/tests/functional/import-from-derivation.sh @@ -4,11 +4,11 @@ source common.sh clearStoreIfPossible -if nix-instantiate --readonly-mode ./import-derivation.nix; then +if nix-instantiate --readonly-mode ./import-from-derivation.nix; then echo "read-only evaluation of an imported derivation unexpectedly failed" exit 1 fi -outPath=$(nix-build ./import-derivation.nix --no-out-link) +outPath=$(nix-build ./import-from-derivation.nix --no-out-link) [ "$(cat "$outPath")" = FOO579 ] diff --git a/tests/functional/local.mk b/tests/functional/local.mk index f61823765..3f796291a 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -88,7 +88,7 @@ nix_tests = \ why-depends.sh \ derivation-json.sh \ derivation-advanced-attributes.sh \ - import-derivation.sh \ + import-from-derivation.sh \ nix_path.sh \ nars.sh \ placeholders.sh \ diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 5167fa814..69b6d3194 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -157,7 +157,7 @@ suites = [ 'why-depends.sh', 'derivation-json.sh', 'derivation-advanced-attributes.sh', - 'import-derivation.sh', + 'import-from-derivation.sh', 'nix_path.sh', 'nars.sh', 'placeholders.sh', From 98db531df2527c7740abf38ee08ce8733e625519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 17 Sep 2024 18:46:50 +0200 Subject: [PATCH 14/37] libstore-support: check that we can create the store --- tests/unit/libstore-support/tests/nix_api_store.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/libstore-support/tests/nix_api_store.hh b/tests/unit/libstore-support/tests/nix_api_store.hh index 193b44970..8b034146b 100644 --- a/tests/unit/libstore-support/tests/nix_api_store.hh +++ b/tests/unit/libstore-support/tests/nix_api_store.hh @@ -61,6 +61,10 @@ protected: const char ** params[] = {p1, p2, p3, nullptr}; store = nix_store_open(ctx, "local", params); + if (!store) { + std::string errMsg = nix_err_msg(nullptr, ctx, nullptr); + ASSERT_NE(store, nullptr) << "Could not open store: " << errMsg; + }; } }; } From a20659f4fa5df9020c40086aa05d394e8119c30d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 17 Sep 2024 19:13:29 +0200 Subject: [PATCH 15/37] unitests: fix tmpdir when running with meson on macOS --- tests/unit/libstore-support/tests/nix_api_store.hh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/libstore-support/tests/nix_api_store.hh b/tests/unit/libstore-support/tests/nix_api_store.hh index 8b034146b..b7d5c2c33 100644 --- a/tests/unit/libstore-support/tests/nix_api_store.hh +++ b/tests/unit/libstore-support/tests/nix_api_store.hh @@ -3,6 +3,7 @@ #include "tests/nix_api_util.hh" #include "file-system.hh" +#include #include "nix_api_store.h" #include "nix_api_store_internal.h" @@ -47,7 +48,9 @@ protected: if (fs::create_directory(nixDir)) break; } #else - auto tmpl = nix::defaultTempDir() + "/tests_nix-store.XXXXXX"; + // resolve any symlinks in i.e. on macOS /tmp -> /private/tmp + // because this is not allowed for a nix store. + auto tmpl = nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", true); nixDir = mkdtemp((char *) tmpl.c_str()); #endif From 9e335ee3b9aa2897962be368e2325fbf4c529c88 Mon Sep 17 00:00:00 2001 From: quatquatt <78693624+quatquatt@users.noreply.github.com> Date: Tue, 17 Sep 2024 22:19:56 -0400 Subject: [PATCH 16/37] docs: Provide an example of updating multiple inputs with `nix flake update` docs: Provide an example of updating multiple inputs with `nix flake update` --- src/nix/flake-update.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 63df3b12a..8b0159ff7 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -25,6 +25,19 @@ R""( → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) ``` +* Update multiple inputs: + + ```console + # nix flake update nixpkgs nixpkgs-unstable + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nixpkgs': + 'github:nixos/nixpkgs/8f7492cce28977fbf8bd12c72af08b1f6c7c3e49' (2024-09-14) + → 'github:nixos/nixpkgs/086b448a5d54fd117f4dc2dee55c9f0ff461bdc1' (2024-09-16) + • Updated input 'nixpkgs-unstable': + 'github:nixos/nixpkgs/345c263f2f53a3710abe117f28a5cb86d0ba4059' (2024-09-13) + → 'github:nixos/nixpkgs/99dc8785f6a0adac95f5e2ab05cc2e1bf666d172' (2024-09-16) + ``` + * Update only a single input of a flake in a different directory: ```console From 0624cf0f59dc7363c57d472c6d2b4db07f1dcbec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 18 Sep 2024 08:45:54 +0200 Subject: [PATCH 17/37] mergify: enable merge-queue for backports --- .mergify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index 663c45d92..c297d3d5e 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -14,7 +14,7 @@ queue_rules: pull_request_rules: - name: merge using the merge queue conditions: - - base=master + - base~=master|.+-maintenance - label~=merge-queue|dependencies actions: queue: {} From 8690b6f1381a17bbe8e838d874aa23c541b1cf52 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Sep 2024 12:42:20 +0200 Subject: [PATCH 18/37] Test IFD/filterSource in a chroot Relevant to #11503. --- tests/functional/import-from-derivation.nix | 25 ++++++++++------ tests/functional/import-from-derivation.sh | 33 +++++++++++++++++++-- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/tests/functional/import-from-derivation.nix b/tests/functional/import-from-derivation.nix index 44fa9a45d..cc53451cf 100644 --- a/tests/functional/import-from-derivation.nix +++ b/tests/functional/import-from-derivation.nix @@ -1,7 +1,6 @@ with import ./config.nix; -let - +rec { bar = mkDerivation { name = "bar"; builder = builtins.toFile "builder.sh" @@ -15,12 +14,20 @@ let assert builtins.pathExists bar; import bar; -in + result = mkDerivation { + name = "foo"; + builder = builtins.toFile "builder.sh" + '' + echo -n FOO${toString value} > $out + ''; + }; -mkDerivation { - name = "foo"; - builder = builtins.toFile "builder.sh" - '' - echo -n FOO${toString value} > $out - ''; + addPath = mkDerivation { + name = "add-path"; + src = builtins.filterSource (path: type: true) result; + builder = builtins.toFile "builder.sh" + '' + echo -n BLA$(cat $src) > $out + ''; + }; } diff --git a/tests/functional/import-from-derivation.sh b/tests/functional/import-from-derivation.sh index dc6e80ed2..de94bdd00 100755 --- a/tests/functional/import-from-derivation.sh +++ b/tests/functional/import-from-derivation.sh @@ -4,11 +4,40 @@ source common.sh clearStoreIfPossible -if nix-instantiate --readonly-mode ./import-from-derivation.nix; then +if nix-instantiate --readonly-mode ./import-from-derivation.nix -A result; then echo "read-only evaluation of an imported derivation unexpectedly failed" exit 1 fi -outPath=$(nix-build ./import-from-derivation.nix --no-out-link) +outPath=$(nix-build ./import-from-derivation.nix -A result --no-out-link) [ "$(cat "$outPath")" = FOO579 ] + +# Test filterSource on the result of a derivation. +outPath2=$(nix-build ./import-from-derivation.nix -A addPath --no-out-link) +[[ "$(cat "$outPath2")" = BLAFOO579 ]] + +# Test that IFD works with a chroot store. +if canUseSandbox; then + + store2="$TEST_ROOT/store2" + store2_url="$store2?store=$NIX_STORE_DIR" + + # Copy the derivation outputs to the chroot store to avoid having + # to actually build anything, as that would fail due to the lack + # of a shell in the sandbox. We only care about testing the IFD + # semantics. + for i in bar result addPath; do + nix copy --to "$store2_url" --no-check-sigs "$(nix-build ./import-from-derivation.nix -A $i --no-out-link)" + done + + clearStore + + outPath_check=$(nix-build ./import-from-derivation.nix -A result --no-out-link --store "$store2_url") + [[ "$outPath" = "$outPath_check" ]] + [[ ! -e "$outPath" ]] + [[ -e "$store2/nix/store/$(basename $outPath)" ]] + + outPath2_check=$(nix-build ./import-from-derivation.nix -A addPath --no-out-link --store "$store2_url") + [[ "$outPath2" = "$outPath2_check" ]] +fi From 8105307f0f3caeae1c1d0bca51bee3a417e4304a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikodem=20Rabuli=C5=84ski?= Date: Mon, 6 May 2024 18:16:50 +0200 Subject: [PATCH 19/37] Always initialize curl in parent process on darwin Because of an objc quirk[1], calling curl_global_init for the first time after fork() will always result in a crash. Up until now the solution has been to set OBJC_DISABLE_INITIALIZE_FORK_SAFETY for every nix process to ignore that error. This is less than ideal because we were setting it in package.nix, which meant that running nix tests locally would fail because that variable was not set. Instead of working around that error we address it at the core - by calling curl_global_init inside initLibStore, which should mean curl will already have been initialized by the time we try to do so in a forked process. [1] https://github.com/apple-oss-distributions/objc4/blob/01edf1705fbc3ff78a423cd21e03dfc21eb4d780/runtime/objc-initialize.mm#L614-L636 (cherry-picked and adapted from https://git.lix.systems/lix-project/lix/commit/c7d97802e4f59b8621e67cf62275d6a7fde8fe62) --- misc/launchd/org.nixos.nix-daemon.plist.in | 5 ----- package.nix | 5 ----- src/libstore/globals.cc | 14 +++++++++++++- tests/functional/package.nix | 7 ------- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index e1470cf99..664608305 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -2,11 +2,6 @@ - EnvironmentVariables - - OBJC_DISABLE_INITIALIZE_FORK_SAFETY - YES - Label org.nixos.nix-daemon KeepAlive diff --git a/package.nix b/package.nix index 5c8d9f9b6..8ab184667 100644 --- a/package.nix +++ b/package.nix @@ -325,11 +325,6 @@ in { preInstallCheck = lib.optionalString (! doBuild) '' mkdir -p src/nix-channel - '' - # See https://github.com/NixOS/nix/issues/2523 - # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 - + lib.optionalString stdenv.hostPlatform.isDarwin '' - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8958e6997..b64e73c26 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -12,6 +12,7 @@ #include #include +#include #include #ifndef _WIN32 @@ -363,10 +364,21 @@ void initLibStore(bool loadConfig) { preloadNSS(); + /* Because of an objc quirk[1], calling curl_global_init for the first time + after fork() will always result in a crash. + Up until now the solution has been to set OBJC_DISABLE_INITIALIZE_FORK_SAFETY + for every nix process to ignore that error. + Instead of working around that error we address it at the core - + by calling curl_global_init here, which should mean curl will already + have been initialized by the time we try to do so in a forked process. + + [1] https://github.com/apple-oss-distributions/objc4/blob/01edf1705fbc3ff78a423cd21e03dfc21eb4d780/runtime/objc-initialize.mm#L614-L636 + */ + curl_global_init(CURL_GLOBAL_ALL); +#if __APPLE__ /* On macOS, don't use the per-session TMPDIR (as set e.g. by sshd). This breaks build users because they don't have access to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ if (hasPrefix(defaultTempDir(), "/var/folders/")) unsetenv("TMPDIR"); #endif diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 277711123..675cefa64 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -95,13 +95,6 @@ mkMesonDerivation (finalAttrs: { "--print-errorlogs" ]; - preCheck = - # See https://github.com/NixOS/nix/issues/2523 - # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 - lib.optionalString stdenv.hostPlatform.isDarwin '' - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - ''; - doCheck = true; installPhase = '' From 2b7642632e5d60c6c191a70a5c8cf68903c66140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 18 Sep 2024 08:39:31 +0200 Subject: [PATCH 20/37] devShell: increase priority of clang-tools Before we would get the unwrapped version of clang-tools from clang itself, which doesn't quite work. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 5ca9c1a45..cbcf10021 100644 --- a/flake.nix +++ b/flake.nix @@ -370,7 +370,7 @@ # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear - ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (lib.hiPrio pkgs.buildPackages.clang-tools); buildInputs = attrs.buildInputs or [] ++ [ From f0a4f1908744fc9ad3e2ba3ff6c60cb2d65a56d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 18 Sep 2024 15:39:08 +0200 Subject: [PATCH 21/37] add description + example for nix flake archive Update src/nix/flake-archive.md Update src/nix/flake-archive.md --- src/nix/flake-archive.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/nix/flake-archive.md b/src/nix/flake-archive.md index 85bbeeb16..18c735b11 100644 --- a/src/nix/flake-archive.md +++ b/src/nix/flake-archive.md @@ -22,8 +22,20 @@ R""( # nix flake archive --json --dry-run nixops ``` +* Upload all flake inputs to a different machine for remote evaluation + + ``` + # nix flake archive --to ssh://some-machine + ``` + + On the remote machine the flake can then be accessed via its store path. That's computed like this: + + ``` + # nix flake metadata --json | jq -r '.path' + ``` + # Description -FIXME +Copy a flake and all its inputs to a store. This is useful i.e. to evaluate flakes on a different host. )"" From d772a8b3dc41f689c052662778f488bef44ded49 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Sep 2024 18:05:08 +0200 Subject: [PATCH 22/37] shellcheck --- tests/functional/import-from-derivation.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/import-from-derivation.sh b/tests/functional/import-from-derivation.sh index de94bdd00..c5ed88a1d 100755 --- a/tests/functional/import-from-derivation.sh +++ b/tests/functional/import-from-derivation.sh @@ -28,7 +28,7 @@ if canUseSandbox; then # of a shell in the sandbox. We only care about testing the IFD # semantics. for i in bar result addPath; do - nix copy --to "$store2_url" --no-check-sigs "$(nix-build ./import-from-derivation.nix -A $i --no-out-link)" + nix copy --to "$store2_url" --no-check-sigs "$(nix-build ./import-from-derivation.nix -A "$i" --no-out-link)" done clearStore @@ -36,7 +36,7 @@ if canUseSandbox; then outPath_check=$(nix-build ./import-from-derivation.nix -A result --no-out-link --store "$store2_url") [[ "$outPath" = "$outPath_check" ]] [[ ! -e "$outPath" ]] - [[ -e "$store2/nix/store/$(basename $outPath)" ]] + [[ -e "$store2/nix/store/$(basename "$outPath")" ]] outPath2_check=$(nix-build ./import-from-derivation.nix -A addPath --no-out-link --store "$store2_url") [[ "$outPath2" = "$outPath2_check" ]] From a673084733885c2a4e3126b0eef2a2c1c19e5237 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Sep 2024 19:06:48 +0200 Subject: [PATCH 23/37] Fix tests --- tests/functional/import-from-derivation.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/functional/import-from-derivation.sh b/tests/functional/import-from-derivation.sh index c5ed88a1d..83ef92a6f 100755 --- a/tests/functional/import-from-derivation.sh +++ b/tests/functional/import-from-derivation.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStoreIfPossible if nix-instantiate --readonly-mode ./import-from-derivation.nix -A result; then @@ -13,6 +15,11 @@ outPath=$(nix-build ./import-from-derivation.nix -A result --no-out-link) [ "$(cat "$outPath")" = FOO579 ] +# FIXME: the next tests are broken on CA. +if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + exit 0 +fi + # Test filterSource on the result of a derivation. outPath2=$(nix-build ./import-from-derivation.nix -A addPath --no-out-link) [[ "$(cat "$outPath2")" = BLAFOO579 ]] From 04a47e93f611f08df9f0cadad761a8f5268fff1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 18 Sep 2024 20:32:49 +0200 Subject: [PATCH 24/37] tests/functional/shell: fix test in macOS devshell --- tests/functional/shell.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index c2ac3b24d..04a03eef5 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -31,11 +31,13 @@ env > $TEST_ROOT/expected-env nix shell -f shell-hello.nix hello -c env > $TEST_ROOT/actual-env # Remove/reset variables we expect to be different. # - PATH is modified by nix shell +# - we unset TMPDIR on macOS if it contains /var/folders # - _ is set by bash and is expectedf to differ because it contains the original command # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control sed -i \ -e 's/PATH=.*/PATH=.../' \ -e 's/_=.*/_=.../' \ + -e '/^TMPDIR=\/var\/folders\/.*/d' \ -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ $TEST_ROOT/expected-env $TEST_ROOT/actual-env sort $TEST_ROOT/expected-env > $TEST_ROOT/expected-env.sorted From 5c87c40a5eb11833193d9364f84e25464e4d1044 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 18 Sep 2024 22:42:44 +0200 Subject: [PATCH 25/37] Use close_range when available This fixes the FreeBSD build of nix-util --- configure.ac | 2 +- src/libutil/meson.build | 1 + src/libutil/unix/file-descriptor.cc | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 5c22ed176..198198dea 100644 --- a/configure.ac +++ b/configure.ac @@ -86,7 +86,7 @@ static char buf[1024];]], AC_LANG_POP(C++) -AC_CHECK_FUNCS([statvfs pipe2]) +AC_CHECK_FUNCS([statvfs pipe2 close_range]) # Check for lutimes, optionally used for changing the mtime of diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 797dcae6d..7a058b29c 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -28,6 +28,7 @@ subdir('build-utils-meson/subprojects') # HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 # for found. One therefore uses it with `#if` not `#ifdef`. check_funcs = [ + 'close_range', # Optionally used for changing the mtime of symlinks. 'lutimes', # Optionally used for creating pipes on Unix diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index f867199c0..2c1126e09 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -121,10 +121,13 @@ void Pipe::create() ////////////////////////////////////////////////////////////////////// #if __linux__ || __FreeBSD__ -// In future we can use a syscall wrapper, but at the moment musl and older glibc version don't support it. static int unix_close_range(unsigned int first, unsigned int last, int flags) { +#if !HAVE_CLOSE_RANGE return syscall(SYS_close_range, first, last, (unsigned int)flags); +#else + return close_range(first, last, flags); +#endif } #endif From c75907e47bf653158aa5773112455949cc3e3163 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 18 Sep 2024 23:05:13 +0200 Subject: [PATCH 26/37] Revert "tests.installer: Load profile with -o unset" I must have made a mistake while testing this, because nounset does not work on any of the distributions. This reverts commit 2f0db04da08e67f29577c27fae3c0eb758fc0879. --- tests/installer/default.nix | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 7c82045ad..4aed6eae4 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -217,16 +217,10 @@ let $ssh < Date: Thu, 19 Sep 2024 00:00:17 +0200 Subject: [PATCH 27/37] nix-util / meson: Add -latomic on arm I couldn't get the test program to work correctly after many attempts, so let's just unblock this without making it perfect. --- src/libutil/meson.build | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 797dcae6d..482997096 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -50,6 +50,14 @@ endforeach subdir('build-utils-meson/threads') +# Check if -latomic is needed +# This is needed for std::atomic on some platforms +# We did not manage to test this reliably on all platforms, so we hardcode +# it for now. +if host_machine.cpu_family() == 'arm' + deps_other += cxx.find_library('atomic') +endif + if host_machine.system() == 'windows' socket = cxx.find_library('ws2_32') deps_other += socket From 97fffd8765d61571577b621d14b8a03740367458 Mon Sep 17 00:00:00 2001 From: Noam Yorav-Raphael Date: Thu, 19 Sep 2024 07:20:04 +0300 Subject: [PATCH 28/37] nix-profile.sh.in: fix envvar condition --- scripts/nix-profile.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 3d0e498f4..2d6bf6e95 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,9 +1,9 @@ # This file is tested by tests/installer/default.nix. -if [ -n "$HOME" ] && [ -n "$USER" ]; then +if [ -n "${HOME-}" ] && [ -n "${USER-}" ]; then # Set up the per-user profile. - if [ -n "$NIX_STATE_HOME" ]; then + if [ -n "${NIX_STATE_HOME-}" ]; then NIX_LINK="$NIX_STATE_HOME/profile" else NIX_LINK="$HOME/.nix-profile" From 0c2fdd2f3c0f04bef4b5c74fbb02a5f8227c07df Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2024 19:16:31 +0200 Subject: [PATCH 29/37] Fix missing GC root in zipAttrsWith My SNAFU was that I assumed that all the `Value *`s we put in `attrsSeen` are already reachable (which they are), but I forgot about the `elems` pointer in `ListBuilder`. Fixes #11547. --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9de8ff599..ed1597e5d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3136,7 +3136,7 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg std::optional list; }; - std::map attrsSeen; + std::map, traceable_allocator>> attrsSeen; state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith"); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith"); From 4449b0da744c32cb9cbb06b661a5f5df4444497a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2024 19:52:47 +0200 Subject: [PATCH 30/37] Use HAVE_BOEHMGC Co-authored-by: Robert Hensing --- src/libexpr/primops.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ed1597e5d..8d53a1dfd 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3136,7 +3136,11 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg std::optional list; }; +#if HAVE_BOEHMGC std::map, traceable_allocator>> attrsSeen; +#else + std::map attrsSeen; +#endif state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith"); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith"); From b9f78abb7ff5e9e01ced5b94805ccaa563f46bc6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2024 20:07:04 +0200 Subject: [PATCH 31/37] Alias traceable_allocator to std::allocator when building without GC This allows us to get rid of a bunch of #ifdefs. --- src/libcmd/command.cc | 7 +------ src/libexpr/eval.cc | 4 ---- src/libexpr/eval.hh | 14 +------------- src/libexpr/gc-small-vector.hh | 17 ++--------------- src/libexpr/get-drvs.hh | 4 ---- src/libexpr/primops.cc | 4 ---- src/libexpr/value.hh | 10 ++++------ 7 files changed, 8 insertions(+), 52 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 67fef1909..6d8bfc19b 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -127,14 +127,9 @@ ref EvalCommand::getEvalState() { if (!evalState) { evalState = - #if HAVE_BOEHMGC std::allocate_shared( traceable_allocator(), - #else - std::make_shared( - #endif - lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) - ; + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()); evalState->repair = repair; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2420f15c1..be3bbda22 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -99,11 +99,7 @@ static const char * makeImmutableString(std::string_view s) RootValue allocRootValue(Value * v) { -#if HAVE_BOEHMGC return std::allocate_shared(traceable_allocator(), v); -#else - return std::make_shared(v); -#endif } // Pretty print types for assertion errors diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index da9dd2087..f7ed6be83 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -139,11 +139,7 @@ struct Constant bool impureOnly = false; }; -#if HAVE_BOEHMGC - typedef std::map, traceable_allocator > > ValMap; -#else - typedef std::map ValMap; -#endif +typedef std::map, traceable_allocator > > ValMap; typedef std::unordered_map DocCommentMap; @@ -329,21 +325,13 @@ private: /** * A cache from path names to parse trees. */ -#if HAVE_BOEHMGC typedef std::unordered_map, std::equal_to, traceable_allocator>> FileParseCache; -#else - typedef std::unordered_map FileParseCache; -#endif FileParseCache fileParseCache; /** * A cache from path names to values. */ -#if HAVE_BOEHMGC typedef std::unordered_map, std::equal_to, traceable_allocator>> FileEvalCache; -#else - typedef std::unordered_map FileEvalCache; -#endif FileEvalCache fileEvalCache; /** diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh index 7f4f08fc7..8330dd2dc 100644 --- a/src/libexpr/gc-small-vector.hh +++ b/src/libexpr/gc-small-vector.hh @@ -2,28 +2,15 @@ #include -#if HAVE_BOEHMGC - -#include -#include -#include - -#endif +#include "value.hh" namespace nix { -struct Value; - /** * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap. */ -#if HAVE_BOEHMGC template using SmallVector = boost::container::small_vector>; -#else -template -using SmallVector = boost::container::small_vector; -#endif /** * A vector of value pointers. See `SmallVector`. @@ -39,4 +26,4 @@ using SmallValueVector = SmallVector; template using SmallTemporaryValueVector = SmallVector; -} \ No newline at end of file +} diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index db3eedb05..e4e277af8 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -83,11 +83,7 @@ public: }; -#if HAVE_BOEHMGC typedef std::list> PackageInfos; -#else -typedef std::list PackageInfos; -#endif /** diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8d53a1dfd..ed1597e5d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3136,11 +3136,7 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg std::optional list; }; -#if HAVE_BOEHMGC std::map, traceable_allocator>> attrsSeen; -#else - std::map attrsSeen; -#endif state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith"); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith"); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index f68befe0e..087406965 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -12,7 +12,11 @@ #if HAVE_BOEHMGC #include +#else +template +using traceable_allocator = std::allocator; #endif + #include namespace nix { @@ -498,15 +502,9 @@ void Value::mkBlackhole() } -#if HAVE_BOEHMGC typedef std::vector> ValueVector; typedef std::unordered_map, std::equal_to, traceable_allocator>> ValueMap; typedef std::map, traceable_allocator>> ValueVectorMap; -#else -typedef std::vector ValueVector; -typedef std::unordered_map ValueMap; -typedef std::map ValueVectorMap; -#endif /** From 31d408c351e5a98b8016e1fb6ca8348814145d54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2024 20:45:13 +0200 Subject: [PATCH 32/37] Alias gc_allocator --- src/libexpr/primops.cc | 4 ---- src/libexpr/value.hh | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ed1597e5d..7b6f222a8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -631,11 +631,7 @@ struct CompareValues }; -#if HAVE_BOEHMGC typedef std::list> ValueList; -#else -typedef std::list ValueList; -#endif static Bindings::const_iterator getAttr( diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 087406965..a837ac133 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -15,6 +15,9 @@ #else template using traceable_allocator = std::allocator; + +template +using gc_allocator = std::allocator; #endif #include From 589d8f1f2be4c036f459288e4a90be30371bbf41 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2024 20:54:30 +0200 Subject: [PATCH 33/37] Move GC-related definitions to eval-gc.hh --- src/libcmd/repl.cc | 5 ----- src/libexpr-c/nix_api_external.cc | 6 ------ src/libexpr-c/nix_api_value.cc | 6 ------ src/libexpr/eval-gc.hh | 18 ++++++++++++++++++ src/libexpr/eval.cc | 11 ----------- src/libexpr/value.hh | 11 +---------- src/nix/main.cc | 1 - 7 files changed, 19 insertions(+), 39 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 46b6fbadc..531038321 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -29,11 +29,6 @@ #include "ref.hh" #include "value.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif - #include "strings.hh" namespace nix { diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index fa78eb5df..d673bcb0b 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -14,12 +14,6 @@ #include -#if HAVE_BOEHMGC -# include "gc/gc.h" -# define GC_INCLUDE_NEW 1 -# include "gc_cpp.h" -#endif - void nix_set_string_return(nix_string_return * str, const char * c) { str->str = c; diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index fa2a9cbe2..bae078d31 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -14,12 +14,6 @@ #include "nix_api_value.h" #include "value/context.hh" -#if HAVE_BOEHMGC -# include "gc/gc.h" -# define GC_INCLUDE_NEW 1 -# include "gc_cpp.h" -#endif - // Internal helper functions to check [in] and [out] `Value *` parameters static const nix::Value & check_value_not_null(const nix_value * value) { diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index 005175eb7..584365844 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -3,6 +3,24 @@ #include +#if HAVE_BOEHMGC + +# define GC_INCLUDE_NEW + +# include +# include +# include + +#else + +template +using traceable_allocator = std::allocator; + +template +using gc_allocator = std::allocator; + +#endif + namespace nix { /** diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index be3bbda22..5952ebe41 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,5 +1,4 @@ #include "eval.hh" -#include "eval-gc.hh" #include "eval-settings.hh" #include "primops.hh" #include "print-options.hh" @@ -39,16 +38,6 @@ # include #endif -#if HAVE_BOEHMGC - -# define GC_INCLUDE_NEW - -# include -# include -# include - -#endif - #include "strings-inline.hh" using json = nlohmann::json; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index a837ac133..0ffe74dab 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -4,22 +4,13 @@ #include #include +#include "eval-gc.hh" #include "symbol-table.hh" #include "value/context.hh" #include "source-path.hh" #include "print-options.hh" #include "checked-arithmetic.hh" -#if HAVE_BOEHMGC -#include -#else -template -using traceable_allocator = std::allocator; - -template -using gc_allocator = std::allocator; -#endif - #include namespace nix { diff --git a/src/nix/main.cc b/src/nix/main.cc index 34de79ac8..7a9516d5e 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -2,7 +2,6 @@ #include "current-process.hh" #include "command.hh" #include "common-args.hh" -#include "eval-gc.hh" #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" From 2f4a7a830135e52827b112e4f46a0f46a78dac64 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2024 21:03:59 +0200 Subject: [PATCH 34/37] Add a few more aliases --- src/libcmd/repl.cc | 2 -- src/libexpr/eval-gc.hh | 7 +++++++ src/libexpr/eval.cc | 8 -------- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 531038321..940b16dfd 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -57,9 +57,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl , detail::ReplCompleterMixin - #if HAVE_BOEHMGC , gc - #endif { size_t debugTraceIndex; diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index 584365844..8f492c56d 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -13,12 +13,19 @@ #else +/* Some dummy aliases for Boehm GC definitions to reduce the number of + #ifdefs. */ + template using traceable_allocator = std::allocator; template using gc_allocator = std::allocator; +#define GC_MALLOC_ATOMIC std::malloc +#define GC_STRDUP std::strdup +struct gc { }; + #endif namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5952ebe41..379839ce3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -47,11 +47,7 @@ namespace nix { static char * allocString(size_t size) { char * t; -#if HAVE_BOEHMGC t = (char *) GC_MALLOC_ATOMIC(size); -#else - t = (char *) malloc(size); -#endif if (!t) throw std::bad_alloc(); return t; } @@ -60,11 +56,7 @@ static char * allocString(size_t size) static char * dupString(const char * s) { char * t; -#if HAVE_BOEHMGC t = GC_STRDUP(s); -#else - t = strdup(s); -#endif if (!t) throw std::bad_alloc(); return t; } From b2bb92ef09e380c3a706efe01ef68d7c7803f1aa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2024 22:09:49 +0200 Subject: [PATCH 35/37] Formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- src/libexpr/eval-gc.hh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index 8f492c56d..1be9ff65b 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -22,9 +22,10 @@ using traceable_allocator = std::allocator; template using gc_allocator = std::allocator; -#define GC_MALLOC_ATOMIC std::malloc -#define GC_STRDUP std::strdup -struct gc { }; +# define GC_MALLOC_ATOMIC std::malloc +# define GC_STRDUP std::strdup +struct gc +{}; #endif From 088569463b9de59ce7928ca5a9b4a2413533444a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2024 15:01:32 +0200 Subject: [PATCH 36/37] Fix build without GC --- src/libexpr/eval-gc.hh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index 1be9ff65b..3b420e418 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -23,7 +23,8 @@ template using gc_allocator = std::allocator; # define GC_MALLOC_ATOMIC std::malloc -# define GC_STRDUP std::strdup +# define GC_STRDUP strdup + struct gc {}; From ec47133be343bfdf2aad93dc75b5943f5667f1b0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2024 15:08:45 +0200 Subject: [PATCH 37/37] Fix warning --- src/libexpr-c/nix_api_expr.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 8f21d7022..333e99460 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -16,8 +16,6 @@ #if HAVE_BOEHMGC # include -# define GC_INCLUDE_NEW 1 -# include "gc_cpp.h" #endif nix_err nix_libexpr_init(nix_c_context * context)