wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 10:41:16 +02:00
Code Activity

fix sizeCallback

Browse source
This commit is contained in:
Brian Camacho 2024-12-02 14:53:19 -05:00
parent 85d6efb40d
commit 79a6438c68
2 changed files with 31 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

29
src/libfetchers/git-lfs-fetch.hh
View file

@ -122,7 +122,7 @@ struct Fetch
void init(git_repository * repo, const std::string & gitattributesContent);
bool hasAttribute(const std::string & path, const std::string & attrName, const std::string & attrValue) const;
void fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink) const;
void fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink, std::function<void(uint64_t)> sizeCallback) const;
std::vector<nlohmann::json> fetchUrls(const std::vector<Md> & metadatas) const;
};
@ -155,7 +155,7 @@ static size_t sinkWriteCallback(void * contents, size_t size, size_t nmemb, Sink
return totalSize;
}
// if authHeader is "", downloadToSink assumes to auth is expected
// if authHeader is "", downloadToSink assumes no auth is expected
void downloadToSink(
const std::string & url, const std::string & authHeader, Sink & sink, std::string_view sha256Expected)
{
@ -522,29 +522,30 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Md> & metadatas)
}
}
void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink) const
void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink, std::function<void(uint64_t)> sizeCallback) const
{
constexpr git_object_size_t chunkSize = 128 * 1024; // 128 KiB
auto size = git_blob_rawsize(pointerBlob);
auto pointerSize = git_blob_rawsize(pointerBlob);
if (size >= 1024) {
if (pointerSize >= 1024) {
debug("Skip git-lfs, pointer file too large");
warn("Encountered a file that should have been a pointer, but wasn't: %s", pointerFilePath);
for (git_object_size_t offset = 0; offset < size; offset += chunkSize) {
sink(std::string(
(const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, size - offset)));
sizeCallback(pointerSize);
for (git_object_size_t offset = 0; offset < pointerSize; offset += chunkSize) {
sink(std::string((const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, pointerSize - offset)));
}
return;
}
const auto pointerFileContents = std::string((const char *) git_blob_rawcontent(pointerBlob), size);
const auto pointerFileContents = std::string((const char *) git_blob_rawcontent(pointerBlob), pointerSize);
const auto md = parseLfsMetadata(std::string(pointerFileContents), std::string(pointerFilePath));
if (md == std::nullopt) {
debug("Skip git-lfs, invalid pointer file");
warn("Encountered a file that should have been a pointer, but wasn't: %s", pointerFilePath);
for (git_object_size_t offset = 0; offset < size; offset += chunkSize) {
sizeCallback(pointerSize);
for (git_object_size_t offset = 0; offset < pointerSize; offset += chunkSize) {
sink(std::string(
(const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, size - offset)));
(const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, pointerSize - offset)));
}
return;
}
@ -562,8 +563,9 @@ void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFileP
&& obj.at("actions").at("download").at("header").contains("Authorization")) {
authHeader = obj["actions"]["download"]["header"]["Authorization"];
}
// oid is also the sha256
downloadToSink(ourl, authHeader, sink, oid);
const uint64_t size = obj.at("size");
sizeCallback(size);
downloadToSink(ourl, authHeader, sink, oid); // oid is also the sha256
} catch (const nlohmann::json::out_of_range & e) {
std::stringstream ss;
ss << "bad json from /info/lfs/objects/batch: " << obj << " " << e.what();
@ -573,4 +575,5 @@ void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFileP
} // namespace lfs
} // namespace nix

19
src/libfetchers/git-utils.cc
View file

@ -695,7 +695,12 @@ struct GitSourceAccessor : SourceAccessor
auto pathStr = std::string(path.rel());
if (_lfsFetch.hasAttribute(pathStr, "filter", "lfs")) {
StringSink s;
_lfsFetch.fetch(blob.get(), pathStr, s);
try {
_lfsFetch.fetch(blob.get(), pathStr, s, [&s](uint64_t size){ s.s.reserve(size); });
} catch (Error &e) {
e.addTrace({}, "while smudging git-lfs file '%s' (std::string interface)", pathStr);
throw;
}
return s.s;
}
}
@ -708,8 +713,6 @@ struct GitSourceAccessor : SourceAccessor
Sink & sink,
std::function<void(uint64_t)> sizeCallback = [](uint64_t size){}) override {
auto blob = getBlob(path, false);
auto size = git_blob_rawsize(blob.get());
sizeCallback(size);
if (lfsFetch && path != CanonPath(".gitattributes") && lookup(CanonPath(".gitattributes"))) {
auto& _lfsFetch = *lfsFetch;
@ -720,11 +723,19 @@ struct GitSourceAccessor : SourceAccessor
auto pathStr = std::string(path.rel());
if (_lfsFetch.hasAttribute(pathStr, "filter", "lfs")) {
_lfsFetch.fetch(blob.get(), pathStr, sink);
try {
_lfsFetch.fetch(blob.get(), pathStr, sink, sizeCallback);
} catch (Error &e) {
e.addTrace({}, "while smudging git-lfs file '%s' (callback interface)", pathStr);
throw;
}
return;
}
}
// lfs disabled or does not apply to this path
auto size = git_blob_rawsize(blob.get());
sizeCallback(size);
constexpr git_object_size_t chunkSize = 128 * 1024; // 128 KiB
for (git_object_size_t offset = 0; offset < size; offset += chunkSize) {
sink(std::string((const char *) git_blob_rawcontent(blob.get()) + offset, std::min(chunkSize, size - offset)));