wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 14:51:16 +02:00
Code Activity

fix sizeCallback

Browse source
This commit is contained in:
Brian Camacho 2024-12-02 14:53:19 -05:00
parent 85d6efb40d
commit 79a6438c68
2 changed files with 31 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

29
src/libfetchers/git-lfs-fetch.hh
View file

@ -122,7 +122,7 @@ struct Fetch
void init(git_repository * repo, const std::string & gitattributesContent); void init(git_repository * repo, const std::string & gitattributesContent);
bool hasAttribute(const std::string & path, const std::string & attrName, const std::string & attrValue) const; bool hasAttribute(const std::string & path, const std::string & attrName, const std::string & attrValue) const;
void fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink) const; void fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink, std::function<void(uint64_t)> sizeCallback) const;
std::vector<nlohmann::json> fetchUrls(const std::vector<Md> & metadatas) const; std::vector<nlohmann::json> fetchUrls(const std::vector<Md> & metadatas) const;
}; };
@ -155,7 +155,7 @@ static size_t sinkWriteCallback(void * contents, size_t size, size_t nmemb, Sink
return totalSize; return totalSize;
} }
// if authHeader is "", downloadToSink assumes to auth is expected // if authHeader is "", downloadToSink assumes no auth is expected
void downloadToSink( void downloadToSink(
const std::string & url, const std::string & authHeader, Sink & sink, std::string_view sha256Expected) const std::string & url, const std::string & authHeader, Sink & sink, std::string_view sha256Expected)
{ {
@ -522,29 +522,30 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Md> & metadatas)
} }
} }
void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink) const void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink, std::function<void(uint64_t)> sizeCallback) const
{ {
constexpr git_object_size_t chunkSize = 128 * 1024; // 128 KiB constexpr git_object_size_t chunkSize = 128 * 1024; // 128 KiB
auto size = git_blob_rawsize(pointerBlob); auto pointerSize = git_blob_rawsize(pointerBlob);
if (size >= 1024) { if (pointerSize >= 1024) {
debug("Skip git-lfs, pointer file too large"); debug("Skip git-lfs, pointer file too large");
warn("Encountered a file that should have been a pointer, but wasn't: %s", pointerFilePath); warn("Encountered a file that should have been a pointer, but wasn't: %s", pointerFilePath);
for (git_object_size_t offset = 0; offset < size; offset += chunkSize) { sizeCallback(pointerSize);
sink(std::string( for (git_object_size_t offset = 0; offset < pointerSize; offset += chunkSize) {
(const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, size - offset))); sink(std::string((const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, pointerSize - offset)));
} }
return; return;
} }
const auto pointerFileContents = std::string((const char *) git_blob_rawcontent(pointerBlob), size); const auto pointerFileContents = std::string((const char *) git_blob_rawcontent(pointerBlob), pointerSize);
const auto md = parseLfsMetadata(std::string(pointerFileContents), std::string(pointerFilePath)); const auto md = parseLfsMetadata(std::string(pointerFileContents), std::string(pointerFilePath));
if (md == std::nullopt) { if (md == std::nullopt) {
debug("Skip git-lfs, invalid pointer file"); debug("Skip git-lfs, invalid pointer file");
warn("Encountered a file that should have been a pointer, but wasn't: %s", pointerFilePath); warn("Encountered a file that should have been a pointer, but wasn't: %s", pointerFilePath);
for (git_object_size_t offset = 0; offset < size; offset += chunkSize) { sizeCallback(pointerSize);
for (git_object_size_t offset = 0; offset < pointerSize; offset += chunkSize) {
sink(std::string( sink(std::string(
(const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, size - offset))); (const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, pointerSize - offset)));
} }
return; return;
} }
@ -562,8 +563,9 @@ void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFileP
&& obj.at("actions").at("download").at("header").contains("Authorization")) { && obj.at("actions").at("download").at("header").contains("Authorization")) {
authHeader = obj["actions"]["download"]["header"]["Authorization"]; authHeader = obj["actions"]["download"]["header"]["Authorization"];
} }
// oid is also the sha256 const uint64_t size = obj.at("size");
downloadToSink(ourl, authHeader, sink, oid); sizeCallback(size);
downloadToSink(ourl, authHeader, sink, oid); // oid is also the sha256
} catch (const nlohmann::json::out_of_range & e) { } catch (const nlohmann::json::out_of_range & e) {
std::stringstream ss; std::stringstream ss;
ss << "bad json from /info/lfs/objects/batch: " << obj << " " << e.what(); ss << "bad json from /info/lfs/objects/batch: " << obj << " " << e.what();
@ -573,4 +575,5 @@ void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFileP
} // namespace lfs } // namespace lfs
} // namespace nix } // namespace nix

19
src/libfetchers/git-utils.cc
View file

@ -695,7 +695,12 @@ struct GitSourceAccessor : SourceAccessor
auto pathStr = std::string(path.rel()); auto pathStr = std::string(path.rel());
if (_lfsFetch.hasAttribute(pathStr, "filter", "lfs")) { if (_lfsFetch.hasAttribute(pathStr, "filter", "lfs")) {
StringSink s; StringSink s;
_lfsFetch.fetch(blob.get(), pathStr, s); try {
_lfsFetch.fetch(blob.get(), pathStr, s, [&s](uint64_t size){ s.s.reserve(size); });
} catch (Error &e) {
e.addTrace({}, "while smudging git-lfs file '%s' (std::string interface)", pathStr);
throw;
}
return s.s; return s.s;
} }
} }
@ -708,8 +713,6 @@ struct GitSourceAccessor : SourceAccessor
Sink & sink, Sink & sink,
std::function<void(uint64_t)> sizeCallback = [](uint64_t size){}) override { std::function<void(uint64_t)> sizeCallback = [](uint64_t size){}) override {
auto blob = getBlob(path, false); auto blob = getBlob(path, false);
auto size = git_blob_rawsize(blob.get());
sizeCallback(size);
if (lfsFetch && path != CanonPath(".gitattributes") && lookup(CanonPath(".gitattributes"))) { if (lfsFetch && path != CanonPath(".gitattributes") && lookup(CanonPath(".gitattributes"))) {
auto& _lfsFetch = *lfsFetch; auto& _lfsFetch = *lfsFetch;
@ -720,11 +723,19 @@ struct GitSourceAccessor : SourceAccessor
auto pathStr = std::string(path.rel()); auto pathStr = std::string(path.rel());
if (_lfsFetch.hasAttribute(pathStr, "filter", "lfs")) { if (_lfsFetch.hasAttribute(pathStr, "filter", "lfs")) {
_lfsFetch.fetch(blob.get(), pathStr, sink); try {
_lfsFetch.fetch(blob.get(), pathStr, sink, sizeCallback);
} catch (Error &e) {
e.addTrace({}, "while smudging git-lfs file '%s' (callback interface)", pathStr);
throw;
}
return; return;
} }
} }
// lfs disabled or does not apply to this path
auto size = git_blob_rawsize(blob.get());
sizeCallback(size);
constexpr git_object_size_t chunkSize = 128 * 1024; // 128 KiB constexpr git_object_size_t chunkSize = 128 * 1024; // 128 KiB
for (git_object_size_t offset = 0; offset < size; offset += chunkSize) { for (git_object_size_t offset = 0; offset < size; offset += chunkSize) {
sink(std::string((const char *) git_blob_rawcontent(blob.get()) + offset, std::min(chunkSize, size - offset))); sink(std::string((const char *) git_blob_rawcontent(blob.get()) + offset, std::min(chunkSize, size - offset)));