diff --git a/src/libfetchers/git-lfs-fetch.hh b/src/libfetchers/git-lfs-fetch.hh index b2844771e..d202d8a59 100644 --- a/src/libfetchers/git-lfs-fetch.hh +++ b/src/libfetchers/git-lfs-fetch.hh @@ -122,7 +122,7 @@ struct Fetch void init(git_repository * repo, const std::string & gitattributesContent); bool hasAttribute(const std::string & path, const std::string & attrName, const std::string & attrValue) const; - void fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink) const; + void fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink, std::function sizeCallback) const; std::vector fetchUrls(const std::vector & metadatas) const; }; @@ -155,7 +155,7 @@ static size_t sinkWriteCallback(void * contents, size_t size, size_t nmemb, Sink return totalSize; } -// if authHeader is "", downloadToSink assumes to auth is expected +// if authHeader is "", downloadToSink assumes no auth is expected void downloadToSink( const std::string & url, const std::string & authHeader, Sink & sink, std::string_view sha256Expected) { @@ -522,29 +522,30 @@ std::vector Fetch::fetchUrls(const std::vector & metadatas) } } -void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink) const +void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFilePath, Sink & sink, std::function sizeCallback) const { constexpr git_object_size_t chunkSize = 128 * 1024; // 128 KiB - auto size = git_blob_rawsize(pointerBlob); + auto pointerSize = git_blob_rawsize(pointerBlob); - if (size >= 1024) { + if (pointerSize >= 1024) { debug("Skip git-lfs, pointer file too large"); warn("Encountered a file that should have been a pointer, but wasn't: %s", pointerFilePath); - for (git_object_size_t offset = 0; offset < size; offset += chunkSize) { - sink(std::string( - (const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, size - offset))); + sizeCallback(pointerSize); + for (git_object_size_t offset = 0; offset < pointerSize; offset += chunkSize) { + sink(std::string((const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, pointerSize - offset))); } return; } - const auto pointerFileContents = std::string((const char *) git_blob_rawcontent(pointerBlob), size); + const auto pointerFileContents = std::string((const char *) git_blob_rawcontent(pointerBlob), pointerSize); const auto md = parseLfsMetadata(std::string(pointerFileContents), std::string(pointerFilePath)); if (md == std::nullopt) { debug("Skip git-lfs, invalid pointer file"); warn("Encountered a file that should have been a pointer, but wasn't: %s", pointerFilePath); - for (git_object_size_t offset = 0; offset < size; offset += chunkSize) { + sizeCallback(pointerSize); + for (git_object_size_t offset = 0; offset < pointerSize; offset += chunkSize) { sink(std::string( - (const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, size - offset))); + (const char *) git_blob_rawcontent(pointerBlob) + offset, std::min(chunkSize, pointerSize - offset))); } return; } @@ -562,8 +563,9 @@ void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFileP && obj.at("actions").at("download").at("header").contains("Authorization")) { authHeader = obj["actions"]["download"]["header"]["Authorization"]; } - // oid is also the sha256 - downloadToSink(ourl, authHeader, sink, oid); + const uint64_t size = obj.at("size"); + sizeCallback(size); + downloadToSink(ourl, authHeader, sink, oid); // oid is also the sha256 } catch (const nlohmann::json::out_of_range & e) { std::stringstream ss; ss << "bad json from /info/lfs/objects/batch: " << obj << " " << e.what(); @@ -573,4 +575,5 @@ void Fetch::fetch(const git_blob * pointerBlob, const std::string & pointerFileP } // namespace lfs + } // namespace nix diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 4a78e803c..8a4a544ce 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -695,7 +695,12 @@ struct GitSourceAccessor : SourceAccessor auto pathStr = std::string(path.rel()); if (_lfsFetch.hasAttribute(pathStr, "filter", "lfs")) { StringSink s; - _lfsFetch.fetch(blob.get(), pathStr, s); + try { + _lfsFetch.fetch(blob.get(), pathStr, s, [&s](uint64_t size){ s.s.reserve(size); }); + } catch (Error &e) { + e.addTrace({}, "while smudging git-lfs file '%s' (std::string interface)", pathStr); + throw; + } return s.s; } } @@ -708,8 +713,6 @@ struct GitSourceAccessor : SourceAccessor Sink & sink, std::function sizeCallback = [](uint64_t size){}) override { auto blob = getBlob(path, false); - auto size = git_blob_rawsize(blob.get()); - sizeCallback(size); if (lfsFetch && path != CanonPath(".gitattributes") && lookup(CanonPath(".gitattributes"))) { auto& _lfsFetch = *lfsFetch; @@ -720,11 +723,19 @@ struct GitSourceAccessor : SourceAccessor auto pathStr = std::string(path.rel()); if (_lfsFetch.hasAttribute(pathStr, "filter", "lfs")) { - _lfsFetch.fetch(blob.get(), pathStr, sink); + try { + _lfsFetch.fetch(blob.get(), pathStr, sink, sizeCallback); + } catch (Error &e) { + e.addTrace({}, "while smudging git-lfs file '%s' (callback interface)", pathStr); + throw; + } return; } } + // lfs disabled or does not apply to this path + auto size = git_blob_rawsize(blob.get()); + sizeCallback(size); constexpr git_object_size_t chunkSize = 128 * 1024; // 128 KiB for (git_object_size_t offset = 0; offset < size; offset += chunkSize) { sink(std::string((const char *) git_blob_rawcontent(blob.get()) + offset, std::min(chunkSize, size - offset)));