Merge pull request #10153 from b-camacho/lfs

git-lfs support

src/libexpr/primops/fetchTree.cc

@@ -367,6 +367,12 @@ static RegisterPrimOp primop_fetchTree({
 
           Default: `false`
 
+        - `lfs` (Bool, optional)
+
+          Fetch any [Git LFS](https://git-lfs.com/) files.
+
+          Default: `false`
+
         - `allRefs` (Bool, optional)
 
           By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
@@ -691,6 +697,13 @@ static RegisterPrimOp primop_fetchGit({
 
         Make a shallow clone when fetching the Git tree.
         When this is enabled, the options `ref` and `allRefs` have no effect anymore.
+
+      - `lfs` (default: `false`)
+
+        A boolean that when `true` specifies that [Git LFS] files should be fetched.
+
+        [Git LFS]: https://git-lfs.com/
+
       - `allRefs`
 
         Whether to fetch all references (eg. branches and tags) of the repository.

src/libfetchers-tests/git-utils.cc

@@ -7,6 +7,7 @@
 #include <gtest/gtest.h>
 #include "fs-sink.hh"
 #include "serialise.hh"
+#include "git-lfs-fetch.hh"
 
 namespace nix {
 
@@ -109,4 +110,131 @@ TEST_F(GitUtilsTest, sink_hardlink)
     }
 };
 
+namespace lfs {
+
+TEST_F(GitUtilsTest, parseGitRemoteUrl)
+{
+    {
+        GitUrl result = parseGitUrl("git@example.com:path/repo.git");
+        EXPECT_EQ(result.protocol, "ssh");
+        EXPECT_EQ(result.user, "git");
+        EXPECT_EQ(result.host, "example.com");
+        EXPECT_EQ(result.port, "");
+        EXPECT_EQ(result.path, "path/repo.git");
+    }
+
+    {
+        GitUrl result = parseGitUrl("example.com:/path/repo.git");
+        EXPECT_EQ(result.protocol, "ssh");
+        EXPECT_EQ(result.user, "");
+        EXPECT_EQ(result.host, "example.com");
+        EXPECT_EQ(result.port, "");
+        EXPECT_EQ(result.path, "/path/repo.git");
+    }
+
+    {
+        GitUrl result = parseGitUrl("example.com:path/repo.git");
+        EXPECT_EQ(result.protocol, "ssh");
+        EXPECT_EQ(result.user, "");
+        EXPECT_EQ(result.host, "example.com");
+        EXPECT_EQ(result.port, "");
+        EXPECT_EQ(result.path, "path/repo.git");
+    }
+
+    {
+        GitUrl result = parseGitUrl("https://example.com/path/repo.git");
+        EXPECT_EQ(result.protocol, "https");
+        EXPECT_EQ(result.user, "");
+        EXPECT_EQ(result.host, "example.com");
+        EXPECT_EQ(result.port, "");
+        EXPECT_EQ(result.path, "path/repo.git");
+    }
+
+    {
+        GitUrl result = parseGitUrl("ssh://git@example.com/path/repo.git");
+        EXPECT_EQ(result.protocol, "ssh");
+        EXPECT_EQ(result.user, "git");
+        EXPECT_EQ(result.host, "example.com");
+        EXPECT_EQ(result.port, "");
+        EXPECT_EQ(result.path, "path/repo.git");
+    }
+
+    {
+        GitUrl result = parseGitUrl("ssh://example/path/repo.git");
+        EXPECT_EQ(result.protocol, "ssh");
+        EXPECT_EQ(result.user, "");
+        EXPECT_EQ(result.host, "example");
+        EXPECT_EQ(result.port, "");
+        EXPECT_EQ(result.path, "path/repo.git");
+    }
+
+    {
+        GitUrl result = parseGitUrl("http://example.com:8080/path/repo.git");
+        EXPECT_EQ(result.protocol, "http");
+        EXPECT_EQ(result.user, "");
+        EXPECT_EQ(result.host, "example.com");
+        EXPECT_EQ(result.port, "8080");
+        EXPECT_EQ(result.path, "path/repo.git");
+    }
+
+    {
+        GitUrl result = parseGitUrl("invalid-url");
+        EXPECT_EQ(result.protocol, "");
+        EXPECT_EQ(result.user, "");
+        EXPECT_EQ(result.host, "");
+        EXPECT_EQ(result.port, "");
+        EXPECT_EQ(result.path, "");
+    }
+
+    {
+        GitUrl result = parseGitUrl("");
+        EXPECT_EQ(result.protocol, "");
+        EXPECT_EQ(result.user, "");
+        EXPECT_EQ(result.host, "");
+        EXPECT_EQ(result.port, "");
+        EXPECT_EQ(result.path, "");
+    }
+}
+TEST_F(GitUtilsTest, gitUrlToHttp)
+{
+    {
+        const GitUrl url = parseGitUrl("git@github.com:user/repo.git");
+        EXPECT_EQ(url.toHttp(), "https://github.com/user/repo.git");
+    }
+    {
+        const GitUrl url = parseGitUrl("https://github.com/user/repo.git");
+        EXPECT_EQ(url.toHttp(), "https://github.com/user/repo.git");
+    }
+    {
+        const GitUrl url = parseGitUrl("http://github.com/user/repo.git");
+        EXPECT_EQ(url.toHttp(), "http://github.com/user/repo.git");
+    }
+    {
+        const GitUrl url = parseGitUrl("ssh://git@github.com:22/user/repo.git");
+        EXPECT_EQ(url.toHttp(), "https://github.com:22/user/repo.git");
+    }
+    {
+        const GitUrl url = parseGitUrl("invalid-url");
+        EXPECT_EQ(url.toHttp(), "");
+    }
+}
+
+TEST_F(GitUtilsTest, gitUrlToSsh)
+{
+    {
+        const GitUrl url = parseGitUrl("https://example.com/user/repo.git");
+        const auto [host, path] = url.toSsh();
+        EXPECT_EQ(host, "example.com");
+        EXPECT_EQ(path, "user/repo.git");
+    }
+    {
+        const GitUrl url = parseGitUrl("git@example.com:user/repo.git");
+        const auto [host, path] = url.toSsh();
+        EXPECT_EQ(host, "git@example.com");
+        EXPECT_EQ(path, "user/repo.git");
+    }
+}
+
+} // namespace lfs
+
 } // namespace nix

src/libfetchers/git-lfs-fetch.cc

@@ -0,0 +1,279 @@
+#include "git-lfs-fetch.hh"
+#include "git-utils.hh"
+#include "filetransfer.hh"
+#include "processes.hh"
+#include "url.hh"
+#include "users.hh"
+#include "hash.hh"
+
+#include <git2/attr.h>
+#include <git2/config.h>
+#include <git2/errors.h>
+#include <git2/remote.h>
+
+#include <nlohmann/json.hpp>
+
+namespace nix::lfs {
+
+// if authHeader is "", downloadToSink assumes no auth is expected
+static void downloadToSink(
+    const std::string & url,
+    const std::string & authHeader,
+    // FIXME: passing a StringSink is superfluous, we may as well
+    // return a string. Or use an abstract Sink for streaming.
+    StringSink & sink,
+    std::string sha256Expected,
+    size_t sizeExpected)
+{
+    FileTransferRequest request(url);
+    Headers headers;
+    if (!authHeader.empty())
+        headers.push_back({"Authorization", authHeader});
+    request.headers = headers;
+    getFileTransfer()->download(std::move(request), sink);
+
+    auto sizeActual = sink.s.length();
+    if (sizeExpected != sizeActual)
+        throw Error("size mismatch while fetching %s: expected %d but got %d", url, sizeExpected, sizeActual);
+
+    auto sha256Actual = hashString(HashAlgorithm::SHA256, sink.s).to_string(HashFormat::Base16, false);
+    if (sha256Actual != sha256Expected)
+        throw Error(
+            "hash mismatch while fetching %s: expected sha256:%s but got sha256:%s", url, sha256Expected, sha256Actual);
+}
+
+static std::string getLfsApiToken(const ParsedURL & url)
+{
+    auto [status, output] = runProgram(RunOptions{
+        .program = "ssh",
+        .args = {*url.authority, "git-lfs-authenticate", url.path, "download"},
+    });
+
+    if (output.empty())
+        throw Error(
+            "git-lfs-authenticate: no output (cmd: ssh %s git-lfs-authenticate %s download)",
+            url.authority.value_or(""),
+            url.path);
+
+    auto queryResp = nlohmann::json::parse(output);
+    if (!queryResp.contains("header"))
+        throw Error("no header in git-lfs-authenticate response");
+    if (!queryResp["header"].contains("Authorization"))
+        throw Error("no Authorization in git-lfs-authenticate response");
+
+    return queryResp["header"]["Authorization"].get<std::string>();
+}
+
+typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
+typedef std::unique_ptr<git_config_entry, Deleter<git_config_entry_free>> GitConfigEntry;
+
+static std::string getLfsEndpointUrl(git_repository * repo)
+{
+    GitConfig config;
+    if (git_repository_config(Setter(config), repo)) {
+        GitConfigEntry entry;
+        if (!git_config_get_entry(Setter(entry), config.get(), "lfs.url")) {
+            auto value = std::string(entry->value);
+            if (!value.empty()) {
+                debug("Found explicit lfs.url value: %s", value);
+                return value;
+            }
+        }
+    }
+
+    git_remote * remote = nullptr;
+    if (git_remote_lookup(&remote, repo, "origin"))
+        return "";
+
+    const char * url_c_str = git_remote_url(remote);
+    if (!url_c_str)
+        return "";
+
+    return std::string(url_c_str);
+}
+
+static std::optional<Pointer> parseLfsPointer(std::string_view content, std::string_view filename)
+{
+    // https://github.com/git-lfs/git-lfs/blob/2ef4108/docs/spec.md
+    //
+    // example git-lfs pointer file:
+    // version https://git-lfs.github.com/spec/v1
+    // oid sha256:f5e02aa71e67f41d79023a128ca35bad86cf7b6656967bfe0884b3a3c4325eaf
+    // size 10000000
+    // (ending \n)
+
+    if (!content.starts_with("version ")) {
+        // Invalid pointer file
+        return std::nullopt;
+    }
+
+    if (!content.starts_with("version https://git-lfs.github.com/spec/v1")) {
+        // In case there's new spec versions in the future, but for now only v1 exists
+        debug("Invalid version found on potential lfs pointer file, skipping");
+        return std::nullopt;
+    }
+
+    std::string oid;
+    std::string size;
+
+    for (auto & line : tokenizeString<Strings>(content, "\n")) {
+        if (line.starts_with("version ")) {
+            continue;
+        }
+        if (line.starts_with("oid sha256:")) {
+            oid = line.substr(11); // skip "oid sha256:"
+            continue;
+        }
+        if (line.starts_with("size ")) {
+            size = line.substr(5); // skip "size "
+            continue;
+        }
+
+        debug("Custom extension '%s' found, ignoring", line);
+    }
+
+    if (oid.length() != 64 || !std::all_of(oid.begin(), oid.end(), ::isxdigit)) {
+        debug("Invalid sha256 %s, skipping", oid);
+        return std::nullopt;
+    }
+
+    if (size.length() == 0 || !std::all_of(size.begin(), size.end(), ::isdigit)) {
+        debug("Invalid size %s, skipping", size);
+        return std::nullopt;
+    }
+
+    return std::make_optional(Pointer{oid, std::stoul(size)});
+}
+
+Fetch::Fetch(git_repository * repo, git_oid rev)
+{
+    this->repo = repo;
+    this->rev = rev;
+
+    const auto remoteUrl = lfs::getLfsEndpointUrl(repo);
+
+    this->url = nix::parseURL(nix::fixGitURL(remoteUrl)).canonicalise();
+}
+
+bool Fetch::shouldFetch(const CanonPath & path) const
+{
+    const char * attr = nullptr;
+    git_attr_options opts = GIT_ATTR_OPTIONS_INIT;
+    opts.attr_commit_id = this->rev;
+    opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM;
+    if (git_attr_get_ext(&attr, (git_repository *) (this->repo), &opts, path.rel_c_str(), "filter"))
+        throw Error("cannot get git-lfs attribute: %s", git_error_last()->message);
+    debug("Git filter for '%s' is '%s'", path, attr ? attr : "null");
+    return attr != nullptr && !std::string(attr).compare("lfs");
+}
+
+static nlohmann::json pointerToPayload(const std::vector<Pointer> & items)
+{
+    nlohmann::json jArray = nlohmann::json::array();
+    for (const auto & pointer : items)
+        jArray.push_back({{"oid", pointer.oid}, {"size", pointer.size}});
+    return jArray;
+}
+
+std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointers) const
+{
+    ParsedURL httpUrl(url);
+    httpUrl.scheme = url.scheme == "ssh" ? "https" : url.scheme;
+    FileTransferRequest request(httpUrl.to_string() + "/info/lfs/objects/batch");
+    request.post = true;
+    Headers headers;
+    if (this->url.scheme == "ssh")
+        headers.push_back({"Authorization", lfs::getLfsApiToken(this->url)});
+    headers.push_back({"Content-Type", "application/vnd.git-lfs+json"});
+    headers.push_back({"Accept", "application/vnd.git-lfs+json"});
+    request.headers = headers;
+    nlohmann::json oidList = pointerToPayload(pointers);
+    nlohmann::json data = {{"operation", "download"}};
+    data["objects"] = oidList;
+    request.data = data.dump();
+
+    FileTransferResult result = getFileTransfer()->upload(request);
+    auto responseString = result.data;
+
+    std::vector<nlohmann::json> objects;
+    // example resp here:
+    // {"objects":[{"oid":"f5e02aa71e67f41d79023a128ca35bad86cf7b6656967bfe0884b3a3c4325eaf","size":10000000,"actions":{"download":{"href":"https://gitlab.com/b-camacho/test-lfs.git/gitlab-lfs/objects/f5e02aa71e67f41d79023a128ca35bad86cf7b6656967bfe0884b3a3c4325eaf","header":{"Authorization":"Basic
+    // Yi1jYW1hY2hvOmV5SjBlWEFpT2lKS1YxUWlMQ0poYkdjaU9pSklVekkxTmlKOS5leUprWVhSaElqcDdJbUZqZEc5eUlqb2lZaTFqWVcxaFkyaHZJbjBzSW1wMGFTSTZJbUptTURZNFpXVTFMVEprWmpVdE5HWm1ZUzFpWWpRMExUSXpNVEV3WVRReU1qWmtaaUlzSW1saGRDSTZNVGN4TkRZeE16ZzBOU3dpYm1KbUlqb3hOekUwTmpFek9EUXdMQ0psZUhBaU9qRTNNVFEyTWpFd05EVjkuZk9yMDNkYjBWSTFXQzFZaTBKRmJUNnJTTHJPZlBwVW9lYllkT0NQZlJ4QQ=="}}},"authenticated":true}]}
+
+    try {
+        auto resp = nlohmann::json::parse(responseString);
+        if (resp.contains("objects"))
+            objects.insert(objects.end(), resp["objects"].begin(), resp["objects"].end());
+        else
+            throw Error("response does not contain 'objects'");
+
+        return objects;
+    } catch (const nlohmann::json::parse_error & e) {
+        printMsg(lvlTalkative, "Full response: '%1%'", responseString);
+        throw Error("response did not parse as json: %s", e.what());
+    }
+}
+
+void Fetch::fetch(
+    const std::string & content,
+    const CanonPath & pointerFilePath,
+    StringSink & sink,
+    std::function<void(uint64_t)> sizeCallback) const
+{
+    debug("trying to fetch '%s' using git-lfs", pointerFilePath);
+
+    if (content.length() >= 1024) {
+        warn("encountered file '%s' that should have been a git-lfs pointer, but is too large", pointerFilePath);
+        sizeCallback(content.length());
+        sink(content);
+        return;
+    }
+
+    const auto pointer = parseLfsPointer(content, pointerFilePath.rel());
+    if (pointer == std::nullopt) {
+        warn("encountered file '%s' that should have been a git-lfs pointer, but is invalid", pointerFilePath);
+        sizeCallback(content.length());
+        sink(content);
+        return;
+    }
+
+    Path cacheDir = getCacheDir() + "/git-lfs";
+    std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false)
+                      + "/" + pointer->oid;
+    Path cachePath = cacheDir + "/" + key;
+    if (pathExists(cachePath)) {
+        debug("using cache entry %s -> %s", key, cachePath);
+        sink(readFile(cachePath));
+        return;
+    }
+    debug("did not find cache entry for %s", key);
+
+    std::vector<Pointer> pointers;
+    pointers.push_back(pointer.value());
+    const auto objUrls = fetchUrls(pointers);
+
+    const auto obj = objUrls[0];
+    try {
+        std::string sha256 = obj.at("oid"); // oid is also the sha256
+        std::string ourl = obj.at("actions").at("download").at("href");
+        std::string authHeader = "";
+        if (obj.at("actions").at("download").contains("header")
+            && obj.at("actions").at("download").at("header").contains("Authorization")) {
+            authHeader = obj["actions"]["download"]["header"]["Authorization"];
+        }
+        const uint64_t size = obj.at("size");
+        sizeCallback(size);
+        downloadToSink(ourl, authHeader, sink, sha256, size);
+
+        debug("creating cache entry %s -> %s", key, cachePath);
+        if (!pathExists(dirOf(cachePath)))
+            createDirs(dirOf(cachePath));
+        writeFile(cachePath, sink.s);
+
+        debug("%s fetched with git-lfs", pointerFilePath);
+    } catch (const nlohmann::json::out_of_range & e) {
+        throw Error("bad json from /info/lfs/objects/batch: %s %s", obj, e.what());
+    }
+}
+
+} // namespace nix::lfs

src/libfetchers/git-lfs-fetch.hh

@@ -0,0 +1,43 @@
+#include "canon-path.hh"
+#include "serialise.hh"
+#include "url.hh"
+
+#include <git2/repository.h>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix::lfs {
+
+/**
+ * git-lfs pointer
+ * @see https://github.com/git-lfs/git-lfs/blob/2ef4108/docs/spec.md
+ */
+struct Pointer
+{
+    std::string oid; // git-lfs managed object id. you give this to the lfs server
+                     // for downloads
+    size_t size;     // in bytes
+};
+
+struct Fetch
+{
+    // Reference to the repository
+    const git_repository * repo;
+
+    // Git commit being fetched
+    git_oid rev;
+
+    // derived from git remote url
+    nix::ParsedURL url;
+
+    Fetch(git_repository * repo, git_oid rev);
+    bool shouldFetch(const CanonPath & path) const;
+    void fetch(
+        const std::string & content,
+        const CanonPath & pointerFilePath,
+        StringSink & sink,
+        std::function<void(uint64_t)> sizeCallback) const;
+    std::vector<nlohmann::json> fetchUrls(const std::vector<Pointer> & pointers) const;
+};
+
+} // namespace nix::lfs

src/libfetchers/git-utils.cc

@@ -1,4 +1,5 @@
 #include "git-utils.hh"
+#include "git-lfs-fetch.hh"
 #include "cache.hh"
 #include "finally.hh"
 #include "processes.hh"
@@ -60,14 +61,6 @@ namespace nix {
 
 struct GitSourceAccessor;
 
-// Some wrapper types that ensure that the git_*_free functions get called.
-template<auto del>
-struct Deleter
-{
-    template <typename T>
-    void operator()(T * p) const { del(p); };
-};
-
 typedef std::unique_ptr<git_repository, Deleter<git_repository_free>> Repository;
 typedef std::unique_ptr<git_tree_entry, Deleter<git_tree_entry_free>> TreeEntry;
 typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
@@ -85,20 +78,6 @@ typedef std::unique_ptr<git_odb, Deleter<git_odb_free>> ObjectDb;
 typedef std::unique_ptr<git_packbuilder, Deleter<git_packbuilder_free>> PackBuilder;
 typedef std::unique_ptr<git_indexer, Deleter<git_indexer_free>> Indexer;
 
-// A helper to ensure that we don't leak objects returned by libgit2.
-template<typename T>
-struct Setter
-{
-    T & t;
-    typename T::pointer p = nullptr;
-
-    Setter(T & t) : t(t) { }
-
-    ~Setter() { if (p) t = T(p); }
-
-    operator typename T::pointer * () { return &p; }
-};
-
 Hash toHash(const git_oid & oid)
 {
     #ifdef GIT_EXPERIMENTAL_SHA256
@@ -506,12 +485,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     /**
      * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
      */
-    ref<GitSourceAccessor> getRawAccessor(const Hash & rev);
+    ref<GitSourceAccessor> getRawAccessor(
+        const Hash & rev,
+        bool smudgeLfs = false);
 
     ref<SourceAccessor> getAccessor(
         const Hash & rev,
         bool exportIgnore,
-        std::string displayPrefix) override;
+        std::string displayPrefix,
+        bool smudgeLfs = false) override;
 
     ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
 
@@ -670,24 +652,40 @@ ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create,
 /**
  * Raw git tree input accessor.
  */
+
 struct GitSourceAccessor : SourceAccessor
 {
     ref<GitRepoImpl> repo;
     Object root;
+    std::optional<lfs::Fetch> lfsFetch = std::nullopt;
 
-    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
         : repo(repo_)
         , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get()))
     {
+        if (smudgeLfs)
+            lfsFetch = std::make_optional(lfs::Fetch(*repo, hashToOID(rev)));
     }
 
     std::string readBlob(const CanonPath & path, bool symlink)
     {
-        auto blob = getBlob(path, symlink);
+        const auto blob = getBlob(path, symlink);
 
-        auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+        if (lfsFetch) {
+            if (lfsFetch->shouldFetch(path)) {
+                StringSink s;
+                try {
+                    auto contents = std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+                    lfsFetch->fetch(contents, path, s, [&s](uint64_t size){ s.s.reserve(size); });
+                } catch (Error & e) {
+                    e.addTrace({}, "while smudging git-lfs file '%s'", path);
+                    throw;
+                }
+                return s.s;
+            }
+        }
 
-        return std::string(data);
+        return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
     }
 
     std::string readFile(const CanonPath & path) override
@@ -1191,19 +1189,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
+ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(
+    const Hash & rev,
+    bool smudgeLfs)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitSourceAccessor>(self, rev);
+    return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
 }
 
 ref<SourceAccessor> GitRepoImpl::getAccessor(
     const Hash & rev,
     bool exportIgnore,
-    std::string displayPrefix)
+    std::string displayPrefix,
+    bool smudgeLfs)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev);
+    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
     rawGitAccessor->setPathDisplay(std::move(displayPrefix));
     if (exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);

src/libfetchers/git-utils.hh

@@ -89,7 +89,8 @@ struct GitRepo
     virtual ref<SourceAccessor> getAccessor(
         const Hash & rev,
         bool exportIgnore,
-        std::string displayPrefix) = 0;
+        std::string displayPrefix,
+        bool smudgeLfs = false) = 0;
 
     virtual ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
 
@@ -126,4 +127,26 @@ struct GitRepo
 
 ref<GitRepo> getTarballCache();
 
+// A helper to ensure that the `git_*_free` functions get called.
+template<auto del>
+struct Deleter
+{
+    template <typename T>
+    void operator()(T * p) const { del(p); };
+};
+
+// A helper to ensure that we don't leak objects returned by libgit2.
+template<typename T>
+struct Setter
+{
+    T & t;
+    typename T::pointer p = nullptr;
+
+    Setter(T & t) : t(t) { }
+
+    ~Setter() { if (p) t = T(p); }
+
+    operator typename T::pointer * () { return &p; }
+};
+
 }

src/libfetchers/git.cc

@@ -185,7 +185,7 @@ struct GitInputScheme : InputScheme
         for (auto & [name, value] : url.query) {
             if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys")
                 attrs.emplace(name, value);
-            else if (name == "shallow" || name == "submodules" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit")
+            else if (name == "shallow" || name == "submodules" || name == "lfs" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit")
                 attrs.emplace(name, Explicit<bool> { value == "1" });
             else
                 url2.query.emplace(name, value);
@@ -210,6 +210,7 @@ struct GitInputScheme : InputScheme
             "rev",
             "shallow",
             "submodules",
+            "lfs",
             "exportIgnore",
             "lastModified",
             "revCount",
@@ -262,6 +263,8 @@ struct GitInputScheme : InputScheme
         if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
         if (getShallowAttr(input))
             url.query.insert_or_assign("shallow", "1");
+        if (getLfsAttr(input))
+            url.query.insert_or_assign("lfs", "1");
         if (getSubmodulesAttr(input))
             url.query.insert_or_assign("submodules", "1");
         if (maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false))
@@ -411,6 +414,11 @@ struct GitInputScheme : InputScheme
         return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
     }
 
+    bool getLfsAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "lfs").value_or(false);
+    }
+
     bool getExportIgnoreAttr(const Input & input) const
     {
         return maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false);
@@ -678,7 +686,8 @@ struct GitInputScheme : InputScheme
         verifyCommit(input, repo);
 
         bool exportIgnore = getExportIgnoreAttr(input);
-        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»");
+        bool smudgeLfs = getLfsAttr(input);
+        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
 
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
@@ -698,6 +707,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
                 attrs.insert_or_assign("submodules", Explicit<bool>{ true });
+                attrs.insert_or_assign("lfs", Explicit<bool>{ smudgeLfs });
                 attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
                 auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
@@ -838,7 +848,7 @@ struct GitInputScheme : InputScheme
     {
         auto makeFingerprint = [&](const Hash & rev)
         {
-            return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "");
+            return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : "");
         };
 
         if (auto rev = input.getRev())

src/libfetchers/meson.build

@@ -48,6 +48,7 @@ sources = files(
   'fetch-to-store.cc',
   'fetchers.cc',
   'filtering-source-accessor.cc',
+  'git-lfs-fetch.cc',
   'git-utils.cc',
   'git.cc',
   'github.cc',
@@ -69,6 +70,7 @@ headers = files(
   'fetch-to-store.hh',
   'fetchers.hh',
   'filtering-source-accessor.hh',
+  'git-lfs-fetch.hh',
   'git-utils.hh',
   'mounted-source-accessor.hh',
   'registry.hh',

src/libstore/filetransfer.cc

@@ -94,7 +94,7 @@ struct curlFileTransfer : public FileTransfer
             : fileTransfer(fileTransfer)
             , request(request)
             , act(*logger, lvlTalkative, actFileTransfer,
-                fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
+                request.post ? "" : fmt(request.data ?  "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
             , callback(std::move(callback))
             , finalSink([this](std::string_view data) {
@@ -271,11 +271,21 @@ struct curlFileTransfer : public FileTransfer
             return getInterrupted();
         }
 
+        int silentProgressCallback(double dltotal, double dlnow)
+        {
+            return getInterrupted();
+        }
+
         static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
         {
             return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
         }
 
+        static int silentProgressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
+        {
+            return ((TransferItem *) userp)->silentProgressCallback(dltotal, dlnow);
+        }
+
         static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
         {
             if (type == CURLINFO_TEXT)
@@ -340,8 +350,11 @@ struct curlFileTransfer : public FileTransfer
             curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
 
-            curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
-            curl_easy_setopt(req, CURLOPT_PROGRESSDATA, this);
+            if (request.post)
+                curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, silentProgressCallbackWrapper);
+            else
+                curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, progressCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_XFERINFODATA, this);
             curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0);
 
             curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
@@ -353,7 +366,10 @@ struct curlFileTransfer : public FileTransfer
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
             if (request.data) {
-                curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
+                if (request.post)
+                    curl_easy_setopt(req, CURLOPT_POST, 1L);
+                else
+                    curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
                 curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
                 curl_easy_setopt(req, CURLOPT_READDATA, this);
                 curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
@@ -430,7 +446,8 @@ struct curlFileTransfer : public FileTransfer
                 if (httpStatus == 304 && result.etag == "")
                     result.etag = request.expectedETag;
 
-                act.progress(result.bodySize, result.bodySize);
+                if (!request.post)
+                    act.progress(result.bodySize, result.bodySize);
                 done = true;
                 callback(std::move(result));
             }

src/libstore/filetransfer.hh

@@ -65,6 +65,7 @@ struct FileTransferRequest
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
+    bool post = false;
     size_t tries = fileTransferSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;