Pluggable fetchers

Flakes are now fetched using an extensible mechanism. Also lots of
other flake cleanups.

src/libstore/fetchers/fetchers.cc

@@ -0,0 +1,56 @@
+#include "fetchers.hh"
+#include "parse.hh"
+#include "store-api.hh"
+
+namespace nix::fetchers {
+
+std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
+
+void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
+{
+    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
+    inputSchemes->push_back(std::move(inputScheme));
+}
+
+std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
+{
+    for (auto & inputScheme : *inputSchemes) {
+        auto res = inputScheme->inputFromURL(url);
+        if (res) return res;
+    }
+    throw Error("input '%s' is unsupported", url.url);
+}
+
+std::unique_ptr<Input> inputFromURL(const std::string & url)
+{
+    return inputFromURL(parseURL(url));
+}
+
+std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
+{
+    auto [tree, input] = fetchTreeInternal(store);
+
+    if (tree.actualPath == "")
+        tree.actualPath = store->toRealPath(store->printStorePath(tree.storePath));
+
+    if (!tree.narHash)
+        tree.narHash = store->queryPathInfo(tree.storePath)->narHash;
+
+    if (input->narHash)
+        assert(input->narHash == tree.narHash);
+
+    return {std::move(tree), input};
+}
+
+std::shared_ptr<const Input> Input::applyOverrides(
+    std::optional<std::string> ref,
+    std::optional<Hash> rev) const
+{
+    if (ref)
+        throw Error("don't know how to apply '%s' to '%s'", *ref, to_string());
+    if (rev)
+        throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string());
+    return shared_from_this();
+}
+
+}

src/libstore/fetchers/fetchers.hh

@@ -0,0 +1,75 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+#include "path.hh"
+
+#include <memory>
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct Input;
+
+struct Tree
+{
+    Path actualPath;
+    StorePath storePath;
+    Hash narHash;
+    std::optional<Hash> rev;
+    std::optional<uint64_t> revCount;
+    std::optional<time_t> lastModified;
+};
+
+struct Input : std::enable_shared_from_this<Input>
+{
+    std::string type;
+    std::optional<Hash> narHash;
+
+    virtual bool operator ==(const Input & other) const { return false; }
+
+    virtual bool isDirect() const { return true; }
+
+    virtual bool isImmutable() const { return (bool) narHash; }
+
+    virtual bool contains(const Input & other) const { return false; }
+
+    virtual std::optional<std::string> getRef() const { return {}; }
+
+    virtual std::optional<Hash> getRev() const { return {}; }
+
+    virtual std::string to_string() const = 0;
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
+
+    virtual std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const;
+
+    virtual std::optional<Path> getSourcePath() const { return {}; }
+
+    virtual void clone(const Path & destDir) const
+    {
+        throw Error("do not know how to clone input '%s'", to_string());
+    }
+
+private:
+
+    virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
+};
+
+struct ParsedURL;
+
+struct InputScheme
+{
+    virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
+};
+
+std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
+
+std::unique_ptr<Input> inputFromURL(const std::string & url);
+
+void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
+
+}

src/libstore/fetchers/git.cc

@@ -0,0 +1,382 @@
+#include "fetchers.hh"
+#include "parse.hh"
+#include "globals.hh"
+#include "tarfile.hh"
+#include "store-api.hh"
+#include "regex.hh"
+
+#include <sys/time.h>
+
+#include <nlohmann/json.hpp>
+
+using namespace std::string_literals;
+
+namespace nix::fetchers {
+
+static Path getCacheInfoPathFor(const std::string & name, const Hash & rev)
+{
+    Path cacheDir = getCacheDir() + "/nix/git-revs-v2";
+    std::string linkName =
+        name == "source"
+        ? rev.gitRev()
+        : hashString(htSHA512, name + std::string("\0"s) + rev.gitRev()).to_string(Base32, false);
+    return cacheDir + "/" + linkName + ".link";
+}
+
+static void cacheGitInfo(Store & store, const std::string & name, const Tree & tree)
+{
+    nlohmann::json json;
+    json["storePath"] = store.printStorePath(tree.storePath);
+    json["name"] = name;
+    json["rev"] = tree.rev->gitRev();
+    json["revCount"] = *tree.revCount;
+    json["lastModified"] = *tree.lastModified;
+
+    auto cacheInfoPath = getCacheInfoPathFor(name, *tree.rev);
+    createDirs(dirOf(cacheInfoPath));
+    writeFile(cacheInfoPath, json.dump());
+}
+
+static std::optional<Tree> lookupGitInfo(
+    ref<Store> store,
+    const std::string & name,
+    const Hash & rev)
+{
+    try {
+        auto json = nlohmann::json::parse(readFile(getCacheInfoPathFor(name, rev)));
+
+        assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == rev);
+
+        auto storePath = store->parseStorePath((std::string) json["storePath"]);
+
+        if (store->isValidPath(storePath)) {
+            Tree tree{
+                .actualPath = store->toRealPath(store->printStorePath(storePath)),
+                .storePath = std::move(storePath),
+                .rev = rev,
+                .revCount = json["revCount"],
+                .lastModified = json["lastModified"],
+            };
+            return tree;
+        }
+
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT) throw;
+    }
+
+    return {};
+}
+
+struct GitInput : Input
+{
+    ParsedURL url;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    GitInput(const ParsedURL & url) : url(url)
+    {
+        type = "git";
+    }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const GitInput *>(&other);
+        return
+            other2
+            && url.url == other2->url.url
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    std::string to_string() const override
+    {
+        ParsedURL url2(url);
+        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
+        if (ref) url2.query.insert_or_assign("ref", *ref);
+        return url2.to_string();
+    }
+
+    void clone(const Path & destDir) const override
+    {
+        auto [isLocal, actualUrl] = getActualUrl();
+
+        Strings args = {"clone"};
+
+        args.push_back(actualUrl);
+
+        if (ref) {
+            args.push_back("--branch");
+            args.push_back(*ref);
+        }
+
+        if (rev) throw Error("cloning a specific revision is not implemented");
+
+        args.push_back(destDir);
+
+        runProgram("git", true, args);
+    }
+
+    std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const override
+    {
+        if (!ref && !rev) return shared_from_this();
+
+        auto res = std::make_shared<GitInput>(*this);
+
+        if (ref) res->ref = ref;
+        if (rev) res->rev = rev;
+
+        if (!res->ref && res->rev)
+            throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string());
+
+        return res;
+    }
+
+    std::optional<Path> getSourcePath() const
+    {
+        if (url.scheme == "git+file" && !ref && !rev)
+            return url.path;
+        return {};
+    }
+
+    std::pair<bool, std::string> getActualUrl() const
+    {
+        // Don't clone git+file:// URIs (but otherwise treat them the
+        // same as remote URIs, i.e. don't use the working tree or
+        // HEAD).
+        static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
+        bool isLocal = url.scheme == "git+file" && !forceHttp;
+        return {isLocal, isLocal ? url.path : std::string(url.base, 4)};
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto name = "source";
+
+        auto input = std::make_shared<GitInput>(*this);
+
+        assert(!rev || rev->type == htSHA1);
+
+        if (rev) {
+            if (auto tree = lookupGitInfo(store, name, *rev))
+                return {std::move(*tree), input};
+        }
+
+        auto [isLocal, actualUrl] = getActualUrl();
+
+        // If this is a local directory and no ref or revision is
+        // given, then allow the use of an unclean working tree.
+        if (!input->ref && !input->rev && isLocal) {
+            bool clean = false;
+
+            /* Check whether this repo has any commits. There are
+               probably better ways to do this. */
+            bool haveCommits = !readDirectory(actualUrl + "/.git/refs/heads").empty();
+
+            try {
+                if (haveCommits) {
+                    runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
+                    clean = true;
+                }
+            } catch (ExecError & e) {
+                if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
+            }
+
+            if (!clean) {
+
+                /* This is an unclean working tree. So copy all tracked files. */
+
+                if (!settings.allowDirty)
+                    throw Error("Git tree '%s' is dirty", actualUrl);
+
+                if (settings.warnDirty)
+                    warn("Git tree '%s' is dirty", actualUrl);
+
+                auto files = tokenizeString<std::set<std::string>>(
+                    runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s);
+
+                PathFilter filter = [&](const Path & p) -> bool {
+                    assert(hasPrefix(p, actualUrl));
+                    std::string file(p, actualUrl.size() + 1);
+
+                    auto st = lstat(p);
+
+                    if (S_ISDIR(st.st_mode)) {
+                        auto prefix = file + "/";
+                        auto i = files.lower_bound(prefix);
+                        return i != files.end() && hasPrefix(*i, prefix);
+                    }
+
+                    return files.count(file);
+                };
+
+                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+
+                auto tree = Tree {
+                    .actualPath = store->printStorePath(storePath),
+                    .storePath = std::move(storePath),
+                    .revCount = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "rev-list", "--count", "HEAD" })) : 0,
+                    // FIXME: maybe we should use the timestamp of the last
+                    // modified dirty file?
+                    .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
+                };
+
+                return {std::move(tree), input};
+            }
+        }
+
+        if (!input->ref) input->ref = isLocal ? "HEAD" : "master";
+
+        Path repoDir;
+
+        if (isLocal) {
+
+            if (!input->rev)
+                input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
+
+            repoDir = actualUrl;
+
+        } else {
+
+            Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
+            repoDir = cacheDir;
+
+            if (!pathExists(cacheDir)) {
+                createDirs(dirOf(cacheDir));
+                runProgram("git", true, { "init", "--bare", repoDir });
+            }
+
+            Path localRefFile =
+                input->ref->compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + *input->ref
+                : cacheDir + "/refs/heads/" + *input->ref;
+
+            bool doFetch;
+            time_t now = time(0);
+
+            /* If a rev was specified, we need to fetch if it's not in the
+               repo. */
+            if (input->rev) {
+                try {
+                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
+                    doFetch = false;
+                } catch (ExecError & e) {
+                    if (WIFEXITED(e.status)) {
+                        doFetch = true;
+                    } else {
+                        throw;
+                    }
+                }
+            } else {
+                /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+                   git fetch to update the local ref to the remote ref. */
+                struct stat st;
+                doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+                    (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+            }
+
+            if (doFetch) {
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
+
+                // FIXME: git stderr messes up our progress indicator, so
+                // we're using --quiet for now. Should process its stderr.
+                try {
+                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
+                } catch (Error & e) {
+                    if (!pathExists(localRefFile)) throw;
+                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
+                }
+
+                struct timeval times[2];
+                times[0].tv_sec = now;
+                times[0].tv_usec = 0;
+                times[1].tv_sec = now;
+                times[1].tv_usec = 0;
+
+                utimes(localRefFile.c_str(), times);
+            }
+
+            if (!input->rev)
+                input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
+        }
+
+        if (auto tree = lookupGitInfo(store, name, *input->rev))
+            return {std::move(*tree), input};
+
+        // FIXME: check whether rev is an ancestor of ref.
+
+        printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
+
+        // FIXME: should pipe this, or find some better way to extract a
+        // revision.
+        auto source = sinkToSource([&](Sink & sink) {
+            RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
+            gitOptions.standardOut = &sink;
+            runProgram2(gitOptions);
+        });
+
+        Path tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+
+        unpackTarfile(*source, tmpDir);
+
+        auto storePath = store->addToStore(name, tmpDir);
+        auto revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }));
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
+
+        auto tree = Tree {
+            .actualPath = store->toRealPath(store->printStorePath(storePath)),
+            .storePath = std::move(storePath),
+            .rev = input->rev,
+            .revCount = revCount,
+            .lastModified = lastModified,
+        };
+
+        cacheGitInfo(*store, name, tree);
+
+        return {std::move(tree), input};
+    }
+};
+
+struct GitInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "git" &&
+            url.scheme != "git+http" &&
+            url.scheme != "git+https" &&
+            url.scheme != "git+ssh" &&
+            url.scheme != "git+file") return nullptr;
+
+        auto input = std::make_unique<GitInput>(url);
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev") {
+                if (!std::regex_match(value, revRegex))
+                    throw BadURL("Git URL '%s' contains an invalid commit hash", url.url);
+                input->rev = Hash(value, htSHA1);
+            }
+            else if (name == "ref") {
+                if (!std::regex_match(value, refRegex))
+                    throw BadURL("Git URL '%s' contains an invalid branch/tag name", url.url);
+                input->ref = value;
+            }
+        }
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
+
+}

src/libstore/fetchers/github.cc

@@ -0,0 +1,183 @@
+#include "fetchers.hh"
+#include "download.hh"
+#include "globals.hh"
+#include "parse.hh"
+#include "regex.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct GitHubInput : Input
+{
+    std::string owner;
+    std::string repo;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const GitHubInput *>(&other);
+        return
+            other2
+            && owner == other2->owner
+            && repo == other2->repo
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    std::string to_string() const override
+    {
+        auto s = fmt("github:%s/%s", owner, repo);
+        assert(!(ref && rev));
+        if (ref) s += "/" + *ref;
+        if (rev) s += "/" + rev->to_string(Base16, false);
+        return s;
+    }
+
+    void clone(const Path & destDir) const override
+    {
+        std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo));
+        input = input->applyOverrides(ref.value_or("master"), rev);
+        input->clone(destDir);
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto rev = this->rev;
+
+    #if 0
+        if (rev) {
+            if (auto gitInfo = lookupGitInfo(store, "source", *rev))
+                return *gitInfo;
+        }
+    #endif
+
+        if (!rev) {
+            auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
+                owner, repo, ref ? *ref : "master");
+            CachedDownloadRequest request(url);
+            request.ttl = rev ? 1000000000 : settings.tarballTtl;
+            auto result = getDownloader()->downloadCached(store, request);
+            auto json = nlohmann::json::parse(readFile(result.path));
+            rev = Hash(json["sha"], htSHA1);
+            debug("HEAD revision for '%s' is %s", url, rev->gitRev());
+        }
+
+        // FIXME: use regular /archive URLs instead? api.github.com
+        // might have stricter rate limits.
+
+        auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
+            owner, repo, rev->to_string(Base16, false));
+
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            url += "?access_token=" + accessToken;
+
+        CachedDownloadRequest request(url);
+        request.unpack = true;
+        request.name = "source";
+        request.ttl = 1000000000;
+        request.getLastModified = true;
+        auto dresult = getDownloader()->downloadCached(store, request);
+
+        assert(dresult.lastModified);
+
+        Tree result{
+            .actualPath = dresult.path,
+            .storePath = store->parseStorePath(dresult.storePath),
+            .rev = *rev,
+            .lastModified = *dresult.lastModified
+        };
+
+    #if 0
+        // FIXME: this can overwrite a cache file that contains a revCount.
+        cacheGitInfo("source", gitInfo);
+    #endif
+
+        auto input = std::make_shared<GitHubInput>(*this);
+        input->ref = {};
+        input->rev = *rev;
+
+        return {std::move(result), input};
+    }
+
+    std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const override
+    {
+        if (!ref && !rev) return shared_from_this();
+
+        auto res = std::make_shared<GitHubInput>(*this);
+
+        if (ref) res->ref = ref;
+        if (rev) res->rev = rev;
+
+        return res;
+    }
+};
+
+struct GitHubInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "github") return nullptr;
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+        auto input = std::make_unique<GitHubInput>();
+        input->type = "github";
+
+        if (path.size() == 2) {
+        } else if (path.size() == 3) {
+            if (std::regex_match(path[2], revRegex))
+                input->rev = Hash(path[2], htSHA1);
+            else if (std::regex_match(path[2], refRegex))
+                input->ref = path[2];
+            else
+                throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+        } else
+            throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev") {
+                if (!std::regex_match(value, revRegex))
+                    throw BadURL("GitHub URL '%s' contains an invalid commit hash", url.url);
+                if (input->rev)
+                    throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
+                input->rev = Hash(value, htSHA1);
+            }
+            else if (name == "ref") {
+                if (!std::regex_match(value, refRegex))
+                    throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
+                if (input->ref)
+                    throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
+                input->ref = value;
+            }
+        }
+
+        if (input->ref && input->rev)
+            throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
+
+        input->owner = path[0];
+        input->repo = path[1];
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+
+}

src/libstore/fetchers/indirect.cc

@@ -0,0 +1,114 @@
+#include "fetchers.hh"
+#include "parse.hh"
+#include "regex.hh"
+
+namespace nix::fetchers {
+
+std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct IndirectInput : Input
+{
+    std::string id;
+    std::optional<Hash> rev;
+    std::optional<std::string> ref;
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const IndirectInput *>(&other);
+        return
+            other2
+            && id == other2->id
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isDirect() const override
+    {
+        return false;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    bool contains(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const IndirectInput *>(&other);
+        return
+            other2
+            && id == other2->id
+            && (!ref || ref == other2->ref)
+            && (!rev || rev == other2->rev);
+    }
+
+    std::string to_string() const override
+    {
+        ParsedURL url;
+        url.scheme = "flake";
+        url.path = id;
+        if (ref) { url.path += '/'; url.path += *ref; };
+        if (rev) { url.path += '/'; url.path += rev->gitRev(); };
+        return url.to_string();
+    }
+
+    std::shared_ptr<const Input> applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const override
+    {
+        if (!ref && !rev) return shared_from_this();
+
+        auto res = std::make_shared<IndirectInput>(*this);
+
+        if (ref) res->ref = ref;
+        if (rev) res->rev = rev;
+
+        return res;
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        throw Error("indirect input '%s' cannot be fetched directly", to_string());
+    }
+};
+
+struct IndirectInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "flake") return nullptr;
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+        auto input = std::make_unique<IndirectInput>();
+        input->type = "indirect";
+
+        if (path.size() == 1) {
+        } else if (path.size() == 2) {
+            if (std::regex_match(path[1], revRegex))
+                input->rev = Hash(path[1], htSHA1);
+            else if (std::regex_match(path[1], refRegex))
+                input->ref = path[1];
+            else
+                throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
+        } else if (path.size() == 3) {
+            if (!std::regex_match(path[1], refRegex))
+                throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
+            input->ref = path[1];
+            if (!std::regex_match(path[2], revRegex))
+                throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
+            input->rev = Hash(path[2], htSHA1);
+        } else
+            throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+        // FIXME: forbid query params?
+
+        input->id = path[0];
+        if (!std::regex_match(input->id, flakeRegex))
+            throw BadURL("'%s' is not a valid flake ID", input->id);
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
+
+}

src/libstore/fetchers/parse.cc

@@ -0,0 +1,129 @@
+#include "parse.hh"
+#include "util.hh"
+#include "regex.hh"
+
+namespace nix::fetchers {
+
+std::regex refRegex(refRegexS, std::regex::ECMAScript);
+std::regex revRegex(revRegexS, std::regex::ECMAScript);
+
+ParsedURL parseURL(const std::string & url)
+{
+    static std::regex uriRegex(
+        "(((" + schemeRegex + "):"
+        + "(//(" + authorityRegex + "))?"
+        + "(" + pathRegex + "))"
+        + "(?:\\?(" + queryRegex + "))?"
+        + "(?:#(" + queryRegex + "))?"
+        + ")",
+        std::regex::ECMAScript);
+
+    std::smatch match;
+
+    if (std::regex_match(url, match, uriRegex)) {
+        auto & base = match[2];
+        std::string scheme = match[3];
+        auto authority = match[4].matched
+            ? std::optional<std::string>(match[5]) : std::nullopt;
+        std::string path = match[6];
+        auto & query = match[7];
+        auto & fragment = match[8];
+
+        auto isFile = scheme.find("file") != std::string::npos;
+
+        if (authority && *authority != "" && isFile)
+            throw Error("file:// URL '%s' has unexpected authority '%s'",
+                url, *authority);
+
+        if (isFile && path.empty())
+            path = "/";
+
+        return ParsedURL{
+            .url = url,
+            .base = base,
+            .scheme = scheme,
+            .authority = authority,
+            .path = path,
+            .query = decodeQuery(query),
+            .fragment = percentDecode(std::string(fragment))
+        };
+    }
+
+    else
+        throw BadURL("'%s' is not a valid URL", url);
+}
+
+std::string percentDecode(std::string_view in)
+{
+    std::string decoded;
+    for (size_t i = 0; i < in.size(); ) {
+        if (in[i] == '%') {
+            if (i + 2 >= in.size())
+                throw BadURL("invalid URI parameter '%s'", in);
+            try {
+                decoded += std::stoul(std::string(in, i + 1, 2), 0, 16);
+                i += 3;
+            } catch (...) {
+                throw BadURL("invalid URI parameter '%s'", in);
+            }
+        } else
+            decoded += in[i++];
+    }
+    return decoded;
+}
+
+std::map<std::string, std::string> decodeQuery(const std::string & query)
+{
+    std::map<std::string, std::string> result;
+
+    for (auto s : tokenizeString<Strings>(query, "&")) {
+        auto e = s.find('=');
+        if (e != std::string::npos)
+            result.emplace(
+                s.substr(0, e),
+                percentDecode(std::string_view(s).substr(e + 1)));
+    }
+
+    return result;
+}
+
+std::string percentEncode(std::string_view s)
+{
+    std::string res;
+    for (auto & c : s)
+        if ((c >= 'a' && c <= 'z')
+            || (c >= 'A' && c <= 'Z')
+            || (c >= '0' && c <= '9')
+            || strchr("-._~!$&'()*+,;=:@", c))
+            res += c;
+        else
+            res += fmt("%%%02x", (unsigned int) c);
+    return res;
+}
+
+std::string encodeQuery(const std::map<std::string, std::string> & ss)
+{
+    std::string res;
+    bool first = true;
+    for (auto & [name, value] : ss) {
+        if (!first) res += '&';
+        first = false;
+        res += percentEncode(name);
+        res += '=';
+        res += percentEncode(value);
+    }
+    return res;
+}
+
+std::string ParsedURL::to_string() const
+{
+    return
+        scheme
+        + ":"
+        + (authority ? "//" + *authority : "")
+        + path
+        + (query.empty() ? "" : "?" + encodeQuery(query))
+        + (fragment.empty() ? "" : "#" + percentEncode(fragment));
+}
+
+}

src/libstore/fetchers/parse.hh

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix::fetchers {
+
+struct ParsedURL
+{
+    std::string url;
+    std::string base; // URL without query/fragment
+    std::string scheme;
+    std::optional<std::string> authority;
+    std::string path;
+    std::map<std::string, std::string> query;
+    std::string fragment;
+
+    std::string to_string() const;
+};
+
+MakeError(BadURL, Error);
+
+std::string percentDecode(std::string_view in);
+
+std::map<std::string, std::string> decodeQuery(const std::string & query);
+
+ParsedURL parseURL(const std::string & url);
+
+}

src/libstore/fetchers/regex.hh

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <regex>
+
+namespace nix::fetchers {
+
+// URI stuff.
+const static std::string pctEncoded = "%[0-9a-fA-F][0-9a-fA-F]";
+const static std::string schemeRegex = "[a-z+]+";
+const static std::string authorityRegex =
+    "(?:(?:[a-z])*@)?"
+    "[a-zA-Z0-9._~-]*";
+const static std::string segmentRegex = "[a-zA-Z0-9._~-]+";
+const static std::string pathRegex = "(?:/?" + segmentRegex + "(?:/" + segmentRegex + ")*|/?)";
+const static std::string pcharRegex =
+    "(?:[a-zA-Z0-9-._~!$&'()*+,;=:@ ]|" + pctEncoded + ")";
+const static std::string queryRegex = "(?:" + pcharRegex + "|[/?])*";
+
+// A Git ref (i.e. branch or tag name).
+const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
+extern std::regex refRegex;
+
+// A Git revision (a SHA-1 commit hash).
+const static std::string revRegexS = "[0-9a-fA-F]{40}";
+extern std::regex revRegex;
+
+// A ref or revision, or a ref followed by a revision.
+const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
+
+const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*";
+
+}

src/libstore/fetchers/registry.cc

@@ -0,0 +1,145 @@
+#include "registry.hh"
+#include "util.hh"
+#include "fetchers.hh"
+#include "globals.hh"
+#include "download.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::shared_ptr<Registry> Registry::read(
+    const Path & path, RegistryType type)
+{
+    auto registry = std::make_shared<Registry>();
+    registry->type = type;
+
+    if (!pathExists(path))
+        return std::make_shared<Registry>();
+
+    auto json = nlohmann::json::parse(readFile(path));
+
+    auto version = json.value("version", 0);
+    if (version != 1)
+        throw Error("flake registry '%s' has unsupported version %d", path, version);
+
+    auto flakes = json["flakes"];
+    for (auto i = flakes.begin(); i != flakes.end(); ++i) {
+        // FIXME: remove 'uri' soon.
+        auto url = i->value("url", i->value("uri", ""));
+        if (url.empty())
+            throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'",
+                path, i.key());
+        registry->entries.push_back(
+            {inputFromURL(i.key()), inputFromURL(url)});
+    }
+
+    return registry;
+}
+
+void Registry::write(const Path & path)
+{
+    nlohmann::json json;
+    json["version"] = 1;
+    for (auto & elem : entries)
+        json["flakes"][elem.first->to_string()] = { {"url", elem.second->to_string()} };
+    createDirs(dirOf(path));
+    writeFile(path, json.dump(4));
+}
+
+void Registry::add(
+    const std::shared_ptr<const Input> & from,
+    const std::shared_ptr<const Input> & to)
+{
+    entries.emplace_back(from, to);
+}
+
+void Registry::remove(const std::shared_ptr<const Input> & input)
+{
+    // FIXME: use C++20 std::erase.
+    for (auto i = entries.begin(); i != entries.end(); )
+        if (*i->first == *input)
+            i = entries.erase(i);
+        else
+            ++i;
+}
+
+Path getUserRegistryPath()
+{
+    return getHome() + "/.config/nix/registry.json";
+}
+
+std::shared_ptr<Registry> getUserRegistry()
+{
+    return Registry::read(getUserRegistryPath(), Registry::User);
+}
+
+#if 0
+std::shared_ptr<Registry> getFlagRegistry(RegistryOverrides registryOverrides)
+{
+    auto flagRegistry = std::make_shared<Registry>();
+    for (auto const & x : registryOverrides)
+        flagRegistry->entries.insert_or_assign(
+            parseFlakeRef2(x.first),
+            parseFlakeRef2(x.second));
+    return flagRegistry;
+}
+#endif
+
+static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
+{
+    static auto reg = [&]() {
+        auto path = settings.flakeRegistry;
+
+        if (!hasPrefix(path, "/")) {
+            CachedDownloadRequest request(path);
+            request.name = "flake-registry.json";
+            request.gcRoot = true;
+            path = getDownloader()->downloadCached(store, request).path;
+        }
+
+        return Registry::read(path, Registry::Global);
+    }();
+
+    return reg;
+}
+
+Registries getRegistries(ref<Store> store)
+{
+    Registries registries;
+    //registries.push_back(getFlagRegistry(registryOverrides));
+    registries.push_back(getUserRegistry());
+    registries.push_back(getGlobalRegistry(store));
+    return registries;
+}
+
+std::shared_ptr<const Input> lookupInRegistries(
+    ref<Store> store,
+    std::shared_ptr<const Input> input)
+{
+    int n = 0;
+
+ restart:
+
+    n++;
+    if (n > 100) throw Error("cycle detected in flake registr for '%s'", input);
+
+    for (auto & registry : getRegistries(store)) {
+        // FIXME: O(n)
+        for (auto & entry : registry->entries) {
+            if (entry.first->contains(*input)) {
+                input = entry.second->applyOverrides(
+                    !entry.first->getRef() && input->getRef() ? input->getRef() : std::optional<std::string>(),
+                    !entry.first->getRev() && input->getRev() ? input->getRev() : std::optional<Hash>());
+                goto restart;
+            }
+        }
+    }
+
+    if (!input->isDirect())
+        throw Error("cannot find flake '%s' in the flake registries", input->to_string());
+
+    return input;
+}
+
+}

src/libstore/fetchers/registry.hh

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct Input;
+
+struct Registry
+{
+    enum RegistryType {
+        Flag = 0,
+        User = 1,
+        Global = 2,
+    };
+
+    RegistryType type;
+
+    std::vector<std::pair<std::shared_ptr<const Input>, std::shared_ptr<const Input>>> entries;
+
+    static std::shared_ptr<Registry> read(
+        const Path & path, RegistryType type);
+
+    void write(const Path & path);
+
+    void add(
+        const std::shared_ptr<const Input> & from,
+        const std::shared_ptr<const Input> & to);
+
+    void remove(const std::shared_ptr<const Input> & input);
+};
+
+typedef std::vector<std::shared_ptr<Registry>> Registries;
+
+std::shared_ptr<Registry> getUserRegistry();
+
+Path getUserRegistryPath();
+
+Registries getRegistries(ref<Store> store);
+
+std::shared_ptr<const Input> lookupInRegistries(
+    ref<Store> store,
+    std::shared_ptr<const Input> input);
+
+}

src/libstore/globals.hh

@@ -365,6 +365,15 @@ public:
     bool isExperimentalFeatureEnabled(const std::string & name);
 
     void requireExperimentalFeature(const std::string & name);
+
+    Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
+        "Path or URI of the global flake registry."};
+
+    Setting<bool> allowDirty{this, true, "allow-dirty",
+        "Whether to allow dirty Git/Mercurial trees."};
+
+    Setting<bool> warnDirty{this, true, "warn-dirty",
+        "Whether to warn about dirty Git/Mercurial trees."};
 };
 
 

src/libstore/local.mk

@@ -4,7 +4,7 @@ libstore_NAME = libnixstore
 
 libstore_DIR := $(d)
 
-libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
+libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/fetchers/*.cc)
 
 libstore_LIBS = libutil libnixrust
 

src/libstore/store-api.cc

@@ -6,6 +6,7 @@
 #include "thread-pool.hh"
 #include "json.hh"
 #include "derivations.hh"
+#include "fetchers/parse.hh"
 
 #include <future>
 
@@ -864,27 +865,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
     Store::Params params;
     auto q = uri.find('?');
     if (q != std::string::npos) {
-        for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
-            auto e = s.find('=');
-            if (e != std::string::npos) {
-                auto value = s.substr(e + 1);
-                std::string decoded;
-                for (size_t i = 0; i < value.size(); ) {
-                    if (value[i] == '%') {
-                        if (i + 2 >= value.size())
-                            throw Error("invalid URI parameter '%s'", value);
-                        try {
-                            decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
-                            i += 3;
-                        } catch (...) {
-                            throw Error("invalid URI parameter '%s'", value);
-                        }
-                    } else
-                        decoded += value[i++];
-                }
-                params[s.substr(0, e)] = decoded;
-            }
-        }
+        params = fetchers::decodeQuery(uri.substr(q + 1));
         uri = uri_.substr(0, q);
     }
     return {uri, params};