1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-07 10:11:47 +02:00

Remove TreeInfo

The attributes previously stored in TreeInfo (narHash, revCount,
lastModified) are now stored in Input. This makes it less arbitrary
what attributes are stored where.

As a result, the lock file format has changed. An entry like

    "info": {
      "lastModified": 1585405475,
      "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
    },
    "locked": {
      "owner": "NixOS",
      "repo": "nixpkgs",
      "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
      "type": "github"
    },

is now stored as

    "locked": {
      "owner": "NixOS",
      "repo": "nixpkgs",
      "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
      "type": "github",
      "lastModified": 1585405475,
      "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
    },

The 'Input' class is now a dumb set of attributes. All the fetcher
implementations subclass InputScheme, not Input. This simplifies the
API.

Also, fix substitution of flake inputs. This was broken since lazy
flake fetching started using fetchTree internally.
This commit is contained in:
Eelco Dolstra 2020-05-30 00:44:11 +02:00
parent 5633c0975b
commit 950b46821f
30 changed files with 963 additions and 1145 deletions

View file

@ -22,110 +22,121 @@ static bool isNotDotGitDirectory(const Path & path)
return not std::regex_match(path, gitDirRegex);
}
struct GitInput : Input
struct GitInputScheme : InputScheme
{
ParsedURL url;
std::optional<std::string> ref;
std::optional<Hash> rev;
bool shallow = false;
bool submodules = false;
GitInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "git"; }
bool operator ==(const Input & other) const override
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
auto other2 = dynamic_cast<const GitInput *>(&other);
return
other2
&& url == other2->url
&& rev == other2->rev
&& ref == other2->ref;
}
if (url.scheme != "git" &&
url.scheme != "git+http" &&
url.scheme != "git+https" &&
url.scheme != "git+ssh" &&
url.scheme != "git+file") return {};
bool isImmutable() const override
{
return (bool) rev || narHash;
}
auto url2(url);
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
url2.query.clear();
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
ParsedURL url2(url);
if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref);
if (shallow) url2.query.insert_or_assign("shallow", "1");
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
if (shallow)
attrs.emplace("shallow", true);
if (submodules)
attrs.emplace("submodules", true);
return attrs;
attrs.emplace("type", "git");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
void clone(const Path & destDir) const override
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
auto [isLocal, actualUrl] = getActualUrl();
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
throw Error("unsupported Git input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
maybeGetBoolAttr(attrs, "shallow");
maybeGetBoolAttr(attrs, "submodules");
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
}
Input input;
input.attrs = attrs;
return input;
}
ParsedURL toURL(const Input & input) override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme != "git") url.scheme = "git+" + url.scheme;
if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
url.query.insert_or_assign("shallow", "1");
return url;
}
bool hasAllInfo(const Input & input) override
{
bool maybeDirty = !input.getRef();
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
return
maybeGetIntAttr(input.attrs, "lastModified")
&& (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
}
Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev) override
{
auto res(input);
if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) res.attrs.insert_or_assign("ref", *ref);
if (!res.getRef() && res.getRev())
throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string());
return res;
}
void clone(const Input & input, const Path & destDir) override
{
auto [isLocal, actualUrl] = getActualUrl(input);
Strings args = {"clone"};
args.push_back(actualUrl);
if (ref) {
if (auto ref = input.getRef()) {
args.push_back("--branch");
args.push_back(*ref);
}
if (rev) throw Error("cloning a specific revision is not implemented");
if (input.getRev()) throw Error("cloning a specific revision is not implemented");
args.push_back(destDir);
runProgram("git", true, args);
}
std::shared_ptr<const Input> applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const override
std::optional<Path> getSourcePath(const Input & input) override
{
if (!ref && !rev) return shared_from_this();
auto res = std::make_shared<GitInput>(*this);
if (ref) res->ref = ref;
if (rev) res->rev = rev;
if (!res->ref && res->rev)
throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string());
return res;
}
std::optional<Path> getSourcePath() const override
{
if (url.scheme == "file" && !ref && !rev)
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
return url.path;
return {};
}
void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
auto sourcePath = getSourcePath();
auto sourcePath = getSourcePath(input);
assert(sourcePath);
runProgram("git", true,
@ -136,23 +147,25 @@ struct GitInput : Input
{ "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
}
std::pair<bool, std::string> getActualUrl() const
std::pair<bool, std::string> getActualUrl(const Input & input) const
{
// Don't clone file:// URIs (but otherwise treat them the
// same as remote URIs, i.e. don't use the working tree or
// HEAD).
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file" && !forceHttp;
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
auto input = std::make_shared<GitInput>(*this);
Input input(_input);
assert(!rev || rev->type == htSHA1);
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
@ -163,39 +176,38 @@ struct GitInput : Input
return Attrs({
{"type", cacheType},
{"name", name},
{"rev", input->rev->gitRev()},
{"rev", input.getRev()->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
-> std::pair<Tree, Input>
{
assert(input->rev);
assert(!rev || rev == input->rev);
assert(input.getRev());
assert(!_input.getRev() || _input.getRev() == input.getRev());
if (!shallow)
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
return {
Tree {
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
.lastModified = getIntAttr(infoAttrs, "lastModified"),
},
},
input
};
};
if (rev) {
if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
auto [isLocal, actualUrl_] = getActualUrl();
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
if (!input->ref && !input->rev && isLocal) {
if (!input.getRef() && !input.getRev() && isLocal) {
bool clean = false;
/* Check whether this repo has any commits. There are
@ -254,35 +266,37 @@ struct GitInput : Input
auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
auto tree = Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
.lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
}
};
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
return {std::move(tree), input};
return {
Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
}, input
};
}
}
if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
Attrs mutableAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
{"ref", *input.getRef()},
});
Path repoDir;
if (isLocal) {
if (!input->rev)
input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
if (!input.getRev())
input.attrs.insert_or_assign("rev",
Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
@ -290,8 +304,8 @@ struct GitInput : Input
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
@ -305,18 +319,18 @@ struct GitInput : Input
}
Path localRefFile =
input->ref->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input->ref
: cacheDir + "/refs/heads/" + *input->ref;
input.getRef()->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input.getRef()
: cacheDir + "/refs/heads/" + *input.getRef();
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (input->rev) {
if (input.getRev()) {
try {
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
@ -339,7 +353,7 @@ struct GitInput : Input
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
try {
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input.getRef(), *input.getRef()) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@ -354,8 +368,8 @@ struct GitInput : Input
utimes(localRefFile.c_str(), times);
}
if (!input->rev)
input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
if (!input.getRev())
input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@ -365,7 +379,7 @@ struct GitInput : Input
// FIXME: check whether rev is an ancestor of ref.
printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
/* Now that we know the ref, check again whether we have it in
the store. */
@ -387,7 +401,7 @@ struct GitInput : Input
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
@ -396,7 +410,7 @@ struct GitInput : Input
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
@ -406,18 +420,18 @@ struct GitInput : Input
auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
Attrs infoAttrs({
{"rev", input->rev->gitRev()},
{"rev", input.getRev()->gitRev()},
{"lastModified", lastModified},
});
if (!shallow)
infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
if (!this->rev)
if (!_input.getRev())
getCache()->add(
store,
mutableAttrs,
@ -436,60 +450,6 @@ struct GitInput : Input
}
};
struct GitInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "git" &&
url.scheme != "git+http" &&
url.scheme != "git+https" &&
url.scheme != "git+ssh" &&
url.scheme != "git+file") return nullptr;
auto url2(url);
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "git");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
throw Error("unsupported Git input attribute '%s'", name);
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
}