1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-04 03:01:47 +02:00

Merge remote-tracking branch 'upstream/master' into path-info

This commit is contained in:
John Ericson 2022-03-10 15:48:14 +00:00
commit 8ba089597f
434 changed files with 23391 additions and 36322 deletions

View file

@ -7,8 +7,8 @@ namespace nix {
static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
string s = state.coerceToString(pos, *args[0], context);
mkString(v, s, PathSet());
auto s = state.coerceToString(pos, *args[0], context);
v.mkString(*s);
}
static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
@ -18,7 +18,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
{
PathSet context;
state.forceString(*args[0], context, pos);
mkBool(v, !context.empty());
v.mkBool(!context.empty());
}
static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
@ -33,13 +33,13 @@ static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
string s = state.coerceToString(pos, *args[0], context);
auto s = state.coerceToString(pos, *args[0], context);
PathSet context2;
for (auto & p : context)
context2.insert(p.at(0) == '=' ? string(p, 1) : p);
context2.insert(p.at(0) == '=' ? std::string(p, 1) : p);
mkString(v, s, context2);
v.mkString(*s, context2);
}
static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
@ -76,13 +76,13 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
auto contextInfos = std::map<Path, ContextInfo>();
for (const auto & p : context) {
Path drv;
string output;
std::string output;
const Path * path = &p;
if (p.at(0) == '=') {
drv = string(p, 1);
drv = std::string(p, 1);
path = &drv;
} else if (p.at(0) == '!') {
std::pair<string, string> ctx = decodeContext(p);
std::pair<std::string, std::string> ctx = decodeContext(p);
drv = ctx.first;
output = ctx.second;
path = &drv;
@ -103,28 +103,26 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
}
}
state.mkAttrs(v, contextInfos.size());
auto attrs = state.buildBindings(contextInfos.size());
auto sPath = state.symbols.create("path");
auto sAllOutputs = state.symbols.create("allOutputs");
for (const auto & info : contextInfos) {
auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
state.mkAttrs(infoVal, 3);
auto infoAttrs = state.buildBindings(3);
if (info.second.path)
mkBool(*state.allocAttr(infoVal, sPath), true);
infoAttrs.alloc(sPath).mkBool(true);
if (info.second.allOutputs)
mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
infoAttrs.alloc(sAllOutputs).mkBool(true);
if (!info.second.outputs.empty()) {
auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
auto & outputsVal = infoAttrs.alloc(state.sOutputs);
state.mkList(outputsVal, info.second.outputs.size());
size_t i = 0;
for (const auto & output : info.second.outputs) {
mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
}
for (const auto & [i, output] : enumerate(info.second.outputs))
(outputsVal.listElems()[i] = state.allocValue())->mkString(output);
}
infoVal.attrs->sort();
attrs.alloc(info.first).mkAttrs(infoAttrs);
}
v.attrs->sort();
v.mkAttrs(attrs);
}
static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext);
@ -168,7 +166,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
.errPos = *i.pos
});
}
context.insert("=" + string(i.name));
context.insert("=" + std::string(i.name));
}
}
@ -181,14 +179,14 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
.errPos = *i.pos
});
}
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
context.insert("!" + name + "!" + string(i.name));
for (auto elem : iter->value->listItems()) {
auto name = state.forceStringNoCtx(*elem, *iter->pos);
context.insert(concatStrings("!", name, "!", i.name));
}
}
}
mkString(v, orig, context);
v.mkString(orig, context);
}
static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext);

View file

@ -12,24 +12,24 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
std::string url;
std::optional<Hash> rev;
std::optional<std::string> ref;
std::string name = "source";
std::string_view name = "source";
PathSet context;
state.forceValue(*args[0]);
state.forceValue(*args[0], pos);
if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
for (auto & attr : *args[0]->attrs) {
string n(attr.name);
std::string_view n(attr.name);
if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
url = state.coerceToString(*attr.pos, *attr.value, context, false, false).toOwned();
else if (n == "rev") {
// Ugly: unlike fetchGit, here the "rev" attribute can
// be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
if (std::regex_match(value, revRegex))
if (std::regex_match(value.begin(), value.end(), revRegex))
rev = Hash::parseAny(value, htSHA1);
else
ref = value;
@ -50,7 +50,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
});
} else
url = state.coerceToString(pos, *args[0], context, false, false);
url = state.coerceToString(pos, *args[0], context, false, false).toOwned();
// FIXME: git externals probably can be used to bypass the URI
// whitelist. Ah well.
@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "hg");
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
attrs.insert_or_assign("name", name);
attrs.insert_or_assign("name", std::string(name));
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::Input::fromAttrs(std::move(attrs));
@ -70,22 +70,21 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
// FIXME: use name
auto [tree, input2] = input.fetch(state.store);
state.mkAttrs(v, 8);
auto attrs2 = state.buildBindings(8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
attrs2.alloc(state.sOutPath).mkString(storePath, {storePath});
if (input2.getRef())
mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef());
attrs2.alloc("branch").mkString(*input2.getRef());
// Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2.getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
attrs2.alloc("rev").mkString(rev2.gitRev());
attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
if (auto revCount = input2.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
v.attrs->sort();
attrs2.alloc("revCount").mkInt(*revCount);
v.mkAttrs(attrs2);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
state.allowPath(tree.storePath);
}
static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);

View file

@ -19,54 +19,53 @@ void emitTreeAttrs(
bool emptyRevFallback,
bool forceDirty)
{
assert(input.isImmutable());
assert(input.isLocked());
state.mkAttrs(v, 8);
auto attrs = state.buildBindings(8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
attrs.alloc(state.sOutPath).mkString(storePath, {storePath});
// FIXME: support arbitrary input attributes.
auto narHash = input.getNarHash();
assert(narHash);
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
narHash->to_string(SRI, true));
attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
if (input.getType() == "git")
mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
attrs.alloc("submodules").mkBool(
fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
if (!forceDirty) {
if (auto rev = input.getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
attrs.alloc("rev").mkString(rev->gitRev());
attrs.alloc("shortRev").mkString(rev->gitShortRev());
} else if (emptyRevFallback) {
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(htSHA1);
mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
attrs.alloc("rev").mkString(emptyHash.gitRev());
attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
}
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
attrs.alloc("revCount").mkInt(*revCount);
else if (emptyRevFallback)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
attrs.alloc("revCount").mkInt(0);
}
if (auto lastModified = input.getLastModified()) {
mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
attrs.alloc("lastModified").mkInt(*lastModified);
attrs.alloc("lastModifiedDate").mkString(
fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
}
v.attrs->sort();
v.mkAttrs(attrs);
}
std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file")
std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
{
state.checkURI(uri);
return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
@ -74,53 +73,66 @@ std::string fixURI(std::string uri, EvalState &state, const std::string & defaul
std::string fixURIForGit(std::string uri, EvalState & state)
{
static std::regex scp_uri("([^/].*)@(.*):(.*)");
/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
* them by removing the `:` and assuming a scheme of `ssh://`
* */
static std::regex scp_uri("([^/]*)@(.*):(.*)");
if (uri[0] != '/' && std::regex_match(uri, scp_uri))
return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
else
return fixURI(uri, state);
}
void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
{
string n(name);
attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
}
struct FetchTreeParams {
bool emptyRevFallback = false;
bool allowNameArgument = false;
};
static void fetchTree(
EvalState &state,
const Pos &pos,
Value **args,
Value &v,
const std::optional<std::string> type,
EvalState & state,
const Pos & pos,
Value * * args,
Value & v,
std::optional<std::string> type,
const FetchTreeParams & params = FetchTreeParams{}
) {
fetchers::Input input;
PathSet context;
state.forceValue(*args[0]);
state.forceValue(*args[0], pos);
if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
fetchers::Attrs attrs;
if (auto aType = args[0]->attrs->get(state.sType)) {
if (type)
throw Error({
.msg = hintfmt("unexpected attribute 'type'"),
.errPos = pos
});
type = state.forceStringNoCtx(*aType->value, *aType->pos);
} else if (!type)
throw Error({
.msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos
});
attrs.emplace("type", type.value());
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
if (attr.value->type() == nPath || attr.value->type() == nString)
addURI(
state,
attrs,
attr.name,
state.coerceToString(*attr.pos, *attr.value, context, false, false)
);
else if (attr.value->type() == nString)
addURI(state, attrs, attr.name, attr.value->string.s);
if (attr.name == state.sType) continue;
state.forceValue(*attr.value, *attr.pos);
if (attr.value->type() == nPath || attr.value->type() == nString) {
auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false).toOwned();
attrs.emplace(attr.name,
attr.name == "url"
? type == "git"
? fixURIForGit(s, state)
: fixURI(s, state)
: s);
}
else if (attr.value->type() == nBool)
attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
else if (attr.value->type() == nInt)
@ -130,15 +142,6 @@ static void fetchTree(
attr.name, showType(*attr.value));
}
if (type)
attrs.emplace("type", type.value());
if (!attrs.count("type"))
throw Error({
.msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos
});
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
throw Error({
@ -146,10 +149,9 @@ static void fetchTree(
.errPos = pos
});
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
auto url = state.coerceToString(pos, *args[0], context, false, false);
auto url = state.coerceToString(pos, *args[0], context, false, false).toOwned();
if (type == "git") {
fetchers::Attrs attrs;
@ -164,20 +166,19 @@ static void fetchTree(
if (!evalSettings.pureEval && !input.isDirect())
input = lookupInRegistries(state.store, input).first;
if (evalSettings.pureEval && !input.isImmutable())
throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
if (evalSettings.pureEval && !input.isLocked())
throw Error("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", pos);
auto [tree, input2] = input.fetch(state.store);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
state.allowPath(tree.storePath);
emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
}
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
settings.requireExperimentalFeature("flakes");
settings.requireExperimentalFeature(Xp::Flakes);
fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
}
@ -185,19 +186,19 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree);
static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
const string & who, bool unpack, std::string name)
const std::string & who, bool unpack, std::string name)
{
std::optional<std::string> url;
std::optional<Hash> expectedHash;
state.forceValue(*args[0]);
state.forceValue(*args[0], pos);
if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
for (auto & attr : *args[0]->attrs) {
string n(attr.name);
std::string n(attr.name);
if (n == "url")
url = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "sha256")
@ -229,27 +230,41 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
if (evalSettings.pureEval && !expectedHash)
throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
// early exit if pinned and already in the store
if (expectedHash && expectedHash->type == htSHA256) {
auto expectedPath = state.store->makeFixedOutputPath(
name,
FixedOutputInfo {
{
.method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
.hash = *expectedHash,
},
{}
});
if (state.store->isValidPath(expectedPath)) {
state.allowAndSetStorePathString(expectedPath, v);
return;
}
}
// TODO: fetching may fail, yet the path may be substitutable.
// https://github.com/NixOS/nix/issues/4313
auto storePath =
unpack
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
auto realPath = state.store->toRealPath(storePath);
if (expectedHash) {
auto hash = unpack
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, realPath);
: hashFile(htSHA256, state.store->toRealPath(storePath));
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
if (state.allowedPaths)
state.allowedPaths->insert(realPath);
auto path = state.store->printStorePath(storePath);
mkString(v, path, PathSet({path}));
state.allowAndSetStorePathString(storePath, v);
}
static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
@ -291,13 +306,13 @@ static RegisterPrimOp primop_fetchTarball({
stdenv.mkDerivation { }
```
The fetched tarball is cached for a certain amount of time (1 hour
by default) in `~/.cache/nix/tarballs/`. You can change the cache
timeout either on the command line with `--option tarball-ttl number
of seconds` or in the Nix configuration file with this option: `
number of seconds to cache `.
The fetched tarball is cached for a certain amount of time (1
hour by default) in `~/.cache/nix/tarballs/`. You can change the
cache timeout either on the command line with `--tarball-ttl`
*number-of-seconds* or in the Nix configuration file by adding
the line `tarball-ttl = ` *number-of-seconds*.
Note that when obtaining the hash with ` nix-prefetch-url ` the
Note that when obtaining the hash with `nix-prefetch-url` the
option `--unpack` is required.
This function can also verify the contents against a hash. In that
@ -397,7 +412,7 @@ static RegisterPrimOp primop_fetchGit({
```
> **Note**
>
>
> It is nice to always specify the branch which a revision
> belongs to. Without the branch being specified, the fetcher
> might fail if the default branch changes. Additionally, it can
@ -434,12 +449,12 @@ static RegisterPrimOp primop_fetchGit({
```
> **Note**
>
>
> Nix will refetch the branch in accordance with
> the option `tarball-ttl`.
> **Note**
>
>
> This behavior is disabled in *Pure evaluation mode*.
)",
.fun = prim_fetchGit,

View file

@ -1,86 +1,76 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "../../cpptoml/cpptoml.h"
#include "../../toml11/toml.hpp"
namespace nix {
static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v)
static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & val)
{
using namespace cpptoml;
auto toml = state.forceStringNoCtx(*args[0], pos);
std::istringstream tomlStream(toml);
std::istringstream tomlStream(std::string{toml});
std::function<void(Value &, std::shared_ptr<base>)> visit;
std::function<void(Value &, toml::value)> visit;
visit = [&](Value & v, std::shared_ptr<base> t) {
visit = [&](Value & v, toml::value t) {
if (auto t2 = t->as_table()) {
switch(t.type())
{
case toml::value_t::table:
{
auto table = toml::get<toml::table>(t);
size_t size = 0;
for (auto & i : *t2) { (void) i; size++; }
size_t size = 0;
for (auto & i : table) { (void) i; size++; }
state.mkAttrs(v, size);
auto attrs = state.buildBindings(size);
for (auto & i : *t2) {
auto & v2 = *state.allocAttr(v, state.symbols.create(i.first));
for(auto & elem : table)
visit(attrs.alloc(elem.first), elem.second);
if (auto i2 = i.second->as_table_array()) {
size_t size2 = i2->get().size();
state.mkList(v2, size2);
for (size_t j = 0; j < size2; ++j)
visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
v.mkAttrs(attrs);
}
else
visit(v2, i.second);
}
break;;
case toml::value_t::array:
{
auto array = toml::get<std::vector<toml::value>>(t);
size_t size = array.size();
state.mkList(v, size);
for (size_t i = 0; i < size; ++i)
visit(*(v.listElems()[i] = state.allocValue()), array[i]);
}
break;;
case toml::value_t::boolean:
v.mkBool(toml::get<bool>(t));
break;;
case toml::value_t::integer:
v.mkInt(toml::get<int64_t>(t));
break;;
case toml::value_t::floating:
v.mkFloat(toml::get<NixFloat>(t));
break;;
case toml::value_t::string:
v.mkString(toml::get<std::string>(t));
break;;
case toml::value_t::local_datetime:
case toml::value_t::offset_datetime:
case toml::value_t::local_date:
case toml::value_t::local_time:
// We fail since Nix doesn't have date and time types
throw std::runtime_error("Dates and times are not supported");
break;;
case toml::value_t::empty:
v.mkNull();
break;;
v.attrs->sort();
}
else if (auto t2 = t->as_array()) {
size_t size = t2->get().size();
state.mkList(v, size);
for (size_t i = 0; i < size; ++i)
visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
}
// Handle cases like 'a = [[{ a = true }]]', which IMHO should be
// parsed as a array containing an array containing a table,
// but instead are parsed as an array containing a table array
// containing a table.
else if (auto t2 = t->as_table_array()) {
size_t size = t2->get().size();
state.mkList(v, size);
for (size_t j = 0; j < size; ++j)
visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]);
}
else if (t->is_value()) {
if (auto val = t->as<int64_t>())
mkInt(v, val->get());
else if (auto val = t->as<NixFloat>())
mkFloat(v, val->get());
else if (auto val = t->as<bool>())
mkBool(v, val->get());
else if (auto val = t->as<std::string>())
mkString(v, val->get());
else
throw EvalError("unsupported value type in TOML");
}
else abort();
};
try {
visit(v, parser(tomlStream).parse());
} catch (std::runtime_error & e) {
visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */));
} catch (std::exception & e) { // TODO: toml::syntax_error
throw EvalError({
.msg = hintfmt("while parsing a TOML string: %s", e.what()),
.errPos = pos