wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-07 01:51:47 +02:00
Code Activity

Merge remote-tracking branch 'origin/master' into flakes

Browse source
This commit is contained in:
Eelco Dolstra 2019-03-26 11:44:14 +01:00
commit b5565a7081
36 changed files with 470 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/build-remote/build-remote.cc
View file

@ -38,6 +38,12 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
}
static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
for (auto & feature : requiredFeatures)
if (!settings.systemFeatures.get().count(feature)) return false;
return true;
}
static int _main(int argc, char * * argv)
{
{
@ -97,9 +103,10 @@ static int _main(int argc, char * * argv)
source >> drvPath;
auto requiredFeatures = readStrings<std::set<std::string>>(source);
auto canBuildLocally = amWilling
&& ( neededSystem == settings.thisSystem
|| settings.extraPlatforms.get().count(neededSystem) > 0);
auto canBuildLocally = amWilling
&& ( neededSystem == settings.thisSystem
|| settings.extraPlatforms.get().count(neededSystem) > 0)
&& allSupportedLocally(requiredFeatures);
/* Error ignored here, will be caught later */
mkdir(currentLoad.c_str(), 0777);

16
src/libexpr/eval.cc
View file

@ -131,6 +131,16 @@ std::ostream & operator << (std::ostream & str, const Value & v)
}
const Value *getPrimOp(const Value &v) {
const Value * primOp = &v;
while (primOp->type == tPrimOpApp) {
primOp = primOp->primOpApp.left;
}
assert(primOp->type == tPrimOp);
return primOp;
}
string showType(const Value & v)
{
switch (v.type) {
@ -145,8 +155,10 @@ string showType(const Value & v)
case tApp: return "a function application";
case tLambda: return "a function";
case tBlackhole: return "a black hole";
case tPrimOp: return "a built-in function";
case tPrimOpApp: return "a partially applied built-in function";
case tPrimOp:
return fmt("the built-in function '%s'", string(v.primOp->name));
case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
case tExternal: return v.external->showType();
case tFloat: return "a float";
}

3
src/libexpr/eval.hh
View file

@ -327,6 +327,9 @@ private:
/* Return a string representing the type of the value `v'. */
string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(const string & s);
/* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path);

63
src/libexpr/primops.cc
View file

@ -315,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu
mkBool(v, args[0]->type == tBool);
}
/* Determine whether the argument is a path. */
static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0]);
mkBool(v, args[0]->type == tPath);
}
struct CompareValues
{
@ -687,21 +693,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}
}
/* See prim_unsafeDiscardOutputDependency. */
else if (path.at(0) == '~')
drv.inputSrcs.insert(string(path, 1));
/* Handle derivation outputs of the form ‘!<name>!<path>’. */
else if (path.at(0) == '!') {
std::pair<string, string> ctx = decodeContext(path);
drv.inputDrvs[ctx.first].insert(ctx.second);
}
/* Handle derivation contexts returned by
‘builtins.storePath’. */
else if (isDerivation(path))
drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path);
/* Otherwise it's a source file. */
else
drv.inputSrcs.insert(path);
@ -1004,13 +1001,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
PathSet refs;
for (auto path : context) {
if (path.at(0) == '=') path = string(path, 1);
if (isDerivation(path)) {
/* See prim_unsafeDiscardOutputDependency. */
if (path.at(0) != '~')
throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
path = string(path, 1);
}
if (path.at(0) != '/')
throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
refs.insert(path);
}
@ -1794,41 +1786,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
}
static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
string s = state.coerceToString(pos, *args[0], context);
mkString(v, s, PathSet());
}
static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
state.forceString(*args[0], context, pos);
mkBool(v, !context.empty());
}
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
builder without causing the derivation to be built (for instance,
in the derivation that builds NARs in nix-push, when doing
source-only deployment). This primop marks the string context so
that builtins.derivation adds the path to drv.inputSrcs rather than
drv.inputDrvs. */
static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
string s = state.coerceToString(pos, *args[0], context);
PathSet context2;
for (auto & p : context)
context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p);
mkString(v, s, context2);
}
/* Return the cryptographic hash of a string in base-16. */
static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
@ -2218,6 +2175,7 @@ void EvalState::createBaseEnv()
addPrimOp("__isInt", 1, prim_isInt);
addPrimOp("__isFloat", 1, prim_isFloat);
addPrimOp("__isBool", 1, prim_isBool);
addPrimOp("__isPath", 1, prim_isPath);
addPrimOp("__genericClosure", 1, prim_genericClosure);
addPrimOp("abort", 1, prim_abort);
addPrimOp("__addErrorContext", 2, prim_addErrorContext);
@ -2299,9 +2257,6 @@ void EvalState::createBaseEnv()
addPrimOp("toString", 1, prim_toString);
addPrimOp("__substring", 3, prim_substring);
addPrimOp("__stringLength", 1, prim_stringLength);
addPrimOp("__hasContext", 1, prim_hasContext);
addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
addPrimOp("__hashString", 2, prim_hashString);
addPrimOp("__match", 2, prim_match);
addPrimOp("__split", 2, prim_split);

187
src/libexpr/primops/context.cc Normal file
View file

@ -0,0 +1,187 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "derivations.hh"
namespace nix {
static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
string s = state.coerceToString(pos, *args[0], context);
mkString(v, s, PathSet());
}
static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
state.forceString(*args[0], context, pos);
mkBool(v, !context.empty());
}
static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
builder without causing the derivation to be built (for instance,
in the derivation that builds NARs in nix-push, when doing
source-only deployment). This primop marks the string context so
that builtins.derivation adds the path to drv.inputSrcs rather than
drv.inputDrvs. */
static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
string s = state.coerceToString(pos, *args[0], context);
PathSet context2;
for (auto & p : context)
context2.insert(p.at(0) == '=' ? string(p, 1) : p);
mkString(v, s, context2);
}
static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
/* Extract the context of a string as a structured Nix value.
The context is represented as an attribute set whose keys are the
paths in the context set and whose values are attribute sets with
the following keys:
path: True if the relevant path is in the context as a plain store
path (i.e. the kind of context you get when interpolating
a Nix path (e.g. ./.) into a string). False if missing.
allOutputs: True if the relevant path is a derivation and it is
in the context as a drv file with all of its outputs
(i.e. the kind of context you get when referencing
.drvPath of some derivation). False if missing.
outputs: If a non-empty list, the relevant path is a derivation
and the provided outputs are referenced in the context
(i.e. the kind of context you get when referencing
.outPath of some derivation). Empty list if missing.
Note that for a given path any combination of the above attributes
may be present.
*/
static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
struct ContextInfo {
bool path = false;
bool allOutputs = false;
Strings outputs;
};
PathSet context;
state.forceString(*args[0], context, pos);
auto contextInfos = std::map<Path, ContextInfo>();
for (const auto & p : context) {
Path drv;
string output;
const Path * path = &p;
if (p.at(0) == '=') {
drv = string(p, 1);
path = &drv;
} else if (p.at(0) == '!') {
std::pair<string, string> ctx = decodeContext(p);
drv = ctx.first;
output = ctx.second;
path = &drv;
}
auto isPath = drv.empty();
auto isAllOutputs = (!drv.empty()) && output.empty();
auto iter = contextInfos.find(*path);
if (iter == contextInfos.end()) {
contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
} else {
if (isPath)
iter->second.path = true;
else if (isAllOutputs)
iter->second.allOutputs = true;
else
iter->second.outputs.emplace_back(std::move(output));
}
}
state.mkAttrs(v, contextInfos.size());
auto sPath = state.symbols.create("path");
auto sAllOutputs = state.symbols.create("allOutputs");
for (const auto & info : contextInfos) {
auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
state.mkAttrs(infoVal, 3);
if (info.second.path)
mkBool(*state.allocAttr(infoVal, sPath), true);
if (info.second.allOutputs)
mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
if (!info.second.outputs.empty()) {
auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
state.mkList(outputsVal, info.second.outputs.size());
size_t i = 0;
for (const auto & output : info.second.outputs) {
mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
}
}
infoVal.attrs->sort();
}
v.attrs->sort();
}
static RegisterPrimOp r4("__getContext", 1, prim_getContext);
/* Append the given context to a given string.
See the commentary above unsafeGetContext for details of the
context representation.
*/
static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
auto orig = state.forceString(*args[0], context, pos);
state.forceAttrs(*args[1], pos);
auto sPath = state.symbols.create("path");
auto sAllOutputs = state.symbols.create("allOutputs");
for (auto & i : *args[1]->attrs) {
if (!state.store->isStorePath(i.name))
throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
if (!settings.readOnlyMode)
state.store->ensurePath(i.name);
state.forceAttrs(*i.value, *i.pos);
auto iter = i.value->attrs->find(sPath);
if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos))
context.insert(i.name);
}
iter = i.value->attrs->find(sAllOutputs);
if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) {
throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
}
context.insert("=" + string(i.name));
}
}
iter = i.value->attrs->find(state.sOutputs);
if (iter != i.value->attrs->end()) {
state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
}
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
context.insert("!" + name + "!" + string(i.name));
}
}
}
mkString(v, orig, context);
}
static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
}

31
src/libstore/download.cc
View file

@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader
writeFull(wakeupPipe.writeSide.get(), " ");
}
#ifdef ENABLE_S3
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
{
auto [path, params] = splitUriAndParams(uri);
auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", path);
std::string bucketName(path, 5, slash - 5);
std::string key(path, slash + 1);
return {bucketName, key, params};
}
#endif
void enqueueDownload(const DownloadRequest & request,
Callback<DownloadResult> callback) override
{
@ -622,12 +638,15 @@ struct CurlDownloader : public Downloader
// FIXME: do this on a worker thread
try {
#ifdef ENABLE_S3
S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
auto slash = request.uri.find('/', 5);
if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", request.uri);
std::string bucketName(request.uri, 5, slash - 5);
std::string key(request.uri, slash + 1);
auto [bucketName, key, params] = parseS3Uri(request.uri);
std::string profile = get(params, "profile", "");
std::string region = get(params, "region", Aws::Region::US_EAST_1);
std::string scheme = get(params, "scheme", "");
std::string endpoint = get(params, "endpoint", "");
S3Helper s3Helper(profile, region, scheme, endpoint);
// FIXME: implement ETag
auto s3Res = s3Helper.getObject(bucketName, key);
DownloadResult res;

119
src/libstore/gc.cc
View file

@ -129,8 +129,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
check if the root is in a directory in or linked from the
gcroots directory. */
if (settings.checkRootReachability) {
Roots roots = findRoots();
if (roots.find(gcRoot) == roots.end())
Roots roots = findRoots(false);
if (roots[storePath].count(gcRoot) == 0)
printError(
format(
"warning: '%1%' is not in a directory where the garbage collector looks for roots; "
@ -197,10 +197,11 @@ void LocalStore::addTempRoot(const Path & path)
}
std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
{
std::set<std::pair<pid_t, Path>> tempRoots;
static std::string censored = "{censored}";
void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
{
/* Read the `temproots' directory for per-process temporary root
files. */
for (auto & i : readDirectory(tempRootsDir)) {
@ -250,14 +251,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
Path root(contents, pos, end - pos);
debug("got temporary root '%s'", root);
assertStorePath(root);
tempRoots.emplace(pid, root);
tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
pos = end + 1;
}
fds.push_back(fd); /* keep open */
}
return tempRoots;
}
@ -266,7 +265,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
auto foundRoot = [&](const Path & path, const Path & target) {
Path storePath = toStorePath(target);
if (isStorePath(storePath) && isValidPath(storePath))
roots[path] = storePath;
roots[storePath].emplace(path);
else
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
};
@ -306,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
else if (type == DT_REG) {
Path storePath = storeDir + "/" + baseNameOf(path);
if (isStorePath(storePath) && isValidPath(storePath))
roots[path] = storePath;
roots[storePath].emplace(path);
}
}
@ -321,10 +320,8 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
}
Roots LocalStore::findRootsNoTemp()
void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
{
Roots roots;
/* Process direct roots in {gcroots,profiles}. */
findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
@ -333,32 +330,22 @@ Roots LocalStore::findRootsNoTemp()
NIX_ROOT_FINDER environment variable. This is typically used
to add running programs to the set of roots (to prevent them
from being garbage collected). */
size_t n = 0;
for (auto & root : findRuntimeRoots())
roots[fmt("{memory:%d}", n++)] = root;
return roots;
findRuntimeRoots(roots, censor);
}
Roots LocalStore::findRoots()
Roots LocalStore::findRoots(bool censor)
{
Roots roots = findRootsNoTemp();
Roots roots;
findRootsNoTemp(roots, censor);
FDs fds;
pid_t prev = -1;
size_t n = 0;
for (auto & root : readTempRoots(fds)) {
if (prev != root.first) n = 0;
prev = root.first;
roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second;
}
findTempRoots(fds, roots, censor);
return roots;
}
static void readProcLink(const string & file, StringSet & paths)
static void readProcLink(const string & file, Roots & roots)
{
/* 64 is the starting buffer size gnu readlink uses... */
auto bufsiz = ssize_t{64};
@ -377,8 +364,8 @@ try_again:
goto try_again;
}
if (res > 0 && buf[0] == '/')
paths.emplace(static_cast<char *>(buf), res);
return;
roots[std::string(static_cast<char *>(buf), res)]
.emplace(file);
}
static string quoteRegexChars(const string & raw)
@ -387,20 +374,20 @@ static string quoteRegexChars(const string & raw)
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
static void readFileRoots(const char * path, StringSet & paths)
static void readFileRoots(const char * path, Roots & roots)
{
try {
paths.emplace(readFile(path));
roots[readFile(path)].emplace(path);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != EACCES)
throw;
}
}
PathSet LocalStore::findRuntimeRoots()
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
PathSet roots;
StringSet paths;
Roots unchecked;
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent * ent;
@ -410,10 +397,10 @@ PathSet LocalStore::findRuntimeRoots()
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) {
readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths);
readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths);
readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str();
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) {
if (errno == ENOENT || errno == EACCES)
@ -422,9 +409,8 @@ PathSet LocalStore::findRuntimeRoots()
}
struct dirent * fd_ent;
while (errno = 0, fd_ent = readdir(fdDir.get())) {
if (fd_ent->d_name[0] != '.') {
readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths);
}
if (fd_ent->d_name[0] != '.')
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
}
if (errno) {
if (errno == ESRCH)
@ -434,18 +420,19 @@ PathSet LocalStore::findRuntimeRoots()
fdDir.reset();
try {
auto mapLines =
tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
for (const auto& line : mapLines) {
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
paths.emplace(match[1]);
unchecked[match[1]].emplace(mapFile);
}
auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true);
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile, true);
auto env_end = std::sregex_iterator{};
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
paths.emplace(i->str());
unchecked[i->str()].emplace(envFile);
} catch (SysError & e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
continue;
@ -465,7 +452,7 @@ PathSet LocalStore::findRuntimeRoots()
for (const auto & line : lsofLines) {
std::smatch match;
if (std::regex_match(line, match, lsofRegex))
paths.emplace(match[1]);
unchecked[match[1]].emplace("{lsof}");
}
} catch (ExecError & e) {
/* lsof not installed, lsof failed */
@ -473,21 +460,23 @@ PathSet LocalStore::findRuntimeRoots()
#endif
#if defined(__linux__)
readFileRoots("/proc/sys/kernel/modprobe", paths);
readFileRoots("/proc/sys/kernel/fbsplash", paths);
readFileRoots("/proc/sys/kernel/poweroff_cmd", paths);
readFileRoots("/proc/sys/kernel/modprobe", unchecked);
readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
#endif
for (auto & i : paths)
if (isInStore(i)) {
Path path = toStorePath(i);
if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) {
for (auto & [target, links] : unchecked) {
if (isInStore(target)) {
Path path = toStorePath(target);
if (isStorePath(path) && isValidPath(path)) {
debug(format("got additional root '%1%'") % path);
roots.insert(path);
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
}
}
return roots;
}
}
@ -754,16 +743,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */
printError(format("finding garbage collector roots..."));
Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp();
Roots rootMap;
if (!options.ignoreLiveness)
findRootsNoTemp(rootMap, true);
for (auto & i : rootMap) state.roots.insert(i.second);
for (auto & i : rootMap) state.roots.insert(i.first);
/* Read the temporary roots. This acquires read locks on all
per-process temporary root files. So after this point no paths
can be added to the set of temporary roots. */
FDs fds;
for (auto & root : readTempRoots(fds))
state.tempRoots.insert(root.second);
Roots tempRoots;
findTempRoots(fds, tempRoots, true);
for (auto & root : tempRoots)
state.tempRoots.insert(root.first);
state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
/* After this point the set of roots or temporary roots cannot

8
src/libstore/local-store.hh
View file

@ -180,11 +180,11 @@ private:
typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds);
void findTempRoots(FDs & fds, Roots & roots, bool censor);
public:
Roots findRoots() override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;
@ -267,9 +267,9 @@ private:
void findRoots(const Path & path, unsigned char type, Roots & roots);
Roots findRootsNoTemp();
void findRootsNoTemp(Roots & roots, bool censor);
PathSet findRuntimeRoots();
void findRuntimeRoots(Roots & roots, bool censor);
void removeUnusedLinks(const GCState & state);

4
src/libstore/remote-store.cc
View file

@ -596,7 +596,7 @@ void RemoteStore::syncWithGC()
}
Roots RemoteStore::findRoots()
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection());
conn->to << wopFindRoots;
@ -606,7 +606,7 @@ Roots RemoteStore::findRoots()
while (count--) {
Path link = readString(conn->from);
Path target = readStorePath(*this, conn->from);
result[link] = target;
result[target].emplace(link);
}
return result;
}

2
src/libstore/remote-store.hh
View file

@ -82,7 +82,7 @@ public:
void syncWithGC() override;
Roots findRoots() override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;

1
src/libstore/s3-binary-cache-store.cc
View file

@ -126,6 +126,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
res->endpointOverride = endpoint;
}
res->requestTimeoutMs = 600 * 1000;
res->connectTimeoutMs = 5 * 1000;
res->retryStrategy = std::make_shared<RetryStrategy>();
res->caFile = settings.caFile;
return res;

16
src/libstore/store-api.cc
View file

@ -842,12 +842,11 @@ namespace nix {
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
Store::Params params(extraParams);
Store::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
@ -873,6 +872,15 @@ ref<Store> openStore(const std::string & uri_,
}
uri = uri_.substr(0, q);
}
return {uri, params};
}
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
{
auto [uri, uriParams] = splitUriAndParams(uri_);
auto params = extraParams;
params.insert(uriParams.begin(), uriParams.end());
for (auto fun : *RegisterStoreImplementation::implementations) {
auto store = fun(uri, params);

14
src/libstore/store-api.hh
View file

@ -11,6 +11,8 @@
#include <atomic>
#include <limits>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <memory>
#include <string>
@ -47,7 +49,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
const uint32_t exportMagic = 0x4558494e;
typedef std::map<Path, Path> Roots;
typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
struct GCOptions
@ -483,8 +485,10 @@ public:
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. */
virtual Roots findRoots()
outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */
virtual Roots findRoots(bool censor)
{ unsupported("findRoots"); }
/* Perform a garbage collection. */
@ -798,4 +802,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(bool recursive, const Hash & hash);
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
}

2
src/libutil/archive.cc
View file

@ -331,7 +331,7 @@ struct RestoreSink : ParseSink
filesystem doesn't support preallocation (e.g. on
OpenSolaris). Since preallocation is just an
optimisation, ignore it. */
if (errno && errno != EINVAL)
if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
throw SysError(format("preallocating file of %1% bytes") % len);
}
#endif

14
src/nix-daemon/nix-daemon.cc
View file

@ -475,11 +475,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopFindRoots: {
logger->startWork();
Roots roots = store->findRoots();
Roots roots = store->findRoots(!trusted);
logger->stopWork();
to << roots.size();
size_t size = 0;
for (auto & i : roots)
to << i.first << i.second;
size += i.second.size();
to << size;
for (auto & [target, links] : roots)
for (auto & link : links)
to << link << target;
break;
}

35
src/nix-store/nix-store.cc
View file

@ -427,10 +427,11 @@ static void opQuery(Strings opFlags, Strings opArgs)
maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise),
referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
}
Roots roots = store->findRoots();
for (auto & i : roots)
if (referrers.find(i.second) != referrers.end())
cout << format("%1%\n") % i.first;
Roots roots = store->findRoots(false);
for (auto & [target, links] : roots)
if (referrers.find(target) != referrers.end())
for (auto & link : links)
cout << format("%1% -> %2%\n") % link % target;
break;
}
@ -485,11 +486,16 @@ static void opReadLog(Strings opFlags, Strings opArgs)
static void opDumpDB(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty()) throw UsageError("unknown flag");
if (!opArgs.empty())
throw UsageError("no arguments expected");
PathSet validPaths = store->queryAllValidPaths();
for (auto & i : validPaths)
cout << store->makeValidityRegistration({i}, true, true);
if (!opArgs.empty()) {
for (auto & i : opArgs)
i = store->followLinksToStorePath(i);
for (auto & i : opArgs)
cout << store->makeValidityRegistration({i}, true, true);
} else {
PathSet validPaths = store->queryAllValidPaths();
for (auto & i : validPaths)
cout << store->makeValidityRegistration({i}, true, true);
}
}
@ -585,9 +591,14 @@ static void opGC(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected");
if (printRoots) {
Roots roots = store->findRoots();
for (auto & i : roots)
cout << i.first << " -> " << i.second << std::endl;
Roots roots = store->findRoots(false);
std::set<std::pair<Path, Path>> roots2;
// Transpose and sort the roots.
for (auto & [target, links] : roots)
for (auto & link : links)
roots2.emplace(link, target);
for (auto & [link, target] : roots2)
std::cout << link << " -> " << target << "\n";
}
else {