wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-07 10:11:47 +02:00
Code Activity

Merge remote-tracking branch 'upstream/master' into multi-output-hashDerivationModulo

Browse source
This commit is contained in:
John Ericson 2020-06-17 03:51:02 +00:00
commit 02928f76fd
251 changed files with 10279 additions and 3660 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/build-remote/build-remote.cc
View file

@ -17,7 +17,7 @@
#include "store-api.hh"
#include "derivations.hh"
#include "local-store.hh"
#include "legacy.hh"
#include "../nix/legacy.hh"
using namespace nix;
using std::cin;
@ -200,9 +200,12 @@ static int _main(int argc, char * * argv)
} catch (std::exception & e) {
auto msg = chomp(drainFD(5, false));
printError("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
(msg.empty() ? "" : ": " + msg));
logError({
.name = "Remote build",
.hint = hintfmt("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
(msg.empty() ? "" : ": " + msg))
});
bestMachine->enabled = false;
continue;
}
@ -241,7 +244,7 @@ connected:
uploadLock = -1;
BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
auto drv = store->readDerivation(*drvPath);
drv.inputSrcs = store->parseStorePathSet(inputs);
auto result = sshStore->buildDerivation(*drvPath, drv);

34
src/libexpr/attr-path.cc
View file

@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s)
++i;
while (1) {
if (i == s.end())
throw Error(format("missing closing quote in selection path '%1%'") % s);
throw Error("missing closing quote in selection path '%1%'", s);
if (*i == '"') break;
cur.push_back(*i++);
}
@ -32,15 +32,13 @@ static Strings parseAttrPath(const string & s)
}
Value * findAlongAttrPath(EvalState & state, const string & attrPath,
std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Value & vIn)
{
Strings tokens = parseAttrPath(attrPath);
Error attrError =
Error(format("attribute selection path '%1%' does not match expression") % attrPath);
Value * v = &vIn;
Pos pos = noPos;
for (auto & attr : tokens) {
@ -62,34 +60,36 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
if (v->type != tAttrs)
throw TypeError(
format("the expression selected by the selection path '%1%' should be a set but is %2%")
% attrPath % showType(*v));
"the expression selected by the selection path '%1%' should be a set but is %2%",
attrPath,
showType(*v));
if (attr.empty())
throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
throw Error("empty attribute name in selection path '%1%'", attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
if (a == v->attrs->end())
throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
v = &*a->value;
pos = *a->pos;
}
else if (apType == apIndex) {
if (!v->isList())
throw TypeError(
format("the expression selected by the selection path '%1%' should be a list but is %2%")
% attrPath % showType(*v));
"the expression selected by the selection path '%1%' should be a list but is %2%",
attrPath,
showType(*v));
if (attrIndex >= v->listSize())
throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
v = v->listElems()[attrIndex];
pos = noPos;
}
}
return v;
return {v, pos};
}
@ -98,9 +98,9 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
Value * v2;
try {
auto dummyArgs = state.allocBindings(0);
v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v);
v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
} catch (Error &) {
throw Error("package '%s' has no source location information", what);
throw NoPositionInfo("package '%s' has no source location information", what);
}
// FIXME: is it possible to extract the Pos object instead of doing this

5
src/libexpr/attr-path.hh
View file

@ -7,7 +7,10 @@
namespace nix {
Value * findAlongAttrPath(EvalState & state, const string & attrPath,
MakeError(AttrPathNotFound, Error);
MakeError(NoPositionInfo, Error);
std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Value & vIn);
/* Heuristic to find the filename and lineno or a nix value. */

6
src/libexpr/attr-set.cc
View file

@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
}
Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
{
return allocAttr(vAttrs, symbols.create(name));
}
void Bindings::sort()
{
std::sort(begin(), end());

6
src/libexpr/attr-set.hh
View file

@ -76,7 +76,11 @@ public:
{
auto a = get(name);
if (!a)
throw Error("attribute '%s' missing, at %s", name, pos);
throw Error({
.hint = hintfmt("attribute '%s' missing", name),
.nixCode = NixCode { .errPos = pos }
});
return *a;
}

45
src/libexpr/common-eval-args.cc
View file

@ -1,31 +1,36 @@
#include "common-eval-args.hh"
#include "shared.hh"
#include "download.hh"
#include "filetransfer.hh"
#include "util.hh"
#include "eval.hh"
#include "fetchers.hh"
#include "store-api.hh"
namespace nix {
MixEvalArgs::MixEvalArgs()
{
mkFlag()
.longName("arg")
.description("argument to be passed to Nix functions")
.labels({"name", "expr"})
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
addFlag({
.longName = "arg",
.description = "argument to be passed to Nix functions",
.labels = {"name", "expr"},
.handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
});
mkFlag()
.longName("argstr")
.description("string-valued argument to be passed to Nix functions")
.labels({"name", "string"})
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
addFlag({
.longName = "argstr",
.description = "string-valued argument to be passed to Nix functions",
.labels = {"name", "string"},
.handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
});
mkFlag()
.shortName('I')
.longName("include")
.description("add a path to the list of locations used to look up <...> file names")
.label("path")
.handler([&](std::string s) { searchPath.push_back(s); });
addFlag({
.longName = "include",
.shortName = 'I',
.description = "add a path to the list of locations used to look up <...> file names",
.labels = {"path"},
.handler = {[&](std::string s) { searchPath.push_back(s); }}
});
}
Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
@ -46,9 +51,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
Path lookupFileArg(EvalState & state, string s)
{
if (isUri(s)) {
CachedDownloadRequest request(s);
request.unpack = true;
return getDownloader()->downloadCached(state.store, request).path;
return state.store->toRealPath(
fetchers::downloadTarball(
state.store, resolveUri(s), "source", false).storePath);
} else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
Path p = s.substr(1, s.size() - 2);
return state.findFile(p);

26
src/libexpr/eval-inline.hh
View file

@ -7,20 +7,26 @@
namespace nix {
LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
{
throw EvalError(format(s) % pos);
throw EvalError({
.hint = hintfmt(s),
.nixCode = NixCode { .errPos = pos }
});
}
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
{
throw TypeError(format(s) % showType(v));
throw TypeError(s, showType(v));
}
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
{
throw TypeError(format(s) % showType(v) % pos);
throw TypeError({
.hint = hintfmt(s, showType(v)),
.nixCode = NixCode { .errPos = pos }
});
}
@ -43,7 +49,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
else if (v.type == tApp)
callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.type == tBlackhole)
throwEvalError("infinite recursion encountered, at %1%", pos);
throwEvalError(pos, "infinite recursion encountered");
}
@ -57,9 +63,9 @@ inline void EvalState::forceAttrs(Value & v)
inline void EvalState::forceAttrs(Value & v, const Pos & pos)
{
forceValue(v);
forceValue(v, pos);
if (v.type != tAttrs)
throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
throwTypeError(pos, "value is %1% while a set was expected", v);
}
@ -73,9 +79,9 @@ inline void EvalState::forceList(Value & v)
inline void EvalState::forceList(Value & v, const Pos & pos)
{
forceValue(v);
forceValue(v, pos);
if (!v.isList())
throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
throwTypeError(pos, "value is %1% while a list was expected", v);
}
/* Note: Various places expect the allocated memory to be zeroed. */

126
src/libexpr/eval.cc
View file

@ -5,7 +5,7 @@
#include "derivations.hh"
#include "globals.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "filetransfer.hh"
#include "json.hh"
#include "function-trace.hh"
@ -22,6 +22,8 @@
#if HAVE_BOEHMGC
#define GC_INCLUDE_NEW
#include <gc/gc.h>
#include <gc/gc_cpp.h>
@ -56,6 +58,12 @@ static char * dupStringWithLen(const char * s, size_t size)
}
RootValue allocRootValue(Value * v)
{
return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
}
static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
{
checkInterrupt();
@ -493,52 +501,74 @@ Value & EvalState::getBuiltin(const string & name)
LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
{
throw EvalError(format(s) % s2);
throw EvalError(s, s2);
}
LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos))
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
{
throw EvalError(format(s) % s2 % pos);
throw EvalError({
.hint = hintfmt(s, s2),
.nixCode = NixCode { .errPos = pos }
});
}
LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
{
throw EvalError(format(s) % s2 % s3);
throw EvalError(s, s2, s3);
}
LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos))
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
{
throw EvalError(format(s) % s2 % s3 % pos);
throw EvalError({
.hint = hintfmt(s, s2, s3),
.nixCode = NixCode { .errPos = pos }
});
}
LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2))
{
throw EvalError(format(s) % sym % p1 % p2);
// p1 is where the error occurred; p2 is a position mentioned in the message.
throw EvalError({
.hint = hintfmt(s, sym, p2),
.nixCode = NixCode { .errPos = p1 }
});
}
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
{
throw TypeError(format(s) % pos);
throw TypeError({
.hint = hintfmt(s),
.nixCode = NixCode { .errPos = pos }
});
}
LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
{
throw TypeError(format(s) % s1);
throw TypeError(s, s1);
}
LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos))
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
{
throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
throw TypeError({
.hint = hintfmt(s, fun.showNamePos(), s2),
.nixCode = NixCode { .errPos = pos }
});
}
LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
{
throw AssertionError(format(s) % s1 % pos);
throw AssertionError({
.hint = hintfmt(s, s1),
.nixCode = NixCode { .errPos = pos }
});
}
LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
{
throw UndefinedVarError(format(s) % s1 % pos);
throw UndefinedVarError({
.hint = hintfmt(s, s1),
.nixCode = NixCode { .errPos = pos }
});
}
LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
@ -606,7 +636,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
return j->value;
}
if (!env->prevWith)
throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name);
for (size_t l = env->prevWith; l; --l, env = env->up) ;
}
}
@ -804,7 +834,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
Value v;
e->eval(*this, env, v);
if (v.type != tBool)
throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
throwTypeError(pos, "value is %1% while a Boolean was expected", v);
return v.boolean;
}
@ -918,7 +948,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
Symbol nameSym = state.symbols.create(nameVal.string.s);
Bindings::iterator j = v.attrs->find(nameSym);
if (j != v.attrs->end())
throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos);
throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos);
i.valueExpr->setName(nameSym);
/* Keep sorted order so find can catch duplicates */
@ -1006,7 +1036,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
} else {
state.forceAttrs(*vAttrs, pos);
if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
throwEvalError("attribute '%1%' missing, at %2%", name, pos);
throwEvalError(pos, "attribute '%1%' missing", name);
}
vAttrs = j->value;
pos2 = j->pos;
@ -1132,7 +1162,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
}
if (fun.type != tLambda)
throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos);
throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
ExprLambda & lambda(*fun.lambda.fun);
@ -1160,8 +1190,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
for (auto & i : lambda.formals->formals) {
Bindings::iterator j = arg.attrs->find(i.name);
if (j == arg.attrs->end()) {
if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%",
lambda, i.name, pos);
if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
lambda, i.name);
env2.values[displ++] = i.def->maybeThunk(*this, env2);
} else {
attrsUsed++;
@ -1176,7 +1206,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
user. */
for (auto & i : *arg.attrs)
if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos);
throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
abort(); // can't happen
}
}
@ -1256,7 +1286,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
void ExprIf::eval(EvalState & state, Env & env, Value & v)
{
(state.evalBool(env, cond) ? then : else_)->eval(state, env, v);
(state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v);
}
@ -1265,7 +1295,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
if (!state.evalBool(env, cond, pos)) {
std::ostringstream out;
cond->show(out);
throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
}
body->eval(state, env, v);
}
@ -1417,14 +1447,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
nf = n;
nf += vTmp.fpoint;
} else
throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos);
throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
} else if (firstType == tFloat) {
if (vTmp.type == tInt) {
nf += vTmp.integer;
} else if (vTmp.type == tFloat) {
nf += vTmp.fpoint;
} else
throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos);
throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
} else
s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
}
@ -1435,7 +1465,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
mkFloat(v, nf);
else if (firstType == tPath) {
if (!context.empty())
throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos);
throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
auto path = canonPath(s.str());
mkPath(v, path.c_str());
} else
@ -1484,7 +1514,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
{
forceValue(v, pos);
if (v.type != tInt)
throwTypeError("value is %1% while an integer was expected, at %2%", v, pos);
throwTypeError(pos, "value is %1% while an integer was expected", v);
return v.integer;
}
@ -1495,16 +1525,16 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
if (v.type == tInt)
return v.integer;
else if (v.type != tFloat)
throwTypeError("value is %1% while a float was expected, at %2%", v, pos);
throwTypeError(pos, "value is %1% while a float was expected", v);
return v.fpoint;
}
bool EvalState::forceBool(Value & v, const Pos & pos)
{
forceValue(v);
forceValue(v, pos);
if (v.type != tBool)
throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
throwTypeError(pos, "value is %1% while a Boolean was expected", v);
return v.boolean;
}
@ -1517,9 +1547,9 @@ bool EvalState::isFunctor(Value & fun)
void EvalState::forceFunction(Value & v, const Pos & pos)
{
forceValue(v);
forceValue(v, pos);
if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
throwTypeError("value is %1% while a function was expected, at %2%", v, pos);
throwTypeError(pos, "value is %1% while a function was expected", v);
}
@ -1528,7 +1558,7 @@ string EvalState::forceString(Value & v, const Pos & pos)
forceValue(v, pos);
if (v.type != tString) {
if (pos)
throwTypeError("value is %1% while a string was expected, at %2%", v, pos);
throwTypeError(pos, "value is %1% while a string was expected", v);
else
throwTypeError("value is %1% while a string was expected", v);
}
@ -1557,8 +1587,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
string s = forceString(v, pos);
if (v.string.context) {
if (pos)
throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%",
v.string.s, v.string.context[0], pos);
throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
v.string.s, v.string.context[0]);
else
throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
v.string.s, v.string.context[0]);
@ -1594,7 +1624,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
bool coerceMore, bool copyToStore)
{
forceValue(v);
forceValue(v, pos);
string s;
@ -1614,7 +1644,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
return *maybeString;
}
auto i = v.attrs->find(sOutPath);
if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos);
if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
}
@ -1645,7 +1675,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
}
}
throwTypeError("cannot coerce %1% to a string, at %2%", v, pos);
throwTypeError(pos, "cannot coerce %1% to a string", v);
}
@ -1661,7 +1691,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
else {
auto p = settings.readOnlyMode
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p);
srcToStore.insert_or_assign(path, std::move(p));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
@ -1676,7 +1706,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
{
string path = coerceToString(pos, v, context, false, false);
if (path == "" || path[0] != '/')
throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos);
throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path);
return path;
}
@ -1883,8 +1913,10 @@ void EvalState::printStats()
string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
{
throw TypeError(format("cannot coerce %1% to a string, at %2%") %
showType() % pos);
throw TypeError({
.hint = hintfmt("cannot coerce %1% to a string", showType()),
.nixCode = NixCode { .errPos = pos }
});
}

3
src/libexpr/eval.hh
View file

@ -272,6 +272,7 @@ public:
Env & allocEnv(size_t size);
Value * allocAttr(Value & vAttrs, const Symbol & name);
Value * allocAttr(Value & vAttrs, const std::string & name);
Bindings * allocBindings(size_t capacity);
@ -367,7 +368,7 @@ struct EvalSettings : Config
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
};
extern EvalSettings evalSettings;

1
src/libexpr/function-trace.cc
View file

@ -1,4 +1,5 @@
#include "function-trace.hh"
#include "logging.hh"
namespace nix {

63
src/libexpr/json-to-value.cc
View file

@ -4,7 +4,6 @@
#include <nlohmann/json.hpp>
using json = nlohmann::json;
using std::unique_ptr;
namespace nix {
@ -13,69 +12,69 @@ namespace nix {
class JSONSax : nlohmann::json_sax<json> {
class JSONState {
protected:
unique_ptr<JSONState> parent;
Value * v;
std::unique_ptr<JSONState> parent;
RootValue v;
public:
virtual unique_ptr<JSONState> resolve(EvalState &)
virtual std::unique_ptr<JSONState> resolve(EvalState &)
{
throw std::logic_error("tried to close toplevel json parser state");
};
explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
explicit JSONState(Value* v) : v(v) {};
JSONState(JSONState& p) = delete;
Value& value(EvalState & state)
}
explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {}
explicit JSONState(Value * v) : v(allocRootValue(v)) {}
JSONState(JSONState & p) = delete;
Value & value(EvalState & state)
{
if (v == nullptr)
v = state.allocValue();
return *v;
};
virtual ~JSONState() {};
virtual void add() {};
if (!v)
v = allocRootValue(state.allocValue());
return **v;
}
virtual ~JSONState() {}
virtual void add() {}
};
class JSONObjectState : public JSONState {
using JSONState::JSONState;
ValueMap attrs = ValueMap();
virtual unique_ptr<JSONState> resolve(EvalState & state) override
ValueMap attrs;
std::unique_ptr<JSONState> resolve(EvalState & state) override
{
Value& v = parent->value(state);
Value & v = parent->value(state);
state.mkAttrs(v, attrs.size());
for (auto & i : attrs)
v.attrs->push_back(Attr(i.first, i.second));
return std::move(parent);
}
virtual void add() override { v = nullptr; };
void add() override { v = nullptr; }
public:
void key(string_t& name, EvalState & state)
void key(string_t & name, EvalState & state)
{
attrs[state.symbols.create(name)] = &value(state);
attrs.insert_or_assign(state.symbols.create(name), &value(state));
}
};
class JSONListState : public JSONState {
ValueVector values = ValueVector();
virtual unique_ptr<JSONState> resolve(EvalState & state) override
ValueVector values;
std::unique_ptr<JSONState> resolve(EvalState & state) override
{
Value& v = parent->value(state);
Value & v = parent->value(state);
state.mkList(v, values.size());
for (size_t n = 0; n < values.size(); ++n) {
v.listElems()[n] = values[n];
}
return std::move(parent);
}
virtual void add() override {
values.push_back(v);
void add() override {
values.push_back(*v);
v = nullptr;
};
}
public:
JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
{
values.reserve(reserve);
}
};
EvalState & state;
unique_ptr<JSONState> rs;
std::unique_ptr<JSONState> rs;
template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
{
@ -107,12 +106,12 @@ public:
return handle_value(mkInt, val);
}
bool number_float(number_float_t val, const string_t& s)
bool number_float(number_float_t val, const string_t & s)
{
return handle_value(mkFloat, val);
}
bool string(string_t& val)
bool string(string_t & val)
{
return handle_value<void(Value&, const char*)>(mkString, val.c_str());
}
@ -123,7 +122,7 @@ public:
return true;
}
bool key(string_t& name)
bool key(string_t & name)
{
dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
return true;

5
src/libexpr/lexer.l
View file

@ -127,14 +127,14 @@ or { return OR_KW; }
try {
yylval->n = boost::lexical_cast<int64_t>(yytext);
} catch (const boost::bad_lexical_cast &) {
throw ParseError(format("invalid integer '%1%'") % yytext);
throw ParseError("invalid integer '%1%'", yytext);
}
return INT;
}
{FLOAT} { errno = 0;
yylval->nf = strtod(yytext, 0);
if (errno != 0)
throw ParseError(format("invalid float '%1%'") % yytext);
throw ParseError("invalid float '%1%'", yytext);
return FLOAT;
}
@ -219,4 +219,3 @@ or { return OR_KW; }
}
%%

4
src/libexpr/local.mk
View file

@ -6,7 +6,9 @@ libexpr_DIR := $(d)
libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
libexpr_LIBS = libutil libstore libnixrust
libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
libexpr_LIBS = libutil libstore libfetchers
libexpr_LDFLAGS =
ifneq ($(OS), FreeBSD)

7
src/libexpr/nixexpr.cc
View file

@ -267,8 +267,11 @@ void ExprVar::bindVars(const StaticEnv & env)
/* Otherwise, the variable must be obtained from the nearest
enclosing `with'. If there is no `with', then we can issue an
"undefined variable" error now. */
if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
if (withLevel == -1)
throw UndefinedVarError({
.hint = hintfmt("undefined variable '%1%'", name),
.nixCode = NixCode { .errPos = pos }
});
fromWith = true;
this->level = withLevel;
}

13
src/libexpr/nixexpr.hh
View file

@ -2,6 +2,7 @@
#include "value.hh"
#include "symbol-table.hh"
#include "error.hh"
#include <map>
@ -209,9 +210,10 @@ struct ExprList : Expr
struct Formal
{
Pos pos;
Symbol name;
Expr * def;
Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { };
};
struct Formals
@ -234,8 +236,10 @@ struct ExprLambda : Expr
: pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
{
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError(format("duplicate formal function argument '%1%' at %2%")
% arg % pos);
throw ParseError({
.hint = hintfmt("duplicate formal function argument '%1%'", arg),
.nixCode = NixCode { .errPos = pos }
});
};
void setName(Symbol & name);
string showNamePos() const;
@ -261,8 +265,9 @@ struct ExprWith : Expr
struct ExprIf : Expr
{
Pos pos;
Expr * cond, * then, * else_;
ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
COMMON_METHODS
};

83
src/libexpr/parser.y
View file

@ -31,7 +31,7 @@ namespace nix {
Expr * result;
Path basePath;
Symbol path;
string error;
ErrorInfo error;
Symbol sLetBody;
ParseData(EvalState & state)
: state(state)
@ -64,15 +64,20 @@ namespace nix {
static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
{
throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
% showAttrPath(attrPath) % pos % prevPos);
throw ParseError({
.hint = hintfmt("attribute '%1%' already defined at %2%",
showAttrPath(attrPath), prevPos),
.nixCode = NixCode { .errPos = pos },
});
}
static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
{
throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
% attr % pos % prevPos);
throw ParseError({
.hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
.nixCode = NixCode { .errPos = pos },
});
}
@ -140,8 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
{
if (!formals->argNames.insert(formal.name).second)
throw ParseError(format("duplicate formal function argument '%1%' at %2%")
% formal.name % pos);
throw ParseError({
.hint = hintfmt("duplicate formal function argument '%1%'",
formal.name),
.nixCode = NixCode { .errPos = pos },
});
formals->formals.push_front(formal);
}
@ -249,8 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
{
data->error = (format("%1%, at %2%")
% error % makeCurPos(*loc, data)).str();
data->error = {
.hint = hintfmt(error),
.nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
};
}
@ -327,15 +337,17 @@ expr_function
{ $$ = new ExprWith(CUR_POS, $2, $4); }
| LET binds IN expr_function
{ if (!$2->dynamicAttrs.empty())
throw ParseError(format("dynamic attributes not allowed in let at %1%")
% CUR_POS);
throw ParseError({
.hint = hintfmt("dynamic attributes not allowed in let"),
.nixCode = NixCode { .errPos = CUR_POS },
});
$$ = new ExprLet($2, $4);
}
| expr_if
;
expr_if
: IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
: IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
| expr_op
;
@ -405,7 +417,10 @@ expr_simple
| URI {
static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
if (noURLLiterals)
throw ParseError("URL literals are disabled, at %s", CUR_POS);
throw ParseError({
.hint = hintfmt("URL literals are disabled"),
.nixCode = NixCode { .errPos = CUR_POS }
});
$$ = new ExprString(data->symbols.create($1));
}
| '(' expr ')' { $$ = $2; }
@ -475,8 +490,10 @@ attrs
$$->push_back(AttrName(str->s));
delete str;
} else
throw ParseError(format("dynamic attributes not allowed in inherit at %1%")
% makeCurPos(@2, data));
throw ParseError({
.hint = hintfmt("dynamic attributes not allowed in inherit"),
.nixCode = NixCode { .errPos = makeCurPos(@2, data) },
});
}
| { $$ = new AttrPath; }
;
@ -531,8 +548,8 @@ formals
;
formal
: ID { $$ = new Formal(data->symbols.create($1), 0); }
| ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
: ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); }
| ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); }
;
%%
@ -544,7 +561,8 @@ formal
#include <unistd.h>
#include "eval.hh"
#include "download.hh"
#include "filetransfer.hh"
#include "fetchers.hh"
#include "store-api.hh"
@ -670,11 +688,13 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
Path res = r.second + suffix;
if (pathExists(res)) return canonPath(res);
}
format f = format(
"file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
+ string(pos ? ", at %2%" : ""));
f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
throw ThrownError(f % path % pos);
throw ThrownError({
.hint = hintfmt(evalSettings.pureEval
? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
: "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
path),
.nixCode = NixCode { .errPos = pos }
});
}
@ -687,11 +707,13 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (isUri(elem.second)) {
try {
CachedDownloadRequest request(elem.second);
request.unpack = true;
res = { true, getDownloader()->downloadCached(store, request).path };
} catch (DownloadError & e) {
printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
res = { true, store->toRealPath(fetchers::downloadTarball(
store, resolveUri(elem.second), "source", false).storePath) };
} catch (FileTransferError & e) {
logWarning({
.name = "Entry download",
.hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
});
res = { false, "" };
}
} else {
@ -699,7 +721,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (pathExists(path))
res = { true, path };
else {
printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second);
logWarning({
.name = "Entry not found",
.hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
});
res = { false, "" };
}
}

478
src/libexpr/primops.cc
View file

File diff suppressed because it is too large Load diff

1
src/libexpr/primops.hh
View file

@ -20,6 +20,7 @@ struct RegisterPrimOp
them. */
/* Load a ValueInitializer from a DSO and return whatever it initializes */
void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
/* Execute a program and parse its output */
void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);

15
src/libexpr/primops/context.cc
View file

@ -146,7 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
auto sAllOutputs = state.symbols.create("allOutputs");
for (auto & i : *args[1]->attrs) {
if (!state.store->isStorePath(i.name))
throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
throw EvalError({
.hint = hintfmt("Context key '%s' is not a store path", i.name),
.nixCode = NixCode { .errPos = *i.pos }
});
if (!settings.readOnlyMode)
state.store->ensurePath(state.store->parseStorePath(i.name));
state.forceAttrs(*i.value, *i.pos);
@ -160,7 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) {
throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
throw EvalError({
.hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
.nixCode = NixCode { .errPos = *i.pos }
});
}
context.insert("=" + string(i.name));
}
@ -170,7 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (iter != i.value->attrs->end()) {
state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
throw EvalError({
.hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
.nixCode = NixCode { .errPos = *i.pos }
});
}
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

237
src/libexpr/primops/fetchGit.cc
View file

@ -1,203 +1,19 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "hash.hh"
#include "tarfile.hh"
#include <sys/time.h>
#include <regex>
#include <nlohmann/json.hpp>
using namespace std::string_literals;
#include "fetchers.hh"
#include "url.hh"
namespace nix {
struct GitInfo
{
Path storePath;
std::string rev;
std::string shortRev;
uint64_t revCount = 0;
};
std::regex revRegex("^[0-9a-fA-F]{40}$");
GitInfo exportGit(ref<Store> store, const std::string & uri,
std::optional<std::string> ref, std::string rev,
const std::string & name)
{
if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
bool clean = true;
try {
runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
clean = false;
}
if (!clean) {
/* This is an unclean working tree. So copy all tracked files. */
GitInfo gitInfo;
gitInfo.rev = "0000000000000000000000000000000000000000";
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, uri));
std::string file(p, uri.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
return gitInfo;
}
// clean working tree, but no ref or rev specified. Use 'HEAD'.
rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
ref = "HEAD"s;
}
if (!ref) ref = "HEAD"s;
if (rev != "" && !std::regex_match(rev, revRegex))
throw Error("invalid Git revision '%s'", rev);
deletePath(getCacheDir() + "/nix/git");
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir));
runProgram("git", true, { "init", "--bare", cacheDir });
}
Path localRefFile;
if (ref->compare(0, 5, "refs/") == 0)
localRefFile = cacheDir + "/" + *ref;
else
localRefFile = cacheDir + "/refs/heads/" + *ref;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (rev != "") {
try {
runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
} else {
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
}
if (doFetch)
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
}
// FIXME: check whether rev is an ancestor of ref.
GitInfo gitInfo;
gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
Path storeLink = cacheDir + "/" + storeLinkName + ".link";
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
try {
auto json = nlohmann::json::parse(readFile(storeLink));
assert(json["name"] == name && json["rev"] == gitInfo.rev);
gitInfo.storePath = json["storePath"];
if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
gitInfo.revCount = json["revCount"];
return gitInfo;
}
} catch (SysError & e) {
if (e.errNo != ENOENT) throw;
}
auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
unpackTarfile(*source, tmpDir);
gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
nlohmann::json json;
json["storePath"] = gitInfo.storePath;
json["uri"] = uri;
json["name"] = name;
json["rev"] = gitInfo.rev;
json["revCount"] = gitInfo.revCount;
writeFile(storeLink, json.dump());
return gitInfo;
}
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
std::string url;
std::optional<std::string> ref;
std::string rev;
std::optional<Hash> rev;
std::string name = "source";
bool fetchSubmodules = false;
PathSet context;
state.forceValue(*args[0]);
@ -213,15 +29,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
else if (n == "ref")
ref = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "submodules")
fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
.nixCode = NixCode { .errPos = *attr.pos }
});
}
if (url.empty())
throw EvalError(format("'url' argument required, at %1%") % pos);
throw EvalError({
.hint = hintfmt("'url' argument required"),
.nixCode = NixCode { .errPos = pos }
});
} else
url = state.coerceToString(pos, *args[0], context, false, false);
@ -230,17 +54,36 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
// whitelist. Ah well.
state.checkURI(url);
auto gitInfo = exportGit(state.store, url, ref, rev, name);
if (evalSettings.pureEval && !rev)
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
auto input = fetchers::inputFromAttrs(attrs);
// FIXME: use name?
auto [tree, input2] = input->fetchTree(state.store);
state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
// Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2->getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
// Backward compatibility: set 'revCount' to 0 for a dirty tree.
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
tree.info.revCount.value_or(0));
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
v.attrs->sort();
if (state.allowedPaths)
state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
state.allowedPaths->insert(tree.actualPath);
}
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

216
src/libexpr/primops/fetchMercurial.cc
View file

@ -1,174 +1,18 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include <sys/time.h>
#include "fetchers.hh"
#include "url.hh"
#include <regex>
#include <nlohmann/json.hpp>
using namespace std::string_literals;
namespace nix {
struct HgInfo
{
Path storePath;
std::string branch;
std::string rev;
uint64_t revCount = 0;
};
std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
HgInfo exportMercurial(ref<Store> store, const std::string & uri,
std::string rev, const std::string & name)
{
if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
if (!clean) {
/* This is an unclean working tree. So copy all tracked
files. */
printTalkative("copying unclean Mercurial working tree '%s'", uri);
HgInfo hgInfo;
hgInfo.rev = "0000000000000000000000000000000000000000";
hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, uri));
std::string file(p, uri.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
return hgInfo;
}
}
if (rev == "") rev = "default";
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
/* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
do so now. */
time_t now = time(0);
struct stat st;
if (stat(stampFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
{
/* Except that if this is a commit hash that we already have,
we don't have to pull again. */
if (!(std::regex_match(rev, commitHashRegex)
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
if (pathExists(cacheDir)) {
try {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
}
}
writeFile(stampFile, "");
}
auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
HgInfo hgInfo;
hgInfo.rev = tokens[0];
hgInfo.revCount = std::stoull(tokens[1]);
hgInfo.branch = tokens[2];
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
try {
auto json = nlohmann::json::parse(readFile(storeLink));
assert(json["name"] == name && json["rev"] == hgInfo.rev);
hgInfo.storePath = json["storePath"];
if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
return hgInfo;
}
} catch (SysError & e) {
if (e.errNo != ENOENT) throw;
}
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
nlohmann::json json;
json["storePath"] = hgInfo.storePath;
json["uri"] = uri;
json["name"] = name;
json["branch"] = hgInfo.branch;
json["rev"] = hgInfo.rev;
json["revCount"] = hgInfo.revCount;
writeFile(storeLink, json.dump());
return hgInfo;
}
static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
std::string url;
std::string rev;
std::optional<Hash> rev;
std::optional<std::string> ref;
std::string name = "source";
PathSet context;
@ -182,16 +26,29 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
string n(attr.name);
if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "rev") {
// Ugly: unlike fetchGit, here the "rev" attribute can
// be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
if (std::regex_match(value, revRegex))
rev = Hash(value, htSHA1);
else
ref = value;
}
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
.nixCode = NixCode { .errPos = *attr.pos }
});
}
if (url.empty())
throw EvalError(format("'url' argument required, at %1%") % pos);
throw EvalError({
.hint = hintfmt("'url' argument required"),
.nixCode = NixCode { .errPos = pos }
});
} else
url = state.coerceToString(pos, *args[0], context, false, false);
@ -200,18 +57,35 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
// whitelist. Ah well.
state.checkURI(url);
auto hgInfo = exportMercurial(state.store, url, rev, name);
if (evalSettings.pureEval && !rev)
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "hg");
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::inputFromAttrs(attrs);
// FIXME: use name
auto [tree, input2] = input->fetchTree(state.store);
state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
if (input2->getRef())
mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
// Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2->getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
if (tree.info.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
v.attrs->sort();
if (state.allowedPaths)
state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
state.allowedPaths->insert(tree.actualPath);
}
static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

172
src/libexpr/primops/fetchTree.cc Normal file
View file

@ -0,0 +1,172 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "fetchers.hh"
#include "filetransfer.hh"
#include <ctime>
#include <iomanip>
namespace nix {
void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
std::shared_ptr<const fetchers::Input> input,
Value & v)
{
state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
assert(tree.info.narHash);
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
tree.info.narHash.to_string(SRI, true));
if (input->getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
}
if (tree.info.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
if (tree.info.lastModified)
mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
v.attrs->sort();
}
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
settings.requireExperimentalFeature("flakes");
std::shared_ptr<const fetchers::Input> input;
PathSet context;
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
fetchers::Attrs attrs;
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
if (attr.value->type == tString)
attrs.emplace(attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
attrs.emplace(attr.name, attr.value->boolean);
else
throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
attr.name, showType(*attr.value));
}
if (!attrs.count("type"))
throw Error({
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.nixCode = NixCode { .errPos = pos }
});
input = fetchers::inputFromAttrs(attrs);
} else
input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
if (evalSettings.pureEval && !input->isImmutable())
throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
// FIXME: use fetchOrSubstituteTree
auto [tree, input2] = input->fetchTree(state.store);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
emitTreeAttrs(state, tree, input2, v);
}
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
const string & who, bool unpack, std::string name)
{
std::optional<std::string> url;
std::optional<Hash> expectedHash;
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
for (auto & attr : *args[0]->attrs) {
string n(attr.name);
if (n == "url")
url = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
.nixCode = NixCode { .errPos = *attr.pos }
});
}
if (!url)
throw EvalError({
.hint = hintfmt("'url' argument required"),
.nixCode = NixCode { .errPos = pos }
});
} else
url = state.forceStringNoCtx(*args[0], pos);
url = resolveUri(*url);
state.checkURI(*url);
if (name == "")
name = baseNameOf(*url);
if (evalSettings.pureEval && !expectedHash)
throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
auto storePath =
unpack
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
auto path = state.store->toRealPath(storePath);
if (expectedHash) {
auto hash = unpack
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
if (state.allowedPaths)
state.allowedPaths->insert(path);
mkString(v, path, PathSet({path}));
}
static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
fetch(state, pos, args, v, "fetchurl", false, "");
}
static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
fetch(state, pos, args, v, "fetchTarball", true, "source");
}
static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
}

7
src/libexpr/primops/fromTOML.cc
View file

@ -1,7 +1,7 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "cpptoml/cpptoml.h"
#include "../../cpptoml/cpptoml.h"
namespace nix {
@ -81,7 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
try {
visit(v, parser(tomlStream).parse());
} catch (std::runtime_error & e) {
throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
throw EvalError({
.hint = hintfmt("while parsing a TOML string: %s", e.what()),
.nixCode = NixCode { .errPos = pos }
});
}
}

4
src/libexpr/value-to-json.cc
View file

@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
default:
throw TypeError(format("cannot convert %1% to JSON") % showType(v));
throw TypeError("cannot convert %1% to JSON", showType(v));
}
}
@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict,
void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
JSONPlaceholder & out, PathSet & context) const
{
throw TypeError(format("cannot convert %1% to JSON") % showType());
throw TypeError("cannot convert %1% to JSON", showType());
}

9
src/libexpr/value.hh
View file

@ -253,12 +253,17 @@ void mkPath(Value & v, const char * s);
#if HAVE_BOEHMGC
typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
#else
typedef std::vector<Value *> ValueVector;
typedef std::map<Symbol, Value *> ValueMap;
#endif
/* A value allocated in traceable memory. */
typedef std::shared_ptr<Value *> RootValue;
RootValue allocRootValue(Value * v);
}

107
src/libfetchers/attrs.cc Normal file
View file

@ -0,0 +1,107 @@
#include "attrs.hh"
#include "fetchers.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
Attrs jsonToAttrs(const nlohmann::json & json)
{
Attrs attrs;
for (auto & i : json.items()) {
if (i.value().is_number())
attrs.emplace(i.key(), i.value().get<int64_t>());
else if (i.value().is_string())
attrs.emplace(i.key(), i.value().get<std::string>());
else if (i.value().is_boolean())
attrs.emplace(i.key(), i.value().get<bool>());
else
throw Error("unsupported input attribute type in lock file");
}
return attrs;
}
nlohmann::json attrsToJson(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
json[attr.first] = v->t;
} else abort();
}
return json;
}
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<std::string>(&i->second))
return *v;
throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
}
std::string getStrAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetStrAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second))
return *v;
throw Error("input attribute '%s' is not an integer", name);
}
int64_t getIntAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetIntAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second))
return *v;
throw Error("input attribute '%s' is not a Boolean", name);
}
bool getBoolAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetBoolAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
{
std::map<std::string, std::string> query;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
query.insert_or_assign(attr.first, fmt("%d", *v));
} else if (auto v = std::get_if<std::string>(&attr.second)) {
query.insert_or_assign(attr.first, *v);
} else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
query.insert_or_assign(attr.first, v->t ? "1" : "0");
} else abort();
}
return query;
}
}

39
src/libfetchers/attrs.hh Normal file
View file

@ -0,0 +1,39 @@
#pragma once
#include "types.hh"
#include <variant>
#include <nlohmann/json_fwd.hpp>
namespace nix::fetchers {
/* Wrap bools to prevent string literals (i.e. 'char *') from being
cast to a bool in Attr. */
template<typename T>
struct Explicit {
T t;
};
typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
nlohmann::json attrsToJson(const Attrs & attrs);
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
std::string getStrAttr(const Attrs & attrs, const std::string & name);
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
int64_t getIntAttr(const Attrs & attrs, const std::string & name);
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
bool getBoolAttr(const Attrs & attrs, const std::string & name);
std::map<std::string, std::string> attrsToQuery(const Attrs & attrs);
}

121
src/libfetchers/cache.cc Normal file
View file

@ -0,0 +1,121 @@
#include "cache.hh"
#include "sqlite.hh"
#include "sync.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
static const char * schema = R"sql(
create table if not exists Cache (
input text not null,
info text not null,
path text not null,
immutable integer not null,
timestamp integer not null,
primary key (input)
);
)sql";
struct CacheImpl : Cache
{
struct State
{
SQLite db;
SQLiteStmt add, lookup;
};
Sync<State> _state;
CacheImpl()
{
auto state(_state.lock());
auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->add.create(state->db,
"insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
state->lookup.create(state->db,
"select info, path, immutable, timestamp from Cache where input = ?");
}
void add(
ref<Store> store,
const Attrs & inAttrs,
const Attrs & infoAttrs,
const StorePath & storePath,
bool immutable) override
{
_state.lock()->add.use()
(attrsToJson(inAttrs).dump())
(attrsToJson(infoAttrs).dump())
(store->printStorePath(storePath))
(immutable)
(time(0)).exec();
}
std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store,
const Attrs & inAttrs) override
{
if (auto res = lookupExpired(store, inAttrs)) {
if (!res->expired)
return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
debug("ignoring expired cache entry '%s'",
attrsToJson(inAttrs).dump());
}
return {};
}
std::optional<Result> lookupExpired(
ref<Store> store,
const Attrs & inAttrs) override
{
auto state(_state.lock());
auto inAttrsJson = attrsToJson(inAttrs).dump();
auto stmt(state->lookup.use()(inAttrsJson));
if (!stmt.next()) {
debug("did not find cache entry for '%s'", inAttrsJson);
return {};
}
auto infoJson = stmt.getStr(0);
auto storePath = store->parseStorePath(stmt.getStr(1));
auto immutable = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3);
store->addTempRoot(storePath);
if (!store->isValidPath(storePath)) {
// FIXME: we could try to substitute 'storePath'.
debug("ignoring disappeared cache entry '%s'", inAttrsJson);
return {};
}
debug("using cache entry '%s' -> '%s', '%s'",
inAttrsJson, infoJson, store->printStorePath(storePath));
return Result {
.expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
.infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
.storePath = std::move(storePath)
};
}
};
ref<Cache> getCache()
{
static auto cache = std::make_shared<CacheImpl>();
return ref<Cache>(cache);
}
}

34
src/libfetchers/cache.hh Normal file
View file

@ -0,0 +1,34 @@
#pragma once
#include "fetchers.hh"
namespace nix::fetchers {
struct Cache
{
virtual void add(
ref<Store> store,
const Attrs & inAttrs,
const Attrs & infoAttrs,
const StorePath & storePath,
bool immutable) = 0;
virtual std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store,
const Attrs & inAttrs) = 0;
struct Result
{
bool expired = false;
Attrs infoAttrs;
StorePath storePath;
};
virtual std::optional<Result> lookupExpired(
ref<Store> store,
const Attrs & inAttrs) = 0;
};
ref<Cache> getCache();
}

75
src/libfetchers/fetchers.cc Normal file
View file

@ -0,0 +1,75 @@
#include "fetchers.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
{
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
inputSchemes->push_back(std::move(inputScheme));
}
std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
{
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromURL(url);
if (res) return res;
}
throw Error("input '%s' is unsupported", url.url);
}
std::unique_ptr<Input> inputFromURL(const std::string & url)
{
return inputFromURL(parseURL(url));
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
{
auto attrs2(attrs);
attrs2.erase("narHash");
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromAttrs(attrs2);
if (res) {
if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
// FIXME: require SRI hash.
res->narHash = newHashAllowEmpty(*narHash, htUnknown);
return res;
}
}
throw Error("input '%s' is unsupported", attrsToJson(attrs));
}
Attrs Input::toAttrs() const
{
auto attrs = toAttrsInternal();
if (narHash)
attrs.emplace("narHash", narHash->to_string(SRI, true));
attrs.emplace("type", type());
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
{
auto [tree, input] = fetchTreeInternal(store);
if (tree.actualPath == "")
tree.actualPath = store->toRealPath(tree.storePath);
if (!tree.info.narHash)
tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
if (input->narHash)
assert(input->narHash == tree.info.narHash);
if (narHash && narHash != input->narHash)
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
return {std::move(tree), input};
}
}

103
src/libfetchers/fetchers.hh Normal file
View file

@ -0,0 +1,103 @@
#pragma once
#include "types.hh"
#include "hash.hh"
#include "path.hh"
#include "tree-info.hh"
#include "attrs.hh"
#include "url.hh"
#include <memory>
namespace nix { class Store; }
namespace nix::fetchers {
struct Input;
struct Tree
{
Path actualPath;
StorePath storePath;
TreeInfo info;
};
struct Input : std::enable_shared_from_this<Input>
{
std::optional<Hash> narHash; // FIXME: implement
virtual std::string type() const = 0;
virtual ~Input() { }
virtual bool operator ==(const Input & other) const { return false; }
/* Check whether this is a "direct" input, that is, not
one that goes through a registry. */
virtual bool isDirect() const { return true; }
/* Check whether this is an "immutable" input, that is,
one that contains a commit hash or content hash. */
virtual bool isImmutable() const { return (bool) narHash; }
virtual bool contains(const Input & other) const { return false; }
virtual std::optional<std::string> getRef() const { return {}; }
virtual std::optional<Hash> getRev() const { return {}; }
virtual ParsedURL toURL() const = 0;
std::string to_string() const
{
return toURL().to_string();
}
Attrs toAttrs() const;
std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
private:
virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
virtual Attrs toAttrsInternal() const = 0;
};
struct InputScheme
{
virtual ~InputScheme() { }
virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
};
std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
std::unique_ptr<Input> inputFromURL(const std::string & url);
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
struct DownloadFileResult
{
StorePath storePath;
std::string etag;
std::string effectiveUrl;
};
DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable);
Tree downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable);
}

441
src/libfetchers/git.cc Normal file
View file

@ -0,0 +1,441 @@
#include "fetchers.hh"
#include "cache.hh"
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include <sys/time.h>
using namespace std::string_literals;
namespace nix::fetchers {
static std::string readHead(const Path & path)
{
return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
}
static bool isNotDotGitDirectory(const Path & path)
{
static const std::regex gitDirRegex("^(?:.*/)?\\.git$");
return not std::regex_match(path, gitDirRegex);
}
struct GitInput : Input
{
ParsedURL url;
std::optional<std::string> ref;
std::optional<Hash> rev;
bool shallow = false;
bool submodules = false;
GitInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "git"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const GitInput *>(&other);
return
other2
&& url == other2->url
&& rev == other2->rev
&& ref == other2->ref;
}
bool isImmutable() const override
{
return (bool) rev || narHash;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
ParsedURL url2(url);
if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref);
if (shallow) url2.query.insert_or_assign("shallow", "1");
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
if (shallow)
attrs.emplace("shallow", true);
if (submodules)
attrs.emplace("submodules", true);
return attrs;
}
std::pair<bool, std::string> getActualUrl() const
{
// Don't clone file:// URIs (but otherwise treat them the
// same as remote URIs, i.e. don't use the working tree or
// HEAD).
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
bool isLocal = url.scheme == "file" && !forceHttp;
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto name = "source";
auto input = std::make_shared<GitInput>(*this);
assert(!rev || rev->type == htSHA1);
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
if (submodules) cacheType += "-submodules";
auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", cacheType},
{"name", name},
{"rev", input->rev->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
{
assert(input->rev);
assert(!rev || rev == input->rev);
return {
Tree {
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
.lastModified = getIntAttr(infoAttrs, "lastModified"),
},
},
input
};
};
if (rev) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
auto [isLocal, actualUrl_] = getActualUrl();
auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
if (!input->ref && !input->rev && isLocal) {
bool clean = false;
/* Check whether this repo has any commits. There are
probably better ways to do this. */
auto gitDir = actualUrl + "/.git";
auto commonGitDir = chomp(runProgram(
"git",
true,
{ "-C", actualUrl, "rev-parse", "--git-common-dir" }
));
if (commonGitDir != ".git")
gitDir = commonGitDir;
bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
try {
if (haveCommits) {
runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
clean = true;
}
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
}
if (!clean) {
/* This is an unclean working tree. So copy all tracked files. */
if (!settings.allowDirty)
throw Error("Git tree '%s' is dirty", actualUrl);
if (settings.warnDirty)
warn("Git tree '%s' is dirty", actualUrl);
auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" });
if (submodules)
gitOpts.emplace_back("--recurse-submodules");
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, gitOpts), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualUrl));
std::string file(p, actualUrl.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
auto tree = Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
.lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
}
};
return {std::move(tree), input};
}
}
if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
Attrs mutableAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
});
Path repoDir;
if (isLocal) {
if (!input->rev)
input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
repoDir = actualUrl;
} else {
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
return makeResult(res->first, std::move(res->second));
}
}
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
repoDir = cacheDir;
if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir));
runProgram("git", true, { "init", "--bare", repoDir });
}
Path localRefFile =
input->ref->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input->ref
: cacheDir + "/refs/heads/" + *input->ref;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (input->rev) {
try {
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
} else {
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
}
if (doFetch) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
try {
auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
? *input->ref
: "refs/heads/" + *input->ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
}
struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
}
if (!input->rev)
input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
if (isShallow && !shallow)
throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
// FIXME: check whether rev is an ancestor of ref.
printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
/* Now that we know the ref, check again whether we have it in
the store. */
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;
if (submodules) {
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);
runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
// TODO: repoDir might lack the ref (it only checks if rev
// exists, see FIXME above) so use a big hammer and fetch
// everything to ensure we get the rev.
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
filter = isNotDotGitDirectory;
} else {
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
unpackTarfile(*source, tmpDir);
}
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
Attrs infoAttrs({
{"rev", input->rev->gitRev()},
{"lastModified", lastModified},
});
if (!shallow)
infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
if (!this->rev)
getCache()->add(
store,
mutableAttrs,
infoAttrs,
storePath,
false);
getCache()->add(
store,
getImmutableAttrs(),
infoAttrs,
storePath,
true);
return makeResult(infoAttrs, std::move(storePath));
}
};
struct GitInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "git" &&
url.scheme != "git+http" &&
url.scheme != "git+https" &&
url.scheme != "git+ssh" &&
url.scheme != "git+file") return nullptr;
auto url2(url);
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "git");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
throw Error("unsupported Git input attribute '%s'", name);
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (std::regex_search(*ref, badGitRefRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
}

195
src/libfetchers/github.cc Normal file
View file

@ -0,0 +1,195 @@
#include "filetransfer.hh"
#include "cache.hh"
#include "fetchers.hh"
#include "globals.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
struct GitHubInput : Input
{
std::string owner;
std::string repo;
std::optional<std::string> ref;
std::optional<Hash> rev;
std::string type() const override { return "github"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const GitHubInput *>(&other);
return
other2
&& owner == other2->owner
&& repo == other2->repo
&& rev == other2->rev
&& ref == other2->ref;
}
bool isImmutable() const override
{
return (bool) rev || narHash;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
auto path = owner + "/" + repo;
assert(!(ref && rev));
if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(Base16, false);
return ParsedURL {
.scheme = "github",
.path = path,
};
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("owner", owner);
attrs.emplace("repo", repo);
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto rev = this->rev;
auto ref = this->ref.value_or("master");
if (!rev) {
auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
owner, repo, ref);
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
rev = Hash(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev->gitRev());
}
auto input = std::make_shared<GitHubInput>(*this);
input->ref = {};
input->rev = *rev;
Attrs immutableAttrs({
{"type", "git-tarball"},
{"rev", rev->gitRev()},
});
if (auto res = getCache()->lookup(store, immutableAttrs)) {
return {
Tree{
.actualPath = store->toRealPath(res->second),
.storePath = std::move(res->second),
.info = TreeInfo {
.lastModified = getIntAttr(res->first, "lastModified"),
},
},
input
};
}
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
owner, repo, rev->to_string(Base16, false));
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
url += "?access_token=" + accessToken;
auto tree = downloadTarball(store, url, "source", true);
getCache()->add(
store,
immutableAttrs,
{
{"rev", rev->gitRev()},
{"lastModified", *tree.info.lastModified}
},
tree.storePath,
true);
return {std::move(tree), input};
}
};
struct GitHubInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "github") return nullptr;
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
auto input = std::make_unique<GitHubInput>();
if (path.size() == 2) {
} else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex))
input->rev = Hash(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
input->ref = path[2];
else
throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
for (auto &[name, value] : url.query) {
if (name == "rev") {
if (input->rev)
throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
input->rev = Hash(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
if (input->ref)
throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
input->ref = value;
}
}
if (input->ref && input->rev)
throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
input->owner = path[0];
input->repo = path[1];
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "github") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
throw Error("unsupported GitHub input attribute '%s'", name);
auto input = std::make_unique<GitHubInput>();
input->owner = getStrAttr(attrs, "owner");
input->repo = getStrAttr(attrs, "repo");
input->ref = maybeGetStrAttr(attrs, "ref");
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
}

11
src/libfetchers/local.mk Normal file
View file

@ -0,0 +1,11 @@
libraries += libfetchers
libfetchers_NAME = libnixfetchers
libfetchers_DIR := $(d)
libfetchers_SOURCES := $(wildcard $(d)/*.cc)
libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
libfetchers_LIBS = libutil libstore

303
src/libfetchers/mercurial.cc Normal file
View file

@ -0,0 +1,303 @@
#include "fetchers.hh"
#include "cache.hh"
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include <sys/time.h>
using namespace std::string_literals;
namespace nix::fetchers {
struct MercurialInput : Input
{
ParsedURL url;
std::optional<std::string> ref;
std::optional<Hash> rev;
MercurialInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "hg"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const MercurialInput *>(&other);
return
other2
&& url == other2->url
&& rev == other2->rev
&& ref == other2->ref;
}
bool isImmutable() const override
{
return (bool) rev || narHash;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
ParsedURL url2(url);
url2.scheme = "hg+" + url2.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref);
return url;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
std::pair<bool, std::string> getActualUrl() const
{
bool isLocal = url.scheme == "file";
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto name = "source";
auto input = std::make_shared<MercurialInput>(*this);
auto [isLocal, actualUrl_] = getActualUrl();
auto actualUrl = actualUrl_; // work around clang bug
// FIXME: return lastModified.
// FIXME: don't clone local repositories.
if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
if (!clean) {
/* This is an unclean working tree. So copy all tracked
files. */
if (!settings.allowDirty)
throw Error("Mercurial tree '%s' is unclean", actualUrl);
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualUrl));
std::string file(p, actualUrl.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
}, input};
}
}
if (!input->ref) input->ref = "default";
auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", "hg"},
{"name", name},
{"rev", input->rev->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
{
assert(input->rev);
assert(!rev || rev == input->rev);
return {
Tree{
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = getIntAttr(infoAttrs, "revCount"),
},
},
input
};
};
if (input->rev) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
assert(input->rev || input->ref);
auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
Attrs mutableAttrs({
{"type", "hg"},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
});
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
return makeResult(res->first, std::move(res->second));
}
}
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
/* If this is a commit hash that we already have, we don't
have to pull again. */
if (!(input->rev
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
if (pathExists(cacheDir)) {
try {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
}
}
auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
input->rev = Hash(tokens[0], htSHA1);
auto revCount = std::stoull(tokens[1]);
input->ref = tokens[2];
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store->addToStore(name, tmpDir);
Attrs infoAttrs({
{"rev", input->rev->gitRev()},
{"revCount", (int64_t) revCount},
});
if (!this->rev)
getCache()->add(
store,
mutableAttrs,
infoAttrs,
storePath,
false);
getCache()->add(
store,
getImmutableAttrs(),
infoAttrs,
storePath,
true);
return makeResult(infoAttrs, std::move(storePath));
}
};
struct MercurialInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "hg+http" &&
url.scheme != "hg+https" &&
url.scheme != "hg+ssh" &&
url.scheme != "hg+file") return nullptr;
auto url2(url);
url2.scheme = std::string(url2.scheme, 3);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "hg");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev")
throw Error("unsupported Mercurial input attribute '%s'", name);
auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
}

148
src/libfetchers/path.cc Normal file
View file

@ -0,0 +1,148 @@
#include "fetchers.hh"
#include "store-api.hh"
namespace nix::fetchers {
struct PathInput : Input
{
Path path;
/* Allow the user to pass in "fake" tree info attributes. This is
useful for making a pinned tree work the same as the repository
from which is exported
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
std::optional<Hash> rev;
std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;
std::string type() const override { return "path"; }
std::optional<Hash> getRev() const override { return rev; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const PathInput *>(&other);
return
other2
&& path == other2->path
&& rev == other2->rev
&& revCount == other2->revCount
&& lastModified == other2->lastModified;
}
bool isImmutable() const override
{
return (bool) narHash;
}
ParsedURL toURL() const override
{
auto query = attrsToQuery(toAttrsInternal());
query.erase("path");
return ParsedURL {
.scheme = "path",
.path = path,
.query = query,
};
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("path", path);
if (rev)
attrs.emplace("rev", rev->gitRev());
if (revCount)
attrs.emplace("revCount", *revCount);
if (lastModified)
attrs.emplace("lastModified", *lastModified);
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto input = std::make_shared<PathInput>(*this);
// FIXME: check whether access to 'path' is allowed.
auto storePath = store->maybeParseStorePath(path);
if (storePath)
store->addTempRoot(*storePath);
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
// FIXME: try to substitute storePath.
storePath = store->addToStore("source", path);
return
{
Tree {
.actualPath = store->toRealPath(*storePath),
.storePath = std::move(*storePath),
.info = TreeInfo {
.revCount = revCount,
.lastModified = lastModified
}
},
input
};
}
};
struct PathInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "path") return nullptr;
auto input = std::make_unique<PathInput>();
input->path = url.path;
for (auto & [name, value] : url.query)
if (name == "rev")
input->rev = Hash(value, htSHA1);
else if (name == "revCount") {
uint64_t revCount;
if (!string2Int(value, revCount))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input->revCount = revCount;
}
else if (name == "lastModified") {
time_t lastModified;
if (!string2Int(value, lastModified))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input->lastModified = lastModified;
}
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "path") return {};
auto input = std::make_unique<PathInput>();
input->path = getStrAttr(attrs, "path");
for (auto & [name, value] : attrs)
if (name == "rev")
input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
else if (name == "revCount")
input->revCount = getIntAttr(attrs, "revCount");
else if (name == "lastModified")
input->lastModified = getIntAttr(attrs, "lastModified");
else if (name == "type" || name == "path")
;
else
throw Error("unsupported path input attribute '%s'", name);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
}

275
src/libfetchers/tarball.cc Normal file
View file

@ -0,0 +1,275 @@
#include "fetchers.hh"
#include "cache.hh"
#include "filetransfer.hh"
#include "globals.hh"
#include "store-api.hh"
#include "archive.hh"
#include "tarfile.hh"
namespace nix::fetchers {
DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable)
{
// FIXME: check store
Attrs inAttrs({
{"type", "file"},
{"url", url},
{"name", name},
});
auto cached = getCache()->lookupExpired(store, inAttrs);
auto useCached = [&]() -> DownloadFileResult
{
return {
.storePath = std::move(cached->storePath),
.etag = getStrAttr(cached->infoAttrs, "etag"),
.effectiveUrl = getStrAttr(cached->infoAttrs, "url")
};
};
if (cached && !cached->expired)
return useCached();
FileTransferRequest request(url);
if (cached)
request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
FileTransferResult res;
try {
res = getFileTransfer()->download(request);
} catch (FileTransferError & e) {
if (cached) {
warn("%s; using cached version", e.msg());
return useCached();
} else
throw;
}
// FIXME: write to temporary file.
Attrs infoAttrs({
{"etag", res.etag},
{"url", res.effectiveUri},
});
std::optional<StorePath> storePath;
if (res.cached) {
assert(cached);
assert(request.expectedETag == res.etag);
storePath = std::move(cached->storePath);
} else {
StringSink sink;
dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data);
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
auto source = StringSource { *sink.s };
store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path);
}
getCache()->add(
store,
inAttrs,
infoAttrs,
*storePath,
immutable);
if (url != res.effectiveUri)
getCache()->add(
store,
{
{"type", "file"},
{"url", res.effectiveUri},
{"name", name},
},
infoAttrs,
*storePath,
immutable);
return {
.storePath = std::move(*storePath),
.etag = res.etag,
.effectiveUrl = res.effectiveUri,
};
}
Tree downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable)
{
Attrs inAttrs({
{"type", "tarball"},
{"url", url},
{"name", name},
});
auto cached = getCache()->lookupExpired(store, inAttrs);
if (cached && !cached->expired)
return Tree {
.actualPath = store->toRealPath(cached->storePath),
.storePath = std::move(cached->storePath),
.info = TreeInfo {
.lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
},
};
auto res = downloadFile(store, url, name, immutable);
std::optional<StorePath> unpackedStorePath;
time_t lastModified;
if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) {
unpackedStorePath = std::move(cached->storePath);
lastModified = getIntAttr(cached->infoAttrs, "lastModified");
} else {
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
unpackTarfile(store->toRealPath(res.storePath), tmpDir);
auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
lastModified = lstat(topDir).st_mtime;
unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
}
Attrs infoAttrs({
{"lastModified", lastModified},
{"etag", res.etag},
});
getCache()->add(
store,
inAttrs,
infoAttrs,
*unpackedStorePath,
immutable);
return Tree {
.actualPath = store->toRealPath(*unpackedStorePath),
.storePath = std::move(*unpackedStorePath),
.info = TreeInfo {
.lastModified = lastModified,
},
};
}
struct TarballInput : Input
{
ParsedURL url;
std::optional<Hash> hash;
TarballInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "tarball"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const TarballInput *>(&other);
return
other2
&& to_string() == other2->to_string()
&& hash == other2->hash;
}
bool isImmutable() const override
{
return hash || narHash;
}
ParsedURL toURL() const override
{
auto url2(url);
// NAR hashes are preferred over file hashes since tar/zip files
// don't have a canonical representation.
if (narHash)
url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
else if (hash)
url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (hash)
attrs.emplace("hash", hash->to_string(SRI, true));
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto tree = downloadTarball(store, url.to_string(), "source", false);
auto input = std::make_shared<TarballInput>(*this);
input->narHash = store->queryPathInfo(tree.storePath)->narHash;
return {std::move(tree), input};
}
};
struct TarballInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
if (!hasSuffix(url.path, ".zip")
&& !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz")
&& !hasSuffix(url.path, ".tar.bz2"))
return nullptr;
auto input = std::make_unique<TarballInput>(url);
auto hash = input->url.query.find("hash");
if (hash != input->url.query.end()) {
// FIXME: require SRI hash.
input->hash = Hash(hash->second);
input->url.query.erase(hash);
}
auto narHash = input->url.query.find("narHash");
if (narHash != input->url.query.end()) {
// FIXME: require SRI hash.
input->narHash = Hash(narHash->second);
input->url.query.erase(narHash);
}
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "hash")
throw Error("unsupported tarball input attribute '%s'", name);
auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
if (auto hash = maybeGetStrAttr(attrs, "hash"))
input->hash = newHashAllowEmpty(*hash, htUnknown);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
}

14
src/libfetchers/tree-info.cc Normal file
View file

@ -0,0 +1,14 @@
#include "tree-info.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
StorePath TreeInfo::computeStorePath(Store & store) const
{
assert(narHash);
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
}
}

29
src/libfetchers/tree-info.hh Normal file
View file

@ -0,0 +1,29 @@
#pragma once
#include "path.hh"
#include "hash.hh"
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; }
namespace nix::fetchers {
struct TreeInfo
{
Hash narHash;
std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;
bool operator ==(const TreeInfo & other) const
{
return
narHash == other.narHash
&& revCount == other.revCount
&& lastModified == other.lastModified;
}
StorePath computeStorePath(Store & store) const;
};
}

69
src/libmain/common-args.cc
View file

@ -1,48 +1,61 @@
#include "common-args.hh"
#include "globals.hh"
#include "loggers.hh"
namespace nix {
MixCommonArgs::MixCommonArgs(const string & programName)
: programName(programName)
{
mkFlag()
.longName("verbose")
.shortName('v')
.description("increase verbosity level")
.handler([]() { verbosity = (Verbosity) (verbosity + 1); });
addFlag({
.longName = "verbose",
.shortName = 'v',
.description = "increase verbosity level",
.handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
});
mkFlag()
.longName("quiet")
.description("decrease verbosity level")
.handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; });
addFlag({
.longName = "quiet",
.description = "decrease verbosity level",
.handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
});
mkFlag()
.longName("debug")
.description("enable debug output")
.handler([]() { verbosity = lvlDebug; });
addFlag({
.longName = "debug",
.description = "enable debug output",
.handler = {[]() { verbosity = lvlDebug; }},
});
mkFlag()
.longName("option")
.labels({"name", "value"})
.description("set a Nix configuration option (overriding nix.conf)")
.arity(2)
.handler([](std::vector<std::string> ss) {
addFlag({
.longName = "option",
.description = "set a Nix configuration option (overriding nix.conf)",
.labels = {"name", "value"},
.handler = {[](std::string name, std::string value) {
try {
globalConfig.set(ss[0], ss[1]);
globalConfig.set(name, value);
} catch (UsageError & e) {
warn(e.what());
}
});
}},
});
mkFlag()
.longName("max-jobs")
.shortName('j')
.label("jobs")
.description("maximum number of parallel builds")
.handler([=](std::string s) {
addFlag({
.longName = "log-format",
.description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
"or \"bar-with-logs\"",
.labels = {"format"},
.handler = {[](std::string format) { setLogFormat(format); }},
});
addFlag({
.longName = "max-jobs",
.shortName = 'j',
.description = "maximum number of parallel builds",
.labels = Strings{"jobs"},
.handler = {[=](std::string s) {
settings.set("max-jobs", s);
});
}}
});
std::string cat = "config";
globalConfig.convertToArgs(*this, cat);

2
src/libmain/local.mk
View file

@ -6,6 +6,8 @@ libmain_DIR := $(d)
libmain_SOURCES := $(wildcard $(d)/*.cc)
libmain_CXXFLAGS += -I src/libutil -I src/libstore
libmain_LDFLAGS = $(OPENSSL_LIBS)
libmain_LIBS = libstore libutil

52
src/libmain/loggers.cc Normal file
View file

@ -0,0 +1,52 @@
#include "loggers.hh"
#include "progress-bar.hh"
namespace nix {
LogFormat defaultLogFormat = LogFormat::raw;
LogFormat parseLogFormat(const std::string & logFormatStr) {
if (logFormatStr == "raw")
return LogFormat::raw;
else if (logFormatStr == "raw-with-logs")
return LogFormat::rawWithLogs;
else if (logFormatStr == "internal-json")
return LogFormat::internalJson;
else if (logFormatStr == "bar")
return LogFormat::bar;
else if (logFormatStr == "bar-with-logs")
return LogFormat::barWithLogs;
throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
}
Logger * makeDefaultLogger() {
switch (defaultLogFormat) {
case LogFormat::raw:
return makeSimpleLogger(false);
case LogFormat::rawWithLogs:
return makeSimpleLogger(true);
case LogFormat::internalJson:
return makeJSONLogger(*makeSimpleLogger());
case LogFormat::bar:
return makeProgressBar();
case LogFormat::barWithLogs:
return makeProgressBar(true);
default:
abort();
}
}
void setLogFormat(const std::string & logFormatStr) {
setLogFormat(parseLogFormat(logFormatStr));
}
void setLogFormat(const LogFormat & logFormat) {
defaultLogFormat = logFormat;
createDefaultLogger();
}
void createDefaultLogger() {
logger = makeDefaultLogger();
}
}

20
src/libmain/loggers.hh Normal file
View file

@ -0,0 +1,20 @@
#pragma once
#include "types.hh"
namespace nix {
enum class LogFormat {
raw,
rawWithLogs,
internalJson,
bar,
barWithLogs,
};
void setLogFormat(const std::string & logFormatStr);
void setLogFormat(const LogFormat & logFormat);
void createDefaultLogger();
}

54
src/nix/progress-bar.cc → src/libmain/progress-bar.cc
View file

@ -7,6 +7,7 @@
#include <atomic>
#include <map>
#include <thread>
#include <iostream>
namespace nix {
@ -105,25 +106,36 @@ public:
updateThread.join();
}
void stop()
void stop() override
{
auto state(state_.lock());
if (!state->active) return;
state->active = false;
std::string status = getStatus(*state);
writeToStderr("\r\e[K");
if (status != "")
writeToStderr("[" + status + "]\n");
updateCV.notify_one();
quitCV.notify_one();
}
bool isVerbose() override {
return printBuildLogs;
}
void log(Verbosity lvl, const FormatOrString & fs) override
{
auto state(state_.lock());
log(*state, lvl, fs.s);
}
void logEI(const ErrorInfo &ei) override
{
auto state(state_.lock());
std::stringstream oss;
oss << ei;
log(*state, ei.level, oss.str());
}
void log(State & state, Verbosity lvl, const std::string & s)
{
if (state.active) {
@ -141,7 +153,7 @@ public:
{
auto state(state_.lock());
if (lvl <= verbosity && !s.empty())
if (lvl <= verbosity && !s.empty() && type != actBuildWaiting)
log(*state, lvl, s + "...");
state->activities.emplace_back(ActInfo());
@ -153,7 +165,7 @@ public:
state->activitiesByType[type].its.emplace(act, i);
if (type == actBuild) {
auto name = storePathToName(getS(fields, 0));
std::string name(storePathToName(getS(fields, 0)));
if (hasSuffix(name, ".drv"))
name = name.substr(0, name.size() - 4);
i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
@ -190,8 +202,8 @@ public:
i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
}
if ((type == actDownload && hasAncestor(*state, actCopyPath, parent))
|| (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent))
if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent))
|| (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent))
|| (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
i->visible = false;
@ -416,7 +428,7 @@ public:
if (!s2.empty()) { res += " ("; res += s2; res += ')'; }
}
showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB);
{
auto s = renderActivity(actOptimiseStore, "%s paths optimised");
@ -442,13 +454,31 @@ public:
return res;
}
void writeToStdout(std::string_view s) override
{
auto state(state_.lock());
if (state->active) {
std::cerr << "\r\e[K";
Logger::writeToStdout(s);
draw(*state);
} else {
Logger::writeToStdout(s);
}
}
};
Logger * makeProgressBar(bool printBuildLogs)
{
return new ProgressBar(
printBuildLogs,
isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
);
}
void startProgressBar(bool printBuildLogs)
{
logger = new ProgressBar(
printBuildLogs,
isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb");
logger = makeProgressBar(printBuildLogs);
}
void stopProgressBar()

2
src/nix/progress-bar.hh → src/libmain/progress-bar.hh
View file

@ -4,6 +4,8 @@
namespace nix {
Logger * makeProgressBar(bool printBuildLogs = false);
void startProgressBar(bool printBuildLogs = false);
void stopProgressBar();

80
src/libmain/shared.cc
View file

@ -2,6 +2,7 @@
#include "shared.hh"
#include "store-api.hh"
#include "util.hh"
#include "loggers.hh"
#include <algorithm>
#include <cctype>
@ -75,7 +76,7 @@ string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end)
{
++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
if (i == end) throw UsageError("'%1%' requires an argument", opt);
return *i;
}
@ -155,7 +156,7 @@ void initNix()
sshd). This breaks build users because they don't have access
to the TMPDIR, in particular in ‘nix-store --serve’. */
#if __APPLE__
if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
unsetenv("TMPDIR");
#endif
}
@ -165,28 +166,32 @@ LegacyArgs::LegacyArgs(const std::string & programName,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
: MixCommonArgs(programName), parseArg(parseArg)
{
mkFlag()
.longName("no-build-output")
.shortName('Q')
.description("do not show build output")
.set(&settings.verboseBuild, false);
addFlag({
.longName = "no-build-output",
.shortName = 'Q',
.description = "do not show build output",
.handler = {[&]() {setLogFormat(LogFormat::raw); }},
});
mkFlag()
.longName("keep-failed")
.shortName('K')
.description("keep temporary directories of failed builds")
.set(&(bool&) settings.keepFailed, true);
addFlag({
.longName = "keep-failed",
.shortName ='K',
.description = "keep temporary directories of failed builds",
.handler = {&(bool&) settings.keepFailed, true},
});
mkFlag()
.longName("keep-going")
.shortName('k')
.description("keep going after a build fails")
.set(&(bool&) settings.keepGoing, true);
addFlag({
.longName = "keep-going",
.shortName ='k',
.description = "keep going after a build fails",
.handler = {&(bool&) settings.keepGoing, true},
});
mkFlag()
.longName("fallback")
.description("build from source if substitution fails")
.set(&(bool&) settings.tryFallback, true);
addFlag({
.longName = "fallback",
.description = "build from source if substitution fails",
.handler = {&(bool&) settings.tryFallback, true},
});
auto intSettingAlias = [&](char shortName, const std::string & longName,
const std::string & description, const std::string & dest) {
@ -205,11 +210,12 @@ LegacyArgs::LegacyArgs(const std::string & programName,
mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
&gcWarning, false);
mkFlag()
.longName("store")
.label("store-uri")
.description("URI of the Nix store to use")
.dest(&(std::string&) settings.storeUri);
addFlag({
.longName = "store",
.description = "URI of the Nix store to use",
.labels = {"store-uri"},
.handler = {&(std::string&) settings.storeUri},
});
}
@ -229,7 +235,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
Strings ss(args);
auto pos = ss.begin();
if (!parseArg(pos, ss.end()))
throw UsageError(format("unexpected argument '%1%'") % args.front());
throw UsageError("unexpected argument '%1%'", args.front());
return true;
}
@ -260,7 +266,10 @@ void printVersion(const string & programName)
cfg.push_back("signed-caches");
#endif
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
std::cout << "User configuration files: " <<
concatStringsSep(":", settings.nixUserConfFiles)
<< "\n";
std::cout << "Store directory: " << settings.nixStore << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n";
}
@ -273,7 +282,7 @@ void showManPage(const string & name)
restoreSignals();
setenv("MANPATH", settings.nixManDir.c_str(), 1);
execlp("man", "man", name.c_str(), nullptr);
throw SysError(format("command 'man %1%' failed") % name.c_str());
throw SysError("command 'man %1%' failed", name.c_str());
}
@ -281,6 +290,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
{
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
ErrorInfo::programName = baseNameOf(programName);
string error = ANSI_RED "error:" ANSI_NORMAL " ";
try {
try {
@ -296,12 +307,13 @@ int handleExceptions(const string & programName, std::function<void()> fun)
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
printError(
format(error + "%1%\nTry '%2% --help' for more information.")
% e.what() % programName);
logError(e.info());
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
if (settings.showTrace && e.prefix() != "")
printError(e.prefix());
logError(e.info());
if (e.prefix() != "" && !settings.showTrace)
printError("(use '--show-trace' to show detailed location information)");
return e.status;
@ -338,7 +350,7 @@ RunPager::RunPager()
execlp("pager", "pager", nullptr);
execlp("less", "less", nullptr);
execlp("more", "more", nullptr);
throw SysError(format("executing '%1%'") % pager);
throw SysError("executing '%1%'", pager);
});
pid.setKillSignal(SIGINT);

6
src/libmain/shared.hh
View file

@ -56,7 +56,7 @@ template<class N> N getIntArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
{
++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
if (i == end) throw UsageError("'%1%' requires an argument", opt);
string s = *i;
N multiplier = 1;
if (allowUnit && !s.empty()) {
@ -66,13 +66,13 @@ template<class N> N getIntArg(const string & opt,
else if (u == 'M') multiplier = 1ULL << 20;
else if (u == 'G') multiplier = 1ULL << 30;
else if (u == 'T') multiplier = 1ULL << 40;
else throw UsageError(format("invalid unit specifier '%1%'") % u);
else throw UsageError("invalid unit specifier '%1%'", u);
s.resize(s.size() - 1);
}
}
N n;
if (!string2Int(s, n))
throw UsageError(format("'%1%' requires an integer argument") % opt);
throw UsageError("'%1%' requires an integer argument", opt);
return n * multiplier;
}

2
src/libmain/stack.cc
View file

@ -1,4 +1,4 @@
#include "types.hh"
#include "error.hh"
#include <cstring>
#include <cstddef>

39
src/libstore/binary-cache-store.cc
View file

@ -40,14 +40,14 @@ void BinaryCacheStore::init()
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
} else {
for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
size_t colon = line.find(':');
if (colon == std::string::npos) continue;
size_t colon= line.find(':');
if (colon ==std::string::npos) continue;
auto name = line.substr(0, colon);
auto value = trim(line.substr(colon + 1, std::string::npos));
if (name == "StoreDir") {
if (value != storeDir)
throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
% getUri() % value % storeDir);
throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
getUri(), value, storeDir);
} else if (name == "WantMassQuery") {
wantMassQuery.setDefault(value == "1" ? "true" : "false");
} else if (name == "Priority") {
@ -93,7 +93,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
{
return storePathToHash(printStorePath(storePath)) + ".narinfo";
return std::string(storePath.hashPart()) + ".narinfo";
}
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
@ -102,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
auto hashPart = storePathToHash(printStorePath(narInfo->path));
std::string hashPart(narInfo->path.hashPart());
{
auto state_(state.lock());
@ -113,9 +113,12 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{
// FIXME: See if we can use the original source to reduce memory usage.
auto nar = make_ref<std::string>(narSource.drain());
if (!repair && isValidPath(info.path)) return;
/* Verify that all references are valid. This may do some .narinfo
@ -161,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
}
}
upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
}
/* Compress the NAR. */
@ -284,7 +287,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
try {
getFile(info->url, *decompressor);
} catch (NoSuchBinaryCacheFile & e) {
throw SubstituteGone(e.what());
throw SubstituteGone(e.info());
}
decompressor->finish();
@ -327,7 +330,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
}
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
// FIXME: some cut&paste from LocalStore::addToStore().
@ -336,7 +339,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
small files. */
StringSink sink;
Hash h;
if (recursive) {
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
} else {
@ -345,9 +348,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s);
}
ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
ValidPathInfo info(makeFixedOutputPath(method, h, name));
addToStore(info, sink.s, repair, CheckSigs, nullptr);
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
return std::move(info.path);
}
@ -356,12 +360,13 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
const StorePathSet & references, RepairFlag repair)
{
ValidPathInfo info(computeStorePathForText(name, s, references));
info.references = cloneStorePathSet(references);
info.references = references;
if (repair || !isValidPath(info.path)) {
StringSink sink;
dumpString(s, sink);
addToStore(info, sink.s, repair, CheckSigs, nullptr);
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
}
return std::move(info.path);
@ -390,14 +395,14 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
{
auto drvPath = path.clone();
auto drvPath = path;
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (!info->deriver) return nullptr;
drvPath = info->deriver->clone();
drvPath = *info->deriver;
} catch (InvalidPath &) {
return nullptr;
}

4
src/libstore/binary-cache-store.hh
View file

@ -74,12 +74,12 @@ public:
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s,

648
src/libstore/build.cc
View file

File diff suppressed because it is too large Load diff

1
src/libstore/builtins.hh
View file

@ -6,7 +6,6 @@ namespace nix {
// TODO: make pluggable.
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
void builtinBuildenv(const BasicDerivation & drv);
void builtinUnpackChannel(const BasicDerivation & drv);
}

148
src/libstore/builtins/buildenv.cc
View file

@ -1,4 +1,4 @@
#include "builtins.hh"
#include "buildenv.hh"
#include <sys/stat.h>
#include <sys/types.h>
@ -7,16 +7,14 @@
namespace nix {
typedef std::map<Path,int> Priorities;
// FIXME: change into local variables.
static Priorities priorities;
static unsigned long symlinks;
struct State
{
std::map<Path, int> priorities;
unsigned long symlinks = 0;
};
/* For each activated package, create symlinks */
static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
{
DirEntries srcFiles;
@ -24,7 +22,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
srcFiles = readDirectory(srcDir);
} catch (SysError & e) {
if (e.errNo == ENOTDIR) {
printError("warning: not including '%s' in the user environment because it's not a directory", srcDir);
logWarning({
.name = "Create links - directory",
.hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
});
return;
}
throw;
@ -43,7 +44,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
throw SysError("getting status of '%1%'", srcFile);
} catch (SysError & e) {
if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
printError("warning: skipping dangling symlink '%s'", dstFile);
logWarning({
.name = "Create links - skipping symlink",
.hint = hintfmt("skipping dangling symlink '%s'", dstFile)
});
continue;
}
throw;
@ -67,22 +71,22 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
auto res = lstat(dstFile.c_str(), &dstSt);
if (res == 0) {
if (S_ISDIR(dstSt.st_mode)) {
createLinks(srcFile, dstFile, priority);
createLinks(state, srcFile, dstFile, priority);
continue;
} else if (S_ISLNK(dstSt.st_mode)) {
auto target = canonPath(dstFile, true);
if (!S_ISDIR(lstat(target).st_mode))
throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile);
throw SysError("unlinking '%1%'", dstFile);
if (mkdir(dstFile.c_str(), 0755) == -1)
throw SysError(format("creating directory '%1%'"));
createLinks(target, dstFile, priorities[dstFile]);
createLinks(srcFile, dstFile, priority);
throw SysError("creating directory '%1%'", dstFile);
createLinks(state, target, dstFile, state.priorities[dstFile]);
createLinks(state, srcFile, dstFile, priority);
continue;
}
} else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile);
throw SysError("getting status of '%1%'", dstFile);
}
else {
@ -90,7 +94,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
auto res = lstat(dstFile.c_str(), &dstSt);
if (res == 0) {
if (S_ISLNK(dstSt.st_mode)) {
auto prevPriority = priorities[dstFile];
auto prevPriority = state.priorities[dstFile];
if (prevPriority == priority)
throw Error(
"packages '%1%' and '%2%' have the same priority %3%; "
@ -101,75 +105,38 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
if (prevPriority < priority)
continue;
if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile);
throw SysError("unlinking '%1%'", dstFile);
} else if (S_ISDIR(dstSt.st_mode))
throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
} else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile);
throw SysError("getting status of '%1%'", dstFile);
}
createSymlink(srcFile, dstFile);
priorities[dstFile] = priority;
symlinks++;
state.priorities[dstFile] = priority;
state.symlinks++;
}
}
typedef std::set<Path> FileProp;
static FileProp done;
static FileProp postponed = FileProp{};
static Path out;
static void addPkg(const Path & pkgDir, int priority)
void buildProfile(const Path & out, Packages && pkgs)
{
if (!done.insert(pkgDir).second) return;
createLinks(pkgDir, out, priority);
State state;
try {
for (const auto & p : tokenizeString<std::vector<string>>(
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
if (!done.count(p))
postponed.insert(p);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
}
}
std::set<Path> done, postponed;
struct Package {
Path path;
bool active;
int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
};
auto addPkg = [&](const Path & pkgDir, int priority) {
if (!done.insert(pkgDir).second) return;
createLinks(state, pkgDir, out, priority);
typedef std::vector<Package> Packages;
void builtinBuildenv(const BasicDerivation & drv)
{
auto getAttr = [&](const string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;
};
out = getAttr("out");
createDirs(out);
/* Convert the stuff we get from the environment back into a
* coherent data type. */
Packages pkgs;
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) {
/* !!! We're trusting the caller to structure derivations env var correctly */
auto active = derivations.front(); derivations.pop_front();
auto priority = stoi(derivations.front()); derivations.pop_front();
auto outputs = stoi(derivations.front()); derivations.pop_front();
for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front();
pkgs.emplace_back(path, active != "false", priority);
try {
for (const auto & p : tokenizeString<std::vector<string>>(
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
if (!done.count(p))
postponed.insert(p);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
}
}
};
/* Symlink to the packages that have been installed explicitly by the
* user. Process in priority order to reduce unnecessary
@ -189,13 +156,42 @@ void builtinBuildenv(const BasicDerivation & drv)
*/
auto priorityCounter = 1000;
while (!postponed.empty()) {
auto pkgDirs = postponed;
postponed = FileProp{};
std::set<Path> pkgDirs;
postponed.swap(pkgDirs);
for (const auto & pkgDir : pkgDirs)
addPkg(pkgDir, priorityCounter++);
}
printError("created %d symlinks in user environment", symlinks);
debug("created %d symlinks in user environment", state.symlinks);
}
void builtinBuildenv(const BasicDerivation & drv)
{
auto getAttr = [&](const string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;
};
Path out = getAttr("out");
createDirs(out);
/* Convert the stuff we get from the environment back into a
* coherent data type. */
Packages pkgs;
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) {
/* !!! We're trusting the caller to structure derivations env var correctly */
auto active = derivations.front(); derivations.pop_front();
auto priority = stoi(derivations.front()); derivations.pop_front();
auto outputs = stoi(derivations.front()); derivations.pop_front();
for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front();
pkgs.emplace_back(path, active != "false", priority);
}
}
buildProfile(out, std::move(pkgs));
createSymlink(getAttr("manifest"), out + "/manifest.nix");
}

21
src/libstore/builtins/buildenv.hh Normal file
View file

@ -0,0 +1,21 @@
#pragma once
#include "derivations.hh"
#include "store-api.hh"
namespace nix {
struct Package {
Path path;
bool active;
int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
};
typedef std::vector<Package> Packages;
void buildProfile(const Path & out, Packages && pkgs);
void builtinBuildenv(const BasicDerivation & drv);
}

14
src/libstore/builtins/fetchurl.cc
View file

@ -1,5 +1,5 @@
#include "builtins.hh"
#include "download.hh"
#include "filetransfer.hh"
#include "store-api.hh"
#include "archive.hh"
#include "compression.hh"
@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto getAttr = [&](const string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;
};
@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto mainUrl = getAttr("url");
bool unpack = get(drv.env, "unpack").value_or("") == "1";
/* Note: have to use a fresh downloader here because we're in
/* Note: have to use a fresh fileTransfer here because we're in
a forked process. */
auto downloader = makeDownloader();
auto fileTransfer = makeFileTransfer();
auto fetch = [&](const std::string & url) {
@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
/* No need to do TLS verification, because we check the hash of
the result anyway. */
DownloadRequest request(url);
FileTransferRequest request(url);
request.verifyTLS = false;
request.decompress = false;
auto decompressor = makeDecompressionSink(
unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
downloader->download(std::move(request), *decompressor);
fileTransfer->download(std::move(request), *decompressor);
decompressor->finish();
});
@ -54,7 +54,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto executable = drv.env.find("executable");
if (executable != drv.env.end() && executable->second == "1") {
if (chmod(storePath.c_str(), 0755) == -1)
throw SysError(format("making '%1%' executable") % storePath);
throw SysError("making '%1%' executable", storePath);
}
};

51
src/libstore/daemon.cc
View file

@ -73,6 +73,18 @@ struct TunnelLogger : public Logger
enqueueMsg(*buf.s);
}
void logEI(const ErrorInfo & ei) override
{
if (ei.level > verbosity) return;
std::stringstream oss;
oss << ei;
StringSink buf;
buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
enqueueMsg(*buf.s);
}
/* startWork() means that we're starting an operation for which we
want to send out stderr to the client. */
void startWork()
@ -281,7 +293,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
StorePathSet paths; // FIXME
paths.insert(path.clone());
paths.insert(path);
auto res = store->querySubstitutablePaths(paths);
logger->stopWork();
to << (res.count(path) != 0);
@ -315,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
StorePathSet paths;
if (op == wopQueryReferences)
for (auto & i : store->queryPathInfo(path)->references)
paths.insert(i.clone());
paths.insert(i);
else if (op == wopQueryReferrers)
store->queryReferrers(path, paths);
else if (op == wopQueryValidDerivers)
@ -329,8 +341,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryDerivationOutputNames: {
auto path = store->parseStorePath(readString(from));
logger->startWork();
StringSet names;
names = store->queryDerivationOutputNames(path);
auto names = store->readDerivation(path).outputNames();
logger->stopWork();
to << names;
break;
@ -355,20 +366,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopAddToStore: {
bool fixed, recursive;
std::string s, baseName;
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
/* Compatibility hack. */
if (!fixed) {
s = "sha256";
recursive = true;
FileIngestionMethod method;
{
bool fixed; uint8_t recursive;
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
s = "sha256";
method = FileIngestionMethod::Recursive;
}
}
HashType hashAlgo = parseHashType(s);
TeeSource savedNAR(from);
RetrieveRegularNARSink savedRegular;
if (recursive) {
if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to
a string so that we can pass it to
addToStoreFromDump(). */
@ -380,7 +397,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
baseName,
method,
hashAlgo);
logger->stopWork();
to << store->printStorePath(path);
@ -572,9 +593,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
SubstitutablePathInfos infos;
StorePathSet paths;
paths.insert(path.clone()); // FIXME
store->querySubstitutablePathInfos(paths, infos);
store->querySubstitutablePathInfos({path}, infos);
logger->stopWork();
auto i = infos.find(path);
if (i == infos.end())
@ -735,7 +754,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
default:
throw Error(format("invalid operation %1%") % op);
throw Error("invalid operation %1%", op);
}
}

79
src/libstore/derivations.cc
View file

@ -9,13 +9,13 @@
namespace nix {
void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
{
recursive = false;
recursive = FileIngestionMethod::Flat;
string algo = hashAlgo;
if (string(algo, 0, 2) == "r:") {
recursive = true;
recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2);
}
@ -27,28 +27,6 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
}
BasicDerivation::BasicDerivation(const BasicDerivation & other)
: platform(other.platform)
, builder(other.builder)
, args(other.args)
, env(other.env)
{
for (auto & i : other.outputs)
outputs.insert_or_assign(i.first,
DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash)));
for (auto & i : other.inputSrcs)
inputSrcs.insert(i.clone());
}
Derivation::Derivation(const Derivation & other)
: BasicDerivation(other)
{
for (auto & i : other.inputDrvs)
inputDrvs.insert_or_assign(i.first.clone(), i.second);
}
const StorePath & BasicDerivation::findOutput(const string & id) const
{
auto i = outputs.find(id);
@ -65,16 +43,16 @@ bool BasicDerivation::isBuiltin() const
StorePath writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair)
const Derivation & drv, std::string_view name, RepairFlag repair)
{
auto references = cloneStorePathSet(drv.inputSrcs);
auto references = drv.inputSrcs;
for (auto & i : drv.inputDrvs)
references.insert(i.first.clone());
references.insert(i.first);
/* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
string suffix = name + drvExtension;
string contents = drv.unparse(*store, false);
auto suffix = std::string(name) + drvExtension;
auto contents = drv.unparse(*store, false);
return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references)
: store->addTextToStore(suffix, contents, references, repair);
@ -87,7 +65,7 @@ static void expect(std::istream & str, const string & s)
char s2[s.size()];
str.read(s2, s.size());
if (string(s2, s.size()) != s)
throw FormatError(format("expected string '%1%'") % s);
throw FormatError("expected string '%1%'", s);
}
@ -114,7 +92,7 @@ static Path parsePath(std::istream & str)
{
string s = parseString(str);
if (s.size() == 0 || s[0] != '/')
throw FormatError(format("bad path '%1%' in derivation") % s);
throw FormatError("bad path '%1%' in derivation", s);
return s;
}
@ -155,7 +133,11 @@ static Derivation parseDerivation(const Store & store, const string & s)
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); auto hash = parseString(str);
expect(str, ")");
drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
drv.outputs.emplace(id, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
}
/* Parse the list of input derivations. */
@ -196,7 +178,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
try {
return parseDerivation(store, readFile(drvPath));
} catch (FormatError & e) {
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
}
}
@ -204,6 +186,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
Derivation Store::derivationFromPath(const StorePath & drvPath)
{
ensurePath(drvPath);
return readDerivation(drvPath);
}
Derivation Store::readDerivation(const StorePath & drvPath)
{
auto accessor = getFSAccessor();
try {
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
@ -351,12 +339,10 @@ static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drv
assert(store.isValidPath(drvPath));
// Cache it
h = drvHashes.insert_or_assign(
drvPath.clone(),
drvPath,
hashDerivationModulo(
store,
readDerivation(
store,
store.toRealPath(store.printStorePath(drvPath))),
store.readDerivation(drvPath),
false)).first;
}
return h->second;
@ -437,11 +423,20 @@ StorePathSet BasicDerivation::outputPaths() const
{
StorePathSet paths;
for (auto & i : outputs)
paths.insert(i.second.path.clone());
paths.insert(i.second.path);
return paths;
}
StringSet BasicDerivation::outputNames() const
{
StringSet names;
for (auto & i : outputs)
names.insert(i.first);
return names;
}
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
{
drv.outputs.clear();
@ -451,7 +446,11 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
auto hash = readString(in);
drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
drv.outputs.emplace(name, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
}
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);

14
src/libstore/derivations.hh
View file

@ -18,12 +18,7 @@ struct DerivationOutput
StorePath path;
std::string hashAlgo; /* hash used for expected hash computation */
std::string hash; /* expected hash, may be null */
DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash)
: path(std::move(path))
, hashAlgo(std::move(hashAlgo))
, hash(std::move(hash))
{ }
void parseHashInfo(bool & recursive, Hash & hash) const;
void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -44,7 +39,6 @@ struct BasicDerivation
StringPairs env;
BasicDerivation() { }
explicit BasicDerivation(const BasicDerivation & other);
virtual ~BasicDerivation() { };
/* Return the path corresponding to the output identifier `id' in
@ -59,6 +53,8 @@ struct BasicDerivation
/* Return the output paths of a derivation. */
StorePathSet outputPaths() const;
/* Return the output names of a derivation. */
StringSet outputNames() const;
};
struct Derivation : BasicDerivation
@ -70,8 +66,6 @@ struct Derivation : BasicDerivation
std::map<std::string, StringSet> * actualInputs = nullptr) const;
Derivation() { }
Derivation(Derivation && other) = default;
explicit Derivation(const Derivation & other);
};
@ -80,7 +74,7 @@ class Store;
/* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */
Derivation readDerivation(const Store & store, const Path & drvPath);

9
src/libstore/export-import.cc
View file

@ -1,3 +1,4 @@
#include "serialise.hh"
#include "store-api.hh"
#include "archive.hh"
#include "worker-protocol.hh"
@ -56,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(), hash.to_string());
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
hashAndWriteSink
<< exportMagic
@ -100,9 +101,11 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (readInt(source) == 1)
readString(source);
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
// Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.source.data };
addToStore(info, source, NoRepair, checkSigs, accessor);
res.push_back(info.path.clone());
res.push_back(info.path);
}
return res;

307
src/libstore/download.cc → src/libstore/filetransfer.cc
View file

@ -1,14 +1,10 @@
#include "download.hh"
#include "filetransfer.hh"
#include "util.hh"
#include "globals.hh"
#include "hash.hh"
#include "store-api.hh"
#include "archive.hh"
#include "s3.hh"
#include "compression.hh"
#include "pathlocks.hh"
#include "finally.hh"
#include "tarfile.hh"
#ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>
@ -31,13 +27,9 @@ using namespace std::string_literals;
namespace nix {
DownloadSettings downloadSettings;
FileTransferSettings fileTransferSettings;
static GlobalConfig::Register r1(&downloadSettings);
CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
: uri(uri), ttl(settings.tarballTtl)
{ }
static GlobalConfig::Register r1(&fileTransferSettings);
std::string resolveUri(const std::string & uri)
{
@ -47,21 +39,21 @@ std::string resolveUri(const std::string & uri)
return uri;
}
struct CurlDownloader : public Downloader
struct curlFileTransfer : public FileTransfer
{
CURLM * curlm = 0;
std::random_device rd;
std::mt19937 mt19937;
struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
struct TransferItem : public std::enable_shared_from_this<TransferItem>
{
CurlDownloader & downloader;
DownloadRequest request;
DownloadResult result;
curlFileTransfer & fileTransfer;
FileTransferRequest request;
FileTransferResult result;
Activity act;
bool done = false; // whether either the success or failure function has been called
Callback<DownloadResult> callback;
Callback<FileTransferResult> callback;
CURL * req = 0;
bool active = false; // whether the handle has been added to the multi object
std::string status;
@ -80,19 +72,36 @@ struct CurlDownloader : public Downloader
curl_off_t writtenToSink = 0;
DownloadItem(CurlDownloader & downloader,
const DownloadRequest & request,
Callback<DownloadResult> && callback)
: downloader(downloader)
/* Get the HTTP status code, or 0 for other protocols. */
long getHTTPStatus()
{
long httpStatus = 0;
long protocol = 0;
curl_easy_getinfo(req, CURLINFO_PROTOCOL, &protocol);
if (protocol == CURLPROTO_HTTP || protocol == CURLPROTO_HTTPS)
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
return httpStatus;
}
TransferItem(curlFileTransfer & fileTransfer,
const FileTransferRequest & request,
Callback<FileTransferResult> && callback)
: fileTransfer(fileTransfer)
, request(request)
, act(*logger, lvlTalkative, actDownload,
, act(*logger, lvlTalkative, actFileTransfer,
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
{request.uri}, request.parentAct)
, callback(std::move(callback))
, finalSink([this](const unsigned char * data, size_t len) {
if (this->request.dataCallback) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
auto httpStatus = getHTTPStatus();
/* Only write data to the sink if this is a
successful response. */
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
}
} else
this->result.data->append((char *) data, len);
})
@ -103,17 +112,17 @@ struct CurlDownloader : public Downloader
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
}
~DownloadItem()
~TransferItem()
{
if (req) {
if (active)
curl_multi_remove_handle(downloader.curlm, req);
curl_multi_remove_handle(fileTransfer.curlm, req);
curl_easy_cleanup(req);
}
if (requestHeaders) curl_slist_free_all(requestHeaders);
try {
if (!done)
fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
} catch (...) {
ignoreException();
}
@ -157,7 +166,7 @@ struct CurlDownloader : public Downloader
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
{
return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb);
return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
}
size_t headerCallback(void * contents, size_t size, size_t nmemb)
@ -199,7 +208,7 @@ struct CurlDownloader : public Downloader
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
{
return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb);
return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
}
int progressCallback(double dltotal, double dlnow)
@ -214,7 +223,7 @@ struct CurlDownloader : public Downloader
static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
{
return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow);
return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
}
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
@ -238,7 +247,7 @@ struct CurlDownloader : public Downloader
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
{
return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
}
void init()
@ -249,7 +258,7 @@ struct CurlDownloader : public Downloader
if (verbosity >= lvlVomit) {
curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback);
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback);
}
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
@ -258,19 +267,19 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(req, CURLOPT_USERAGENT,
("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
(downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
(fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str());
#if LIBCURL_VERSION_NUM >= 0x072b00
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
#endif
#if LIBCURL_VERSION_NUM >= 0x072f00
if (downloadSettings.enableHttp2)
if (fileTransferSettings.enableHttp2)
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
else
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
#endif
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper);
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
@ -298,10 +307,10 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
}
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
/* If no file exist in the specified path, curl continues to work
anyway as if netrc support was disabled. */
@ -317,8 +326,7 @@ struct CurlDownloader : public Downloader
void finish(CURLcode code)
{
long httpStatus = 0;
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
auto httpStatus = getHTTPStatus();
char * effectiveUriCStr;
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
@ -345,7 +353,7 @@ struct CurlDownloader : public Downloader
failEx(writeException);
else if (code == CURLE_OK &&
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */))
{
result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize);
@ -390,6 +398,7 @@ struct CurlDownloader : public Downloader
case CURLE_SSL_CACERT_BADFILE:
case CURLE_TOO_MANY_REDIRECTS:
case CURLE_WRITE_ERROR:
case CURLE_UNSUPPORTED_PROTOCOL:
err = Misc;
break;
default: // Shut up warnings
@ -401,14 +410,14 @@ struct CurlDownloader : public Downloader
auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
: httpStatus != 0
? DownloadError(err,
? FileTransferError(err,
fmt("unable to %s '%s': HTTP error %d",
request.verb(), request.uri, httpStatus)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
)
: DownloadError(err,
: FileTransferError(err,
fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code));
@ -422,13 +431,13 @@ struct CurlDownloader : public Downloader
|| writtenToSink == 0
|| (acceptRanges && encoding.empty())))
{
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
if (writtenToSink)
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
else
warn("%s; retrying in %d ms", exc.what(), ms);
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
downloader.enqueueItem(shared_from_this());
fileTransfer.enqueueItem(shared_from_this());
}
else
fail(exc);
@ -439,12 +448,12 @@ struct CurlDownloader : public Downloader
struct State
{
struct EmbargoComparator {
bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) {
return i1->embargo > i2->embargo;
}
};
bool quit = false;
std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming;
};
Sync<State> state_;
@ -456,7 +465,7 @@ struct CurlDownloader : public Downloader
std::thread workerThread;
CurlDownloader()
curlFileTransfer()
: mt19937(rd())
{
static std::once_flag globalInit;
@ -469,7 +478,7 @@ struct CurlDownloader : public Downloader
#endif
#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
downloadSettings.httpConnections.get());
fileTransferSettings.httpConnections.get());
#endif
wakeupPipe.create();
@ -478,7 +487,7 @@ struct CurlDownloader : public Downloader
workerThread = std::thread([&]() { workerThreadEntry(); });
}
~CurlDownloader()
~curlFileTransfer()
{
stopWorkerThread();
@ -504,7 +513,7 @@ struct CurlDownloader : public Downloader
stopWorkerThread();
});
std::map<CURL *, std::shared_ptr<DownloadItem>> items;
std::map<CURL *, std::shared_ptr<TransferItem>> items;
bool quit = false;
@ -517,7 +526,7 @@ struct CurlDownloader : public Downloader
int running;
CURLMcode mc = curl_multi_perform(curlm, &running);
if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc));
throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
/* Set the promises of any finished requests. */
CURLMsg * msg;
@ -547,7 +556,7 @@ struct CurlDownloader : public Downloader
vomit("download thread waiting for %d ms", sleepTimeMs);
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc));
throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
nextWakeup = std::chrono::steady_clock::time_point();
@ -561,7 +570,7 @@ struct CurlDownloader : public Downloader
throw SysError("reading curl wakeup socket");
}
std::vector<std::shared_ptr<DownloadItem>> incoming;
std::vector<std::shared_ptr<TransferItem>> incoming;
auto now = std::chrono::steady_clock::now();
{
@ -599,7 +608,11 @@ struct CurlDownloader : public Downloader
workerThreadMain();
} catch (nix::Interrupted & e) {
} catch (std::exception & e) {
printError("unexpected error in download thread: %s", e.what());
logError({
.name = "File transfer",
.hint = hintfmt("unexpected error in download thread: %s",
e.what())
});
}
{
@ -609,7 +622,7 @@ struct CurlDownloader : public Downloader
}
}
void enqueueItem(std::shared_ptr<DownloadItem> item)
void enqueueItem(std::shared_ptr<TransferItem> item)
{
if (item->request.data
&& !hasPrefix(item->request.uri, "http://")
@ -641,8 +654,8 @@ struct CurlDownloader : public Downloader
}
#endif
void enqueueDownload(const DownloadRequest & request,
Callback<DownloadResult> callback) override
void enqueueFileTransfer(const FileTransferRequest & request,
Callback<FileTransferResult> callback) override
{
/* Ugly hack to support s3:// URIs. */
if (hasPrefix(request.uri, "s3://")) {
@ -660,9 +673,9 @@ struct CurlDownloader : public Downloader
// FIXME: implement ETag
auto s3Res = s3Helper.getObject(bucketName, key);
DownloadResult res;
FileTransferResult res;
if (!s3Res.data)
throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
res.data = s3Res.data;
callback(std::move(res));
#else
@ -672,26 +685,26 @@ struct CurlDownloader : public Downloader
return;
}
enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
}
};
ref<Downloader> getDownloader()
ref<FileTransfer> getFileTransfer()
{
static ref<Downloader> downloader = makeDownloader();
return downloader;
static ref<FileTransfer> fileTransfer = makeFileTransfer();
return fileTransfer;
}
ref<Downloader> makeDownloader()
ref<FileTransfer> makeFileTransfer()
{
return make_ref<CurlDownloader>();
return make_ref<curlFileTransfer>();
}
std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request)
std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
{
auto promise = std::make_shared<std::promise<DownloadResult>>();
enqueueDownload(request,
{[promise](std::future<DownloadResult> fut) {
auto promise = std::make_shared<std::promise<FileTransferResult>>();
enqueueFileTransfer(request,
{[promise](std::future<FileTransferResult> fut) {
try {
promise->set_value(fut.get());
} catch (...) {
@ -701,15 +714,21 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
return promise->get_future();
}
DownloadResult Downloader::download(const DownloadRequest & request)
FileTransferResult FileTransfer::download(const FileTransferRequest & request)
{
return enqueueDownload(request).get();
return enqueueFileTransfer(request).get();
}
void Downloader::download(DownloadRequest && request, Sink & sink)
FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
{
/* Note: this method is the same as download, but helps in readability */
return enqueueFileTransfer(request).get();
}
void FileTransfer::download(FileTransferRequest && request, Sink & sink)
{
/* Note: we can't call 'sink' via request.dataCallback, because
that would cause the sink to execute on the downloader
that would cause the sink to execute on the fileTransfer
thread. If 'sink' is a coroutine, this will fail. Also, if the
sink is expensive (e.g. one that does decompression and writing
to the Nix store), it would stall the download thread too much.
@ -755,8 +774,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
state->avail.notify_one();
};
enqueueDownload(request,
{[_state](std::future<DownloadResult> fut) {
enqueueFileTransfer(request,
{[_state](std::future<FileTransferResult> fut) {
auto state(_state->lock());
state->quit = true;
try {
@ -801,140 +820,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
}
}
CachedDownloadResult Downloader::downloadCached(
ref<Store> store, const CachedDownloadRequest & request)
{
auto url = resolveUri(request.uri);
auto name = request.name;
if (name == "") {
auto p = url.rfind('/');
if (p != string::npos) name = string(url, p + 1);
}
std::optional<StorePath> expectedStorePath;
if (request.expectedHash) {
expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
if (store->isValidPath(*expectedStorePath)) {
CachedDownloadResult result;
result.storePath = store->printStorePath(*expectedStorePath);
result.path = store->toRealPath(result.storePath);
return result;
}
}
Path cacheDir = getCacheDir() + "/nix/tarballs";
createDirs(cacheDir);
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
Path dataFile = cacheDir + "/" + urlHash + ".info";
Path fileLink = cacheDir + "/" + urlHash + "-file";
PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
std::optional<StorePath> storePath;
string expectedETag;
bool skip = false;
CachedDownloadResult result;
if (pathExists(fileLink) && pathExists(dataFile)) {
storePath = store->parseStorePath(readLink(fileLink));
// FIXME
store->addTempRoot(*storePath);
if (store->isValidPath(*storePath)) {
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
if (ss.size() >= 3 && ss[0] == url) {
time_t lastChecked;
if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
skip = true;
result.effectiveUri = request.uri;
result.etag = ss[1];
} else if (!ss[1].empty()) {
debug(format("verifying previous ETag '%1%'") % ss[1]);
expectedETag = ss[1];
}
}
} else
storePath.reset();
}
if (!skip) {
try {
DownloadRequest request2(url);
request2.expectedETag = expectedETag;
auto res = download(request2);
result.effectiveUri = res.effectiveUri;
result.etag = res.etag;
if (!res.cached) {
StringSink sink;
dumpString(*res.data, sink);
Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(false, hash);
store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
storePath = info.path.clone();
}
assert(storePath);
replaceSymlink(store->printStorePath(*storePath), fileLink);
writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
} catch (DownloadError & e) {
if (!storePath) throw;
warn("warning: %s; using cached result", e.msg());
result.etag = expectedETag;
}
}
if (request.unpack) {
Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
std::optional<StorePath> unpackedStorePath;
if (pathExists(unpackedLink)) {
unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
// FIXME
store->addTempRoot(*unpackedStorePath);
if (!store->isValidPath(*unpackedStorePath))
unpackedStorePath.reset();
}
if (!unpackedStorePath) {
printInfo("unpacking '%s'...", url);
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir);
auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
}
replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
storePath = std::move(*unpackedStorePath);
}
if (expectedStorePath && *storePath != *expectedStorePath) {
unsigned int statusCode = 102;
Hash gotHash = request.unpack
? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
: hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
url, request.expectedHash.to_string(), gotHash.to_string());
}
result.storePath = store->printStorePath(*storePath);
result.path = store->toRealPath(result.storePath);
return result;
}
bool isUri(const string & s)
{
if (s.compare(0, 8, "channel:") == 0) return true;

81
src/libstore/download.hh → src/libstore/filetransfer.hh
View file

@ -9,7 +9,7 @@
namespace nix {
struct DownloadSettings : Config
struct FileTransferSettings : Config
{
Setting<bool> enableHttp2{this, true, "http2",
"Whether to enable HTTP/2 support."};
@ -31,15 +31,15 @@ struct DownloadSettings : Config
"How often Nix will attempt to download a file before giving up."};
};
extern DownloadSettings downloadSettings;
extern FileTransferSettings fileTransferSettings;
struct DownloadRequest
struct FileTransferRequest
{
std::string uri;
std::string expectedETag;
bool verifyTLS = true;
bool head = false;
size_t tries = downloadSettings.tries;
size_t tries = fileTransferSettings.tries;
unsigned int baseRetryTimeMs = 250;
ActivityId parentAct;
bool decompress = true;
@ -47,7 +47,7 @@ struct DownloadRequest
std::string mimeType;
std::function<void(char *, size_t)> dataCallback;
DownloadRequest(const std::string & uri)
FileTransferRequest(const std::string & uri)
: uri(uri), parentAct(getCurActivity()) { }
std::string verb()
@ -56,7 +56,7 @@ struct DownloadRequest
}
};
struct DownloadResult
struct FileTransferResult
{
bool cached = false;
std::string etag;
@ -65,74 +65,53 @@ struct DownloadResult
uint64_t bodySize = 0;
};
struct CachedDownloadRequest
{
std::string uri;
bool unpack = false;
std::string name;
Hash expectedHash;
unsigned int ttl;
CachedDownloadRequest(const std::string & uri);
CachedDownloadRequest() = delete;
};
struct CachedDownloadResult
{
// Note: 'storePath' may be different from 'path' when using a
// chroot store.
Path storePath;
Path path;
std::optional<std::string> etag;
std::string effectiveUri;
};
class Store;
struct Downloader
struct FileTransfer
{
virtual ~Downloader() { }
virtual ~FileTransfer() { }
/* Enqueue a download request, returning a future to the result of
the download. The future may throw a DownloadError
/* Enqueue a data transfer request, returning a future to the result of
the download. The future may throw a FileTransferError
exception. */
virtual void enqueueDownload(const DownloadRequest & request,
Callback<DownloadResult> callback) = 0;
virtual void enqueueFileTransfer(const FileTransferRequest & request,
Callback<FileTransferResult> callback) = 0;
std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
/* Synchronously download a file. */
DownloadResult download(const DownloadRequest & request);
FileTransferResult download(const FileTransferRequest & request);
/* Synchronously upload a file. */
FileTransferResult upload(const FileTransferRequest & request);
/* Download a file, writing its data to a sink. The sink will be
invoked on the thread of the caller. */
void download(DownloadRequest && request, Sink & sink);
/* Check if the specified file is already in ~/.cache/nix/tarballs
and is more recent than ‘tarball-ttl’ seconds. Otherwise,
use the recorded ETag to verify if the server has a more
recent version, and if so, download it to the Nix store. */
CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
void download(FileTransferRequest && request, Sink & sink);
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
};
/* Return a shared Downloader object. Using this object is preferred
/* Return a shared FileTransfer object. Using this object is preferred
because it enables connection reuse and HTTP/2 multiplexing. */
ref<Downloader> getDownloader();
ref<FileTransfer> getFileTransfer();
/* Return a new Downloader object. */
ref<Downloader> makeDownloader();
/* Return a new FileTransfer object. */
ref<FileTransfer> makeFileTransfer();
class DownloadError : public Error
class FileTransferError : public Error
{
public:
Downloader::Error error;
DownloadError(Downloader::Error error, const FormatOrString & fs)
: Error(fs), error(error)
FileTransfer::Error error;
template<typename... Args>
FileTransferError(FileTransfer::Error error, const Args & ... args)
: Error(args...), error(error)
{ }
};
bool isUri(const string & s);
/* Resolve deprecated 'channel:<foo>' URLs. */
std::string resolveUri(const std::string & uri);
}

99
src/libstore/gc.cc
View file

@ -38,10 +38,10 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock)
throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
throw SysError("opening global GC lock '%1%'", fnGCLock);
if (!lockFile(fdGCLock.get(), lockType, false)) {
printError(format("waiting for the big garbage collector lock..."));
printInfo("waiting for the big garbage collector lock...");
lockFile(fdGCLock.get(), lockType, true);
}
@ -65,8 +65,8 @@ static void makeSymlink(const Path & link, const Path & target)
/* Atomically replace the old one. */
if (rename(tempLink.c_str(), link.c_str()) == -1)
throw SysError(format("cannot rename '%1%' to '%2%'")
% tempLink % link);
throw SysError("cannot rename '%1%' to '%2%'",
tempLink , link);
}
@ -91,15 +91,15 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
Path gcRoot(canonPath(_gcRoot));
if (isInStore(gcRoot))
throw Error(format(
throw Error(
"creating a garbage collector root (%1%) in the Nix store is forbidden "
"(are you running nix-build inside the store?)") % gcRoot);
"(are you running nix-build inside the store?)", gcRoot);
if (indirect) {
/* Don't clobber the link if it already exists and doesn't
point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
throw Error("cannot create symlink '%1%'; already exists", gcRoot);
makeSymlink(gcRoot, printStorePath(storePath));
addIndirectRoot(gcRoot);
}
@ -109,10 +109,10 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
throw Error(format(
throw Error(
"path '%1%' is not a valid garbage collector root; "
"it's not in the directory '%2%'")
% gcRoot % rootsDir);
"it's not in the directory '%2%'",
gcRoot, rootsDir);
}
if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
@ -128,11 +128,13 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
gcroots directory. */
if (settings.checkRootReachability) {
auto roots = findRoots(false);
if (roots[storePath.clone()].count(gcRoot) == 0)
printError(
"warning: '%1%' is not in a directory where the garbage collector looks for roots; "
"therefore, '%2%' might be removed by the garbage collector",
gcRoot, printStorePath(storePath));
if (roots[storePath].count(gcRoot) == 0)
logWarning({
.name = "GC root",
.hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
"therefore, '%2%' might be removed by the garbage collector",
gcRoot, printStorePath(storePath))
});
}
/* Grab the global GC root, causing us to block while a GC is in
@ -170,7 +172,7 @@ void LocalStore::addTempRoot(const StorePath & path)
way. */
struct stat st;
if (fstat(state->fdTempRoots.get(), &st) == -1)
throw SysError(format("statting '%1%'") % fnTempRoots);
throw SysError("statting '%1%'", fnTempRoots);
if (st.st_size == 0) break;
/* The garbage collector deleted this file before we could
@ -202,6 +204,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
/* Read the `temproots' directory for per-process temporary root
files. */
for (auto & i : readDirectory(tempRootsDir)) {
if (i.name[0] == '.') {
// Ignore hidden files. Some package managers (notably portage) create
// those to keep the directory alive.
continue;
}
Path path = tempRootsDir + "/" + i.name;
pid_t pid = std::stoi(i.name);
@ -211,7 +218,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
if (!*fd) {
/* It's okay if the file has disappeared. */
if (errno == ENOENT) continue;
throw SysError(format("opening temporary roots file '%1%'") % path);
throw SysError("opening temporary roots file '%1%'", path);
}
/* This should work, but doesn't, for some reason. */
@ -222,7 +229,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
only succeed if the owning process has died. In that case
we don't care about its temporary roots. */
if (lockFile(fd->get(), ltWrite, false)) {
printError(format("removing stale temporary roots file '%1%'") % path);
printInfo("removing stale temporary roots file '%1%'", path);
unlink(path.c_str());
writeFull(fd->get(), "d");
continue;
@ -398,7 +405,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
if (!fdDir) {
if (errno == ENOENT || errno == EACCES)
continue;
throw SysError(format("opening %1%") % fdStr);
throw SysError("opening %1%", fdStr);
}
struct dirent * fd_ent;
while (errno = 0, fd_ent = readdir(fdDir.get())) {
@ -408,13 +415,13 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
if (errno) {
if (errno == ESRCH)
continue;
throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
throw SysError("iterating /proc/%1%/fd", ent->d_name);
}
fdDir.reset();
try {
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
@ -422,7 +429,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
}
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile, true);
auto envString = readFile(envFile);
auto env_end = std::sregex_iterator{};
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
unchecked[i->str()].emplace(envFile);
@ -471,9 +478,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
if (!isValidPath(path)) continue;
debug("got additional root '%1%'", pathS);
if (censor)
roots[path.clone()].insert(censored);
roots[path].insert(censored);
else
roots[path.clone()].insert(links.begin(), links.end());
roots[path].insert(links.begin(), links.end());
}
}
@ -536,7 +543,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT) return;
throw SysError(format("getting status of %1%") % realPath);
throw SysError("getting status of %1%", realPath);
}
printInfo(format("deleting '%1%'") % path);
@ -554,10 +561,10 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
// size.
try {
if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("making '%1%' writable") % realPath);
throw SysError("making '%1%' writable", realPath);
Path tmp = trashDir + "/" + std::string(baseNameOf(path));
if (rename(realPath.c_str(), tmp.c_str()))
throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
state.bytesInvalidated += size;
} catch (SysError & e) {
if (e.errNo == ENOSPC) {
@ -585,11 +592,11 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
if (state.roots.count(path)) {
debug("cannot delete '%1%' because it's a root", printStorePath(path));
state.alive.insert(path.clone());
state.alive.insert(path);
return true;
}
visited.insert(path.clone());
visited.insert(path);
if (!isValidPath(path)) return false;
@ -603,7 +610,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
if (state.gcKeepDerivations && path.isDerivation()) {
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
incoming.insert(i.clone());
incoming.insert(i);
}
/* If keep-outputs is set, then don't delete this path if there
@ -611,13 +618,13 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
if (state.gcKeepOutputs) {
auto derivers = queryValidDerivers(path);
for (auto & i : derivers)
incoming.insert(i.clone());
incoming.insert(i);
}
for (auto & i : incoming)
if (i != path)
if (canReachRoot(state, visited, i)) {
state.alive.insert(path.clone());
state.alive.insert(path);
return true;
}
@ -661,7 +668,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
‘nix-store --delete’ doesn't have the unexpected effect of
recursing into derivations and outputs. */
for (auto & i : visited)
state.dead.insert(i.clone());
state.dead.insert(i);
if (state.shouldDelete)
deletePathRecursive(state, path);
}
@ -676,7 +683,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
void LocalStore::removeUnusedLinks(const GCState & state)
{
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
if (!dir) throw SysError("opening directory '%1%'", linksDir);
long long actualSize = 0, unsharedSize = 0;
@ -689,7 +696,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
struct stat st;
if (lstat(path.c_str(), &st) == -1)
throw SysError(format("statting '%1%'") % path);
throw SysError("statting '%1%'", path);
if (st.st_nlink != 1) {
actualSize += st.st_size;
@ -700,14 +707,14 @@ void LocalStore::removeUnusedLinks(const GCState & state)
printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
if (unlink(path.c_str()) == -1)
throw SysError(format("deleting '%1%'") % path);
throw SysError("deleting '%1%'", path);
state.results.bytesFreed += st.st_size;
}
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError(format("statting '%1%'") % linksDir);
throw SysError("statting '%1%'", linksDir);
long long overhead = st.st_blocks * 512ULL;
printInfo(format("note: currently hard linking saves %.2f MiB")
@ -742,12 +749,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */
printError("finding garbage collector roots...");
printInfo("finding garbage collector roots...");
Roots rootMap;
if (!options.ignoreLiveness)
findRootsNoTemp(rootMap, true);
for (auto & i : rootMap) state.roots.insert(i.first.clone());
for (auto & i : rootMap) state.roots.insert(i.first);
/* Read the temporary roots. This acquires read locks on all
per-process temporary root files. So after this point no paths
@ -756,8 +763,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
Roots tempRoots;
findTempRoots(fds, tempRoots, true);
for (auto & root : tempRoots) {
state.tempRoots.insert(root.first.clone());
state.roots.insert(root.first.clone());
state.tempRoots.insert(root.first);
state.roots.insert(root.first);
}
/* After this point the set of roots or temporary roots cannot
@ -794,14 +801,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
} else if (options.maxFreed > 0) {
if (state.shouldDelete)
printError("deleting garbage...");
printInfo("deleting garbage...");
else
printError("determining live/dead paths...");
printInfo("determining live/dead paths...");
try {
AutoCloseDir dir(opendir(realStoreDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir);
if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
/* Read the store and immediately delete all paths that
aren't valid. When using --max-freed etc., deleting
@ -863,7 +870,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* Clean up the links directory. */
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
printError("deleting unused links...");
printInfo("deleting unused links...");
removeUnusedLinks(state);
}
@ -884,7 +891,7 @@ void LocalStore::autoGC(bool sync)
if (statvfs(realStoreDir.c_str(), &st))
throw SysError("getting filesystem info about '%s'", realStoreDir);
return (uint64_t) st.f_bavail * st.f_bsize;
return (uint64_t) st.f_bavail * st.f_frsize;
};
std::shared_future<void> future;

74
src/libstore/globals.cc
View file

@ -20,13 +20,6 @@ namespace nix {
must be deleted and recreated on startup.) */
#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
/* chroot-like behavior from Apple's sandbox */
#if __APPLE__
#define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh"
#else
#define DEFAULT_ALLOWED_IMPURE_PREFIXES ""
#endif
Settings settings;
static GlobalConfig::Register r1(&settings);
@ -38,6 +31,7 @@ Settings::Settings()
, nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
, nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
, nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
, nixUserConfFiles(getUserConfigFiles())
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
, nixManDir(canonPath(NIX_MAN_DIR))
@ -68,7 +62,12 @@ Settings::Settings()
sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
#endif
allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
/* chroot-like behavior from Apple's sandbox */
#if __APPLE__
sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib");
allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
#endif
}
void loadConfFile()
@ -79,13 +78,29 @@ void loadConfFile()
~/.nix/nix.conf or the command line. */
globalConfig.resetOverriden();
auto dirs = getConfigDirs();
// Iterate over them in reverse so that the ones appearing first in the path take priority
for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
auto files = settings.nixUserConfFiles;
for (auto file = files.rbegin(); file != files.rend(); file++) {
globalConfig.applyConfigFile(*file);
}
}
std::vector<Path> getUserConfigFiles()
{
// Use the paths specified in NIX_USER_CONF_FILES if it has been defined
auto nixConfFiles = getEnv("NIX_USER_CONF_FILES");
if (nixConfFiles.has_value()) {
return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":");
}
// Use the paths specified by the XDG spec
std::vector<Path> files;
auto dirs = getConfigDirs();
for (auto & dir : dirs) {
files.insert(files.end(), dir + "/nix/nix.conf");
}
return files;
}
unsigned int Settings::getDefaultCores()
{
return std::max(1U, std::thread::hardware_concurrency());
@ -115,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
void Settings::requireExperimentalFeature(const std::string & name)
{
if (!isExperimentalFeatureEnabled(name))
throw Error("experimental Nix feature '%s' is disabled", name);
throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
}
bool Settings::isWSL1()
@ -152,21 +167,24 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
{
args.mkFlag()
.longName(name)
.description("Enable sandboxing.")
.handler([=](std::vector<std::string> ss) { override(smEnabled); })
.category(category);
args.mkFlag()
.longName("no-" + name)
.description("Disable sandboxing.")
.handler([=](std::vector<std::string> ss) { override(smDisabled); })
.category(category);
args.mkFlag()
.longName("relaxed-" + name)
.description("Enable sandboxing, but allow builds to disable it.")
.handler([=](std::vector<std::string> ss) { override(smRelaxed); })
.category(category);
args.addFlag({
.longName = name,
.description = "Enable sandboxing.",
.category = category,
.handler = {[=]() { override(smEnabled); }}
});
args.addFlag({
.longName = "no-" + name,
.description = "Disable sandboxing.",
.category = category,
.handler = {[=]() { override(smDisabled); }}
});
args.addFlag({
.longName = "relaxed-" + name,
.description = "Enable sandboxing, but allow builds to disable it.",
.category = category,
.handler = {[=]() { override(smRelaxed); }}
});
}
void MaxBuildJobsSetting::set(const std::string & str)

26
src/libstore/globals.hh
View file

@ -53,9 +53,12 @@ public:
/* The directory where state is stored. */
Path nixStateDir;
/* The directory where configuration files are stored. */
/* The directory where system configuration files are stored. */
Path nixConfDir;
/* A list of user configuration files to load. */
std::vector<Path> nixUserConfFiles;
/* The directory where internal helper programs are stored. */
Path nixLibexecDir;
@ -268,7 +271,7 @@ public:
"listed in 'trusted-public-keys'."};
Setting<StringSet> extraPlatforms{this,
std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
"extra-platforms",
"Additional platforms that can be built on the local system. "
"These may be supported natively (e.g. armv7 on some aarch64 CPUs "
@ -311,12 +314,7 @@ public:
Setting<bool> printMissing{this, true, "print-missing",
"Whether to print what paths need to be built or downloaded."};
Setting<std::string> preBuildHook{this,
#if __APPLE__
nixLibexecDir + "/nix/resolve-system-dependencies",
#else
"",
#endif
Setting<std::string> preBuildHook{this, "",
"pre-build-hook",
"A program to run just before a build to set derivation-specific build settings."};
@ -356,12 +354,21 @@ public:
Setting<Paths> pluginFiles{this, {}, "plugin-files",
"Plugins to dynamically load at nix initialization time."};
Setting<std::string> githubAccessToken{this, "", "github-access-token",
"GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."};
bool isExperimentalFeatureEnabled(const std::string & name);
void requireExperimentalFeature(const std::string & name);
Setting<bool> allowDirty{this, true, "allow-dirty",
"Whether to allow dirty Git/Mercurial trees."};
Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."};
};
@ -374,6 +381,9 @@ void initPlugins();
void loadConfFile();
// Used by the Settings constructor
std::vector<Path> getUserConfigFiles();
extern const string nixVersion;
}

40
src/libstore/http-binary-cache-store.cc
View file

@ -1,5 +1,5 @@
#include "binary-cache-store.hh"
#include "download.hh"
#include "filetransfer.hh"
#include "globals.hh"
#include "nar-info-disk-cache.hh"
@ -85,14 +85,14 @@ protected:
checkEnabled();
try {
DownloadRequest request(cacheUri + "/" + path);
FileTransferRequest request(cacheUri + "/" + path);
request.head = true;
getDownloader()->download(request);
getFileTransfer()->download(request);
return true;
} catch (DownloadError & e) {
} catch (FileTransferError & e) {
/* S3 buckets return 403 if a file doesn't exist and the
bucket is unlistable, so treat 403 as 404. */
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return false;
maybeDisable();
throw;
@ -103,19 +103,19 @@ protected:
const std::string & data,
const std::string & mimeType) override
{
auto req = DownloadRequest(cacheUri + "/" + path);
auto req = FileTransferRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient
req.mimeType = mimeType;
try {
getDownloader()->download(req);
} catch (DownloadError & e) {
getFileTransfer()->upload(req);
} catch (FileTransferError & e) {
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
}
}
DownloadRequest makeRequest(const std::string & path)
FileTransferRequest makeRequest(const std::string & path)
{
DownloadRequest request(cacheUri + "/" + path);
FileTransferRequest request(cacheUri + "/" + path);
return request;
}
@ -124,9 +124,9 @@ protected:
checkEnabled();
auto request(makeRequest(path));
try {
getDownloader()->download(std::move(request), sink);
} catch (DownloadError & e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
getFileTransfer()->download(std::move(request), sink);
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
maybeDisable();
throw;
@ -142,12 +142,12 @@ protected:
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getDownloader()->enqueueDownload(request,
{[callbackPtr, this](std::future<DownloadResult> result) {
getFileTransfer()->enqueueFileTransfer(request,
{[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(result.get().data);
} catch (DownloadError & e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return (*callbackPtr)(std::shared_ptr<std::string>());
maybeDisable();
callbackPtr->rethrow();
@ -163,14 +163,14 @@ static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://" &&
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
) return 0;
(!forceHttp || std::string(uri, 0, 7) != "file://"))
return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init();
return store;
});
}

4
src/libstore/legacy-ssh-store.cc
View file

@ -195,7 +195,7 @@ struct LegacySSHStore : public Store
{ unsupported("queryPathFromHashPart"); }
StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
{ unsupported("addToStore"); }
@ -256,7 +256,7 @@ struct LegacySSHStore : public Store
conn->to.flush();
for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
out.insert(i.clone());
out.insert(i);
}
StorePathSet queryValidPaths(const StorePathSet & paths,

2
src/libstore/local-binary-cache-store.cc
View file

@ -74,7 +74,7 @@ static void atomicWrite(const Path & path, const std::string & s)
AutoDelete del(tmp, false);
writeFile(tmp, s);
if (rename(tmp.c_str(), path.c_str()))
throw SysError(format("renaming '%1%' to '%2%'") % tmp % path);
throw SysError("renaming '%1%' to '%2%'", tmp, path);
del.cancel();
}

10
src/libstore/local-fs-store.cc
View file

@ -22,7 +22,7 @@ struct LocalStoreAccessor : public FSAccessor
{
Path storePath = store->toStorePath(path);
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
throw InvalidPath("path '%1%' is not a valid store path", storePath);
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}
@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
throw SysError(format("getting status of '%1%'") % path);
throw SysError("getting status of '%1%'", path);
}
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
throw Error(format("file '%1%' has unsupported type") % path);
throw Error("file '%1%' has unsupported type", path);
return {
S_ISREG(st.st_mode) ? Type::tRegular :
@ -90,13 +90,13 @@ const string LocalFSStore::drvsLogDir = "drvs";
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
{
auto path = path_.clone();
auto path = path_;
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return nullptr;
path = info->deriver->clone();
path = *info->deriver;
} catch (InvalidPath &) {
return nullptr;
}

205
src/libstore/local-store.cc
View file

@ -87,18 +87,22 @@ LocalStore::LocalStore(const Params & params)
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr)
printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist")
% settings.buildUsersGroup);
logError({
.name = "'build-users-group' not found",
.hint = hintfmt(
"warning: the group '%1%' specified in 'build-users-group' does not exist",
settings.buildUsersGroup)
});
else {
struct stat st;
if (stat(realStoreDir.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % realStoreDir);
throw SysError("getting attributes of path '%1%'", realStoreDir);
if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
throw SysError(format("changing ownership of path '%1%'") % realStoreDir);
throw SysError("changing ownership of path '%1%'", realStoreDir);
if (chmod(realStoreDir.c_str(), perm) == -1)
throw SysError(format("changing permissions on path '%1%'") % realStoreDir);
throw SysError("changing permissions on path '%1%'", realStoreDir);
}
}
}
@ -109,12 +113,12 @@ LocalStore::LocalStore(const Params & params)
struct stat st;
while (path != "/") {
if (lstat(path.c_str(), &st))
throw SysError(format("getting status of '%1%'") % path);
throw SysError("getting status of '%1%'", path);
if (S_ISLNK(st.st_mode))
throw Error(format(
throw Error(
"the path '%1%' is a symlink; "
"this is not allowed for the Nix store and its parent directories")
% path);
"this is not allowed for the Nix store and its parent directories",
path);
path = dirOf(path);
}
}
@ -147,7 +151,7 @@ LocalStore::LocalStore(const Params & params)
globalLock = openLockFile(globalLockPath.c_str(), true);
if (!lockFile(globalLock.get(), ltRead, false)) {
printError("waiting for the big Nix store lock...");
printInfo("waiting for the big Nix store lock...");
lockFile(globalLock.get(), ltRead, true);
}
@ -155,8 +159,8 @@ LocalStore::LocalStore(const Params & params)
upgrade. */
int curSchema = getSchema();
if (curSchema > nixSchemaVersion)
throw Error(format("current Nix store schema is version %1%, but I only support %2%")
% curSchema % nixSchemaVersion);
throw Error("current Nix store schema is version %1%, but I only support %2%",
curSchema, nixSchemaVersion);
else if (curSchema == 0) { /* new store */
curSchema = nixSchemaVersion;
@ -178,7 +182,7 @@ LocalStore::LocalStore(const Params & params)
"please upgrade Nix to version 1.11 first.");
if (!lockFile(globalLock.get(), ltWrite, false)) {
printError("waiting for exclusive access to the Nix store...");
printInfo("waiting for exclusive access to the Nix store...");
lockFile(globalLock.get(), ltWrite, true);
}
@ -256,7 +260,7 @@ LocalStore::~LocalStore()
}
if (future.valid()) {
printError("waiting for auto-GC to finish on exit...");
printInfo("waiting for auto-GC to finish on exit...");
future.get();
}
@ -284,7 +288,7 @@ int LocalStore::getSchema()
if (pathExists(schemaPath)) {
string s = readFile(schemaPath);
if (!string2Int(s, curSchema))
throw Error(format("'%1%' is corrupt") % schemaPath);
throw Error("'%1%' is corrupt", schemaPath);
}
return curSchema;
}
@ -293,14 +297,12 @@ int LocalStore::getSchema()
void LocalStore::openDB(State & state, bool create)
{
if (access(dbDir.c_str(), R_OK | W_OK))
throw SysError(format("Nix database directory '%1%' is not writable") % dbDir);
throw SysError("Nix database directory '%1%' is not writable", dbDir);
/* Open the Nix database. */
string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
if (sqlite3_open_v2(dbPath.c_str(), &db.db,
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open Nix database '%1%'") % dbPath);
state.db = SQLite(dbPath, create);
#ifdef __CYGWIN__
/* The cygwin version of sqlite3 has a patch which calls
@ -312,11 +314,6 @@ void LocalStore::openDB(State & state, bool create)
SetDllDirectoryW(L"");
#endif
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
db.exec("pragma foreign_keys = 1");
/* !!! check whether sqlite has been built with foreign key
support */
@ -350,7 +347,7 @@ void LocalStore::openDB(State & state, bool create)
/* Initialise the database schema, if necessary. */
if (create) {
const char * schema =
static const char schema[] =
#include "schema.sql.gen.hh"
;
db.exec(schema);
@ -374,7 +371,7 @@ void LocalStore::makeStoreWritable()
throw SysError("setting up a private mount namespace");
if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
throw SysError(format("remounting %1% writable") % realStoreDir);
throw SysError("remounting %1% writable", realStoreDir);
}
#endif
}
@ -395,7 +392,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
| 0444
| (st.st_mode & S_IXUSR ? 0111 : 0);
if (chmod(path.c_str(), mode) == -1)
throw SysError(format("changing mode of '%1%' to %2$o") % path % mode);
throw SysError("changing mode of '%1%' to %2$o", path, mode);
}
}
@ -413,7 +410,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
#else
if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
#endif
throw SysError(format("changing modification time of '%1%'") % path);
throw SysError("changing modification time of '%1%'", path);
}
}
@ -422,7 +419,7 @@ void canonicaliseTimestampAndPermissions(const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
throw SysError("getting attributes of path '%1%'", path);
canonicaliseTimestampAndPermissions(path, st);
}
@ -437,17 +434,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
setattrlist() to remove other attributes as well. */
if (lchflags(path.c_str(), 0)) {
if (errno != ENOTSUP)
throw SysError(format("clearing flags of path '%1%'") % path);
throw SysError("clearing flags of path '%1%'", path);
}
#endif
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
throw SysError("getting attributes of path '%1%'", path);
/* Really make sure that the path is of a supported type. */
if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
throw Error(format("file '%1%' has an unsupported type") % path);
throw Error("file '%1%' has an unsupported type", path);
#if __linux__
/* Remove extended attributes / ACLs. */
@ -481,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
assert(!S_ISDIR(st.st_mode));
if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
throw BuildError(format("invalid ownership on file '%1%'") % path);
throw BuildError("invalid ownership on file '%1%'", path);
mode_t mode = st.st_mode & ~S_IFMT;
assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
return;
@ -505,8 +502,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
if (!S_ISLNK(st.st_mode) &&
chown(path.c_str(), geteuid(), getegid()) == -1)
#endif
throw SysError(format("changing owner of '%1%' to %2%")
% path % geteuid());
throw SysError("changing owner of '%1%' to %2%",
path, geteuid());
}
if (S_ISDIR(st.st_mode)) {
@ -525,11 +522,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
be a symlink, since we can't change its ownership. */
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
throw SysError("getting attributes of path '%1%'", path);
if (st.st_uid != geteuid()) {
assert(S_ISLNK(st.st_mode));
throw Error(format("wrong ownership of top-level store path '%1%'") % path);
throw Error("wrong ownership of top-level store path '%1%'", path);
}
}
@ -564,10 +561,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
bool recursive; Hash h;
out->second.parseHashInfo(recursive, h);
FileIngestionMethod method; Hash h;
out->second.parseHashInfo(method, h);
check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
}
else {
@ -589,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtRegisterValidPath.use()
(printStorePath(info.path))
(info.narHash.to_string(Base16))
(info.narHash.to_string(Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
(info.narSize, info.narSize != 0)
@ -604,7 +601,7 @@ uint64_t LocalStore::addValidPath(State & state,
efficiently query whether a path is an output of some
derivation. */
if (info.path.isDerivation()) {
auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string()));
auto drv = readDerivation(info.path);
/* Verify that the output paths in the derivation are correct
(i.e., follow the scheme for computing output paths from
@ -624,7 +621,7 @@ uint64_t LocalStore::addValidPath(State & state,
{
auto state_(Store::state.lock());
state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)),
state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
}
@ -636,7 +633,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
try {
auto info = std::make_shared<ValidPathInfo>(path.clone());
auto info = std::make_shared<ValidPathInfo>(path);
callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
auto state(_state.lock());
@ -689,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
state.stmtUpdatePathInfo.use()
(info.narSize, info.narSize != 0)
(info.narHash.to_string(Base16))
(info.narHash.to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(info.ca, !info.ca.empty())
@ -726,7 +723,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF
{
StorePathSet res;
for (auto & i : paths)
if (isValidPath(i)) res.insert(i.clone());
if (isValidPath(i)) res.insert(i);
return res;
}
@ -794,26 +791,9 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
}
StringSet LocalStore::queryDerivationOutputNames(const StorePath & path)
{
return retrySQLite<StringSet>([&]() {
auto state(_state.lock());
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
(queryValidPathId(*state, path)));
StringSet outputNames;
while (useQueryDerivationOutputs.next())
outputNames.insert(useQueryDerivationOutputs.getStr(0));
return outputNames;
});
}
std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
{
if (hashPart.size() != storePathHashLen) throw Error("invalid hash part");
if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part");
Path prefix = storeDir + "/" + hashPart;
@ -838,7 +818,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
StorePathSet remaining;
for (auto & i : paths)
remaining.insert(i.clone());
remaining.insert(i);
StorePathSet res;
@ -852,9 +832,9 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
StorePathSet remaining2;
for (auto & path : remaining)
if (valid.count(path))
res.insert(path.clone());
res.insert(path);
else
remaining2.insert(path.clone());
remaining2.insert(path);
std::swap(remaining, remaining2);
}
@ -876,16 +856,16 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
auto info = sub->queryPathInfo(path);
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
infos.insert_or_assign(path.clone(), SubstitutablePathInfo{
info->deriver ? info->deriver->clone() : std::optional<StorePath>(),
cloneStorePathSet(info->references),
infos.insert_or_assign(path, SubstitutablePathInfo{
info->deriver,
info->references,
narInfo ? narInfo->fileSize : 0,
info->narSize});
} catch (InvalidPath &) {
} catch (SubstituterDisabled &) {
} catch (Error & e) {
if (settings.tryFallback)
printError(e.what());
logError(e.info());
else
throw;
}
@ -922,7 +902,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
updatePathInfo(*state, i);
else
addValidPath(*state, i, false);
paths.insert(i.path.clone());
paths.insert(i.path);
}
for (auto & i : infos) {
@ -937,8 +917,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
for (auto & i : infos)
if (i.path.isDerivation()) {
// FIXME: inefficient; we already loaded the derivation in addValidPath().
checkDerivationOutputs(i.path,
readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string())));
checkDerivationOutputs(i.path, readDerivation(i.path));
}
/* Do a topological sort of the paths. This will throw an
@ -965,7 +944,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
{
auto state_(Store::state.lock());
state_->pathInfoCache.erase(storePathToHash(printStorePath(path)));
state_->pathInfoCache.erase(std::string(path.hashPart()));
}
}
@ -1017,7 +996,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (info.ca == "" || !info.references.count(info.path))
hashSink = std::make_unique<HashSink>(htSHA256);
else
hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
size_t n = source.read(data, len);
@ -1031,7 +1010,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string());
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
@ -1052,11 +1031,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
bool recursive, HashType hashAlgo, RepairFlag repair)
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{
Hash h = hashString(hashAlgo, dump);
auto dstPath = makeFixedOutputPath(recursive, h, name);
auto dstPath = makeFixedOutputPath(method, h, name);
addTempRoot(dstPath);
@ -1076,7 +1055,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
autoGC();
if (recursive) {
if (method == FileIngestionMethod::Recursive) {
StringSource source(dump);
restorePath(realPath, source);
} else
@ -1089,7 +1068,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
above (if called with recursive == true and hashAlgo ==
sha256); otherwise, compute it here. */
HashResult hash;
if (recursive) {
if (method == FileIngestionMethod::Recursive) {
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
hash.second = dump.size();
} else
@ -1097,10 +1076,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
optimisePath(realPath); // FIXME: combine with hashPath()
ValidPathInfo info(dstPath.clone());
ValidPathInfo info(dstPath);
info.narHash = hash.first;
info.narSize = hash.second;
info.ca = makeFixedOutputCA(recursive, h);
info.ca = makeFixedOutputCA(method, h);
registerValidPath(info);
}
@ -1112,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
Path srcPath(absPath(_srcPath));
@ -1120,12 +1099,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
if (recursive)
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = make_ref<std::string>(readFile(srcPath));
return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair);
return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
}
@ -1160,11 +1139,11 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
optimisePath(realPath);
ValidPathInfo info(dstPath.clone());
ValidPathInfo info(dstPath);
info.narHash = narHash;
info.narSize = sink.s->size();
info.references = cloneStorePathSet(references);
info.ca = "text:" + hash.to_string();
info.references = references;
info.ca = "text:" + hash.to_string(Base32, true);
registerValidPath(info);
}
@ -1214,7 +1193,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path)
bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
{
printError(format("reading the Nix store..."));
printInfo(format("reading the Nix store..."));
bool errors = false;
@ -1246,12 +1225,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
Path linkPath = linksDir + "/" + link.name;
string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
if (hash != link.name) {
printError(
"link '%s' was modified! expected hash '%s', got '%s'",
linkPath, link.name, hash);
logError({
.name = "Invalid hash",
.hint = hintfmt(
"link '%s' was modified! expected hash '%s', got '%s'",
linkPath, link.name, hash)
});
if (repair) {
if (unlink(linkPath.c_str()) == 0)
printError("removed link '%s'", linkPath);
printInfo("removed link '%s'", linkPath);
else
throw SysError("removing corrupt link '%s'", linkPath);
} else {
@ -1275,14 +1257,17 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
if (info->ca == "" || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
dumpPath(toRealPath(printStorePath(i)), *hashSink);
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();
if (info->narHash != nullHash && info->narHash != current.first) {
printError("path '%s' was modified! expected hash '%s', got '%s'",
printStorePath(i), info->narHash.to_string(), current.first.to_string());
logError({
.name = "Invalid hash - path modified",
.hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
});
if (repair) repairPath(i); else errors = true;
} else {
@ -1290,14 +1275,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Fill in missing hashes. */
if (info->narHash == nullHash) {
printError("fixing missing hash on '%s'", printStorePath(i));
printInfo("fixing missing hash on '%s'", printStorePath(i));
info->narHash = current.first;
update = true;
}
/* Fill in missing narSize fields (from old stores). */
if (info->narSize == 0) {
printError("updating size field on '%s' to %s", printStorePath(i), current.second);
printInfo("updating size field on '%s' to %s", printStorePath(i), current.second);
info->narSize = current.second;
update = true;
}
@ -1313,7 +1298,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* It's possible that the path got GC'ed, so ignore
errors on invalid paths. */
if (isValidPath(i))
printError("error: %s", e.msg());
logError(e.info());
else
warn(e.msg());
errors = true;
@ -1333,7 +1318,10 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
if (!done.insert(pathS).second) return;
if (!isStorePath(pathS)) {
printError("path '%s' is not in the Nix store", pathS);
logError({
.name = "Nix path not found",
.hint = hintfmt("path '%s' is not in the Nix store", pathS)
});
return;
}
@ -1352,16 +1340,19 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
}
if (canInvalidate) {
printError("path '%s' disappeared, removing from database...", pathS);
printInfo("path '%s' disappeared, removing from database...", pathS);
auto state(_state.lock());
invalidatePath(*state, path);
} else {
printError("path '%s' disappeared, but it still has valid referrers!", pathS);
logError({
.name = "Missing path with referrers",
.hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
});
if (repair)
try {
repairPath(path);
} catch (Error & e) {
warn(e.msg());
logWarning(e.info());
errors = true;
}
else errors = true;
@ -1401,7 +1392,7 @@ static void makeMutable(const Path & path)
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
if (fd == -1) {
if (errno == ELOOP) return; // it's a symlink
throw SysError(format("opening file '%1%'") % path);
throw SysError("opening file '%1%'", path);
}
unsigned int flags = 0, old;
@ -1419,7 +1410,7 @@ static void makeMutable(const Path & path)
void LocalStore::upgradeStore7()
{
if (getuid() != 0) return;
printError("removing immutable bits from the Nix store (this may take a while)...");
printInfo("removing immutable bits from the Nix store (this may take a while)...");
makeMutable(realStoreDir);
}

6
src/libstore/local-store.hh
View file

@ -135,8 +135,6 @@ public:
StorePathSet queryDerivationOutputs(const StorePath & path) override;
StringSet queryDerivationOutputNames(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@ -149,7 +147,7 @@ public:
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
/* Like addToStore(), but the contents of the path are contained
@ -157,7 +155,7 @@ public:
true) or simply the contents of a regular file (if recursive ==
false). */
StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;

5
src/libstore/local.mk
View file

@ -6,7 +6,7 @@ libstore_DIR := $(d)
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
libstore_LIBS = libutil libnixrust
libstore_LIBS = libutil
libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
ifneq ($(OS), FreeBSD)
@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1)
libstore_LDFLAGS += -lseccomp
endif
libstore_CXXFLAGS = \
libstore_CXXFLAGS += \
-I src/libutil -I src/libstore \
-DNIX_PREFIX=\"$(prefix)\" \
-DNIX_STORE_DIR=\"$(storedir)\" \
-DNIX_DATA_DIR=\"$(datadir)\" \

36
src/libstore/misc.cc
View file

@ -103,7 +103,7 @@ void Store::computeFSClosure(const StorePath & startPath,
StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{
StorePathSet paths;
paths.insert(startPath.clone());
paths.insert(startPath);
computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers);
}
@ -141,11 +141,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
{
auto state(state_.lock());
state->willBuild.insert(drvPath.clone());
state->willBuild.insert(drvPath);
}
for (auto & i : drv.inputDrvs)
pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second)));
pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second }));
};
auto checkOutput = [&](
@ -157,9 +157,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos;
StorePathSet paths; // FIXME
paths.insert(outPath.clone());
querySubstitutablePathInfos(paths, infos);
querySubstitutablePathInfos({outPath}, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
@ -170,10 +168,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
if (drvState->done) return;
assert(drvState->left);
drvState->left--;
drvState->outPaths.insert(outPath.clone());
drvState->outPaths.insert(outPath);
if (!drvState->left) {
for (auto & path : drvState->outPaths)
pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone())));
pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } ));
}
}
}
@ -190,12 +188,12 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
if (!isValidPath(path.path)) {
// FIXME: we could try to substitute the derivation.
auto state(state_.lock());
state->unknown.insert(path.path.clone());
state->unknown.insert(path.path);
return;
}
auto drv = make_ref<Derivation>(derivationFromPath(path.path));
ParsedDerivation parsedDrv(path.path.clone(), *drv);
ParsedDerivation parsedDrv(StorePath(path.path), *drv);
PathSet invalid;
for (auto & j : drv->outputs)
@ -216,13 +214,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
if (isValidPath(path.path)) return;
SubstitutablePathInfos infos;
StorePathSet paths; // FIXME
paths.insert(path.path.clone());
querySubstitutablePathInfos(paths, infos);
querySubstitutablePathInfos({path.path}, infos);
if (infos.empty()) {
auto state(state_.lock());
state->unknown.insert(path.path.clone());
state->unknown.insert(path.path);
return;
}
@ -231,13 +227,13 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
{
auto state(state_.lock());
state->willSubstitute.insert(path.path.clone());
state->willSubstitute.insert(path.path);
state->downloadSize += info->second.downloadSize;
state->narSize += info->second.narSize;
}
for (auto & ref : info->second.references)
pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref)));
pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref }));
}
};
@ -260,12 +256,12 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
throw BuildError("cycle detected in the references of '%s' from '%s'",
printStorePath(path), printStorePath(*parent));
if (!visited.insert(path.clone()).second) return;
parents.insert(path.clone());
if (!visited.insert(path).second) return;
parents.insert(path);
StorePathSet references;
try {
references = cloneStorePathSet(queryPathInfo(path)->references);
references = queryPathInfo(path)->references;
} catch (InvalidPath &) {
}
@ -275,7 +271,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
if (i != path && paths.count(i))
dfsVisit(i, &path);
sorted.push_back(path.clone());
sorted.push_back(path);
parents.erase(path);
};

0
src/libexpr/names.cc → src/libstore/names.cc
View file

0
src/libexpr/names.hh → src/libstore/names.hh
View file

6
src/libstore/nar-accessor.cc
View file

@ -184,7 +184,7 @@ struct NarAccessor : public FSAccessor
auto i = get(path);
if (i.type != FSAccessor::Type::tDirectory)
throw Error(format("path '%1%' inside NAR file is not a directory") % path);
throw Error("path '%1%' inside NAR file is not a directory", path);
StringSet res;
for (auto & child : i.children)
@ -197,7 +197,7 @@ struct NarAccessor : public FSAccessor
{
auto i = get(path);
if (i.type != FSAccessor::Type::tRegular)
throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
throw Error("path '%1%' inside NAR file is not a regular file", path);
if (getNarBytes) return getNarBytes(i.start, i.size);
@ -209,7 +209,7 @@ struct NarAccessor : public FSAccessor
{
auto i = get(path);
if (i.type != FSAccessor::Type::tSymlink)
throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
throw Error("path '%1%' inside NAR file is not a symlink", path);
return i.target;
}
};

17
src/libstore/nar-info-disk-cache.cc
View file

@ -78,12 +78,7 @@ public:
state->db = SQLite(dbPath);
if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(state->db, "setting timeout");
// We can always reproduce the cache.
state->db.exec("pragma synchronous = off");
state->db.exec("pragma main.journal_mode = truncate");
state->db.isCache();
state->db.exec(schema);
@ -194,7 +189,7 @@ public:
return {oInvalid, 0};
auto namePart = queryNAR.getStr(1);
auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart));
auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart));
narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4))
@ -203,9 +198,9 @@ public:
narInfo->narHash = Hash(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7);
for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
narInfo->references.insert(StorePath::fromBaseName(r));
narInfo->references.insert(StorePath(r));
if (!queryNAR.isNull(9))
narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9));
narInfo->deriver = StorePath(queryNAR.getStr(9));
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig);
narInfo->ca = queryNAR.getStr(11);
@ -235,9 +230,9 @@ public:
(std::string(info->path.name()))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
(info->narHash.to_string())
(info->narHash.to_string(Base32, true))
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)

2
src/libstore/nar-info-disk-cache.hh
View file

@ -10,7 +10,7 @@ class NarInfoDiskCache
public:
typedef enum { oValid, oInvalid, oUnknown } Outcome;
virtual ~NarInfoDiskCache() { };
virtual ~NarInfoDiskCache() { }
virtual void createCache(const std::string & uri, const Path & storeDir,
bool wantMassQuery, int priority) = 0;

12
src/libstore/nar-info.cc
View file

@ -4,10 +4,10 @@
namespace nix {
NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
: ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack
: ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack
{
auto corrupt = [&]() {
throw Error(format("NAR info file '%1%' is corrupt") % whence);
throw Error("NAR info file '%1%' is corrupt", whence);
};
auto parseHashField = [&](const string & s) {
@ -56,11 +56,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
auto refs = tokenizeString<Strings>(value, " ");
if (!references.empty()) corrupt();
for (auto & r : refs)
references.insert(StorePath::fromBaseName(r));
references.insert(StorePath(r));
}
else if (name == "Deriver") {
if (value != "unknown-deriver")
deriver = StorePath::fromBaseName(value);
deriver = StorePath(value);
}
else if (name == "System")
system = value;
@ -87,10 +87,10 @@ std::string NarInfo::to_string(const Store & store) const
assert(compression != "");
res += "Compression: " + compression + "\n";
assert(fileHash.type == htSHA256);
res += "FileHash: " + fileHash.to_string(Base32) + "\n";
res += "FileHash: " + fileHash.to_string(Base32, true) + "\n";
res += "FileSize: " + std::to_string(fileSize) + "\n";
assert(narHash.type == htSHA256);
res += "NarHash: " + narHash.to_string(Base32) + "\n";
res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
res += "NarSize: " + std::to_string(narSize) + "\n";
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";

35
src/libstore/optimise-store.cc
View file

@ -19,9 +19,9 @@ static void makeWritable(const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
throw SysError("getting attributes of path '%1%'", path);
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("changing writability of '%1%'") % path);
throw SysError("changing writability of '%1%'", path);
}
@ -47,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
InodeHash inodeHash;
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
if (!dir) throw SysError("opening directory '%1%'", linksDir);
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
// We don't care if we hit non-hash files, anything goes
inodeHash.insert(dirent->d_ino);
}
if (errno) throw SysError(format("reading directory '%1%'") % linksDir);
if (errno) throw SysError("reading directory '%1%'", linksDir);
printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
@ -68,7 +68,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
Strings names;
AutoCloseDir dir(opendir(path.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % path);
if (!dir) throw SysError("opening directory '%1%'", path);
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@ -83,7 +83,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
if (name == "." || name == "..") continue;
names.push_back(name);
}
if (errno) throw SysError(format("reading directory '%1%'") % path);
if (errno) throw SysError("reading directory '%1%'", path);
return names;
}
@ -96,7 +96,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
throw SysError("getting attributes of path '%1%'", path);
#if __APPLE__
/* HFS/macOS has some undocumented security feature disabling hardlinking for
@ -130,7 +130,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
NixOS (example: $fontconfig/var/cache being modified). Skip
those files. FIXME: check the modification time. */
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
printError(format("skipping suspicious writable file '%1%'") % path);
logWarning({
.name = "Suspicious file",
.hint = hintfmt("skipping suspicious writable file '%1%'", path)
});
return;
}
@ -150,7 +153,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
contents of the symlink (i.e. the result of readlink()), not
the contents of the target (which may not even exist). */
Hash hash = hashPath(htSHA256, path).first;
debug(format("'%1%' has hash '%2%'") % path % hash.to_string());
debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true));
/* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
@ -186,7 +189,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
current file with a hard link to that file. */
struct stat stLink;
if (lstat(linkPath.c_str(), &stLink))
throw SysError(format("getting attributes of path '%1%'") % linkPath);
throw SysError("getting attributes of path '%1%'", linkPath);
if (st.st_ino == stLink.st_ino) {
debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
@ -194,7 +197,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
}
if (st.st_size != stLink.st_size) {
printError(format("removing corrupted link '%1%'") % linkPath);
logWarning({
.name = "Corrupted link",
.hint = hintfmt("removing corrupted link '%1%'", linkPath)
});
unlink(linkPath.c_str());
goto retry;
}
@ -229,7 +235,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (unlink(tempLink.c_str()) == -1)
printError(format("unable to unlink '%1%'") % tempLink);
logError({
.name = "Unlink error",
.hint = hintfmt("unable to unlink '%1%'", tempLink)
});
if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it
@ -238,7 +247,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
debug("'%s' has reached maximum number of links", linkPath);
return;
}
throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path);
throw SysError("cannot rename '%1%' to '%2%'", tempLink, path);
}
stats.filesLinked++;

10
src/libstore/parsed-derivations.cc
View file

@ -1,21 +1,25 @@
#include "parsed-derivations.hh"
#include <nlohmann/json.hpp>
namespace nix {
ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
: drvPath(std::move(drvPath)), drv(drv)
ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv)
: drvPath(drvPath), drv(drv)
{
/* Parse the __json attribute, if any. */
auto jsonAttr = drv.env.find("__json");
if (jsonAttr != drv.env.end()) {
try {
structuredAttrs = nlohmann::json::parse(jsonAttr->second);
structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second));
} catch (std::exception & e) {
throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what());
}
}
}
ParsedDerivation::~ParsedDerivation() { }
std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
{
if (structuredAttrs) {

12
src/libstore/parsed-derivations.hh
View file

@ -1,6 +1,6 @@
#include "derivations.hh"
#include <nlohmann/json.hpp>
#include <nlohmann/json_fwd.hpp>
namespace nix {
@ -8,15 +8,17 @@ class ParsedDerivation
{
StorePath drvPath;
BasicDerivation & drv;
std::optional<nlohmann::json> structuredAttrs;
std::unique_ptr<nlohmann::json> structuredAttrs;
public:
ParsedDerivation(StorePath && drvPath, BasicDerivation & drv);
ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv);
const std::optional<nlohmann::json> & getStructuredAttrs() const
~ParsedDerivation();
const nlohmann::json * getStructuredAttrs() const
{
return structuredAttrs;
return structuredAttrs.get();
}
std::optional<std::string> getStringAttr(const std::string & name) const;

93
src/libstore/path.cc
View file

@ -2,38 +2,38 @@
namespace nix {
extern "C" {
rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir);
rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir);
rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName);
rust::String ffi_StorePath_to_string(const StorePath & _this);
StorePath ffi_StorePath_clone(const StorePath & _this);
rust::StringSlice ffi_StorePath_name(const StorePath & _this);
MakeError(BadStorePath, Error);
static void checkName(std::string_view path, std::string_view name)
{
if (name.empty())
throw BadStorePath("store path '%s' has an empty name", path);
if (name.size() > 211)
throw BadStorePath("store path '%s' has a name longer than 211 characters", path);
for (auto c : name)
if (!((c >= '0' && c <= '9')
|| (c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z')
|| c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '='))
throw BadStorePath("store path '%s' contains illegal character '%s'", path, c);
}
StorePath StorePath::make(std::string_view path, std::string_view storeDir)
StorePath::StorePath(std::string_view _baseName)
: baseName(_baseName)
{
return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap();
if (baseName.size() < HashLen + 1)
throw BadStorePath("'%s' is too short to be a valid store path", baseName);
for (auto c : hashPart())
if (c == 'e' || c == 'o' || c == 'u' || c == 't'
|| !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')))
throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c);
checkName(baseName, name());
}
StorePath StorePath::make(unsigned char hash[20], std::string_view name)
StorePath::StorePath(const Hash & hash, std::string_view _name)
: baseName((hash.to_string(Base32, false) + "-").append(std::string(_name)))
{
return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap();
}
StorePath StorePath::fromBaseName(std::string_view baseName)
{
return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap();
}
rust::String StorePath::to_string() const
{
return ffi_StorePath_to_string(*this);
}
StorePath StorePath::clone() const
{
return ffi_StorePath_clone(*this);
checkName(baseName, name());
}
bool StorePath::isDerivation() const
@ -41,18 +41,14 @@ bool StorePath::isDerivation() const
return hasSuffix(name(), drvExtension);
}
std::string_view StorePath::name() const
{
return ffi_StorePath_name(*this);
}
StorePath StorePath::dummy(
StorePath::make(
(unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x"));
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
StorePath Store::parseStorePath(std::string_view path) const
{
return StorePath::make(path, storeDir);
auto p = canonPath(std::string(path));
if (dirOf(p) != storeDir)
throw BadStorePath("path '%s' is not in the Nix store", p);
return StorePath(baseNameOf(p));
}
std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
@ -78,9 +74,7 @@ StorePathSet Store::parseStorePathSet(const PathSet & paths) const
std::string Store::printStorePath(const StorePath & path) const
{
auto s = storeDir + "/";
s += (std::string_view) path.to_string();
return s;
return (storeDir + "/").append(path.to_string());
}
PathSet Store::printStorePathSet(const StorePathSet & paths) const
@ -90,29 +84,6 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const
return res;
}
StorePathSet cloneStorePathSet(const StorePathSet & paths)
{
StorePathSet res;
for (auto & p : paths)
res.insert(p.clone());
return res;
}
StorePathSet storePathsToSet(const StorePaths & paths)
{
StorePathSet res;
for (auto & p : paths)
res.insert(p.clone());
return res;
}
StorePathSet singleton(const StorePath & path)
{
StorePathSet res;
res.insert(path.clone());
return res;
}
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
{
size_t n = s.find("!");

79
src/libstore/path.hh
View file

@ -1,59 +1,59 @@
#pragma once
#include "rust-ffi.hh"
#include "types.hh"
namespace nix {
/* See path.rs. */
struct StorePath;
class Store;
struct Hash;
extern "C" {
void ffi_StorePath_drop(void *);
bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b);
bool ffi_StorePath_eq(const StorePath & a, const StorePath & b);
unsigned char * ffi_StorePath_hash_data(const StorePath & p);
}
struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
class StorePath
{
std::string baseName;
public:
/* Size of the hash part of store paths, in base-32 characters. */
constexpr static size_t HashLen = 32; // i.e. 160 bits
StorePath() = delete;
static StorePath make(std::string_view path, std::string_view storeDir);
StorePath(std::string_view baseName);
static StorePath make(unsigned char hash[20], std::string_view name);
StorePath(const Hash & hash, std::string_view name);
static StorePath fromBaseName(std::string_view baseName);
rust::String to_string() const;
std::string_view to_string() const
{
return baseName;
}
bool operator < (const StorePath & other) const
{
return ffi_StorePath_less_than(*this, other);
return baseName < other.baseName;
}
bool operator == (const StorePath & other) const
{
return ffi_StorePath_eq(*this, other);
return baseName == other.baseName;
}
bool operator != (const StorePath & other) const
{
return !(*this == other);
return baseName != other.baseName;
}
StorePath clone() const;
/* Check whether a file name ends with the extension for
derivations. */
bool isDerivation() const;
std::string_view name() const;
unsigned char * hashData() const
std::string_view name() const
{
return ffi_StorePath_hash_data(*this);
return std::string_view(baseName).substr(HashLen + 1);
}
std::string_view hashPart() const
{
return std::string_view(baseName).substr(0, HashLen);
}
static StorePath dummy;
@ -62,34 +62,19 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
StorePathSet cloneStorePathSet(const StorePathSet & paths);
StorePathSet storePathsToSet(const StorePaths & paths);
StorePathSet singleton(const StorePath & path);
/* Size of the hash part of store paths, in base-32 characters. */
const size_t storePathHashLen = 32; // i.e. 160 bits
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
enum struct FileIngestionMethod : uint8_t {
Flat = false,
Recursive = true
};
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
: path(path.clone()), outputs(outputs)
{ }
StorePathWithOutputs(StorePath && path, std::set<std::string> && outputs)
: path(std::move(path)), outputs(std::move(outputs))
{ }
StorePathWithOutputs(const StorePathWithOutputs & other)
: path(other.path.clone()), outputs(other.outputs)
{ }
std::string to_string(const Store & store) const;
};
@ -102,7 +87,7 @@ namespace std {
template<> struct hash<nix::StorePath> {
std::size_t operator()(const nix::StorePath & path) const noexcept
{
return * (std::size_t *) path.hashData();
return * (std::size_t *) path.to_string().data();
}
};

11
src/libstore/pathlocks.cc
View file

@ -20,7 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create)
fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
if (!fd && (create || errno != ENOENT))
throw SysError(format("opening lock file '%1%'") % path);
throw SysError("opening lock file '%1%'", path);
return fd;
}
@ -51,7 +51,7 @@ bool lockFile(int fd, LockType lockType, bool wait)
while (flock(fd, type) != 0) {
checkInterrupt();
if (errno != EINTR)
throw SysError(format("acquiring/releasing lock"));
throw SysError("acquiring/releasing lock");
else
return false;
}
@ -60,7 +60,7 @@ bool lockFile(int fd, LockType lockType, bool wait)
checkInterrupt();
if (errno == EWOULDBLOCK) return false;
if (errno != EINTR)
throw SysError(format("acquiring/releasing lock"));
throw SysError("acquiring/releasing lock");
}
}
@ -124,7 +124,7 @@ bool PathLocks::lockPaths(const PathSet & paths,
hasn't been unlinked). */
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError(format("statting lock file '%1%'") % lockPath);
throw SysError("statting lock file '%1%'", lockPath);
if (st.st_size != 0)
/* This lock file has been unlinked, so we're holding
a lock on a deleted file. This means that other
@ -160,7 +160,8 @@ void PathLocks::unlock()
if (close(i.first) == -1)
printError(
format("error (ignored): cannot close lock file on '%1%'") % i.second);
"error (ignored): cannot close lock file on '%1%'",
i.second);
debug(format("lock released on '%1%'") % i.second);
}

26
src/libstore/profiles.cc
View file

@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen)
gen.number = n;
struct stat st;
if (lstat(gen.path.c_str(), &st) != 0)
throw SysError(format("statting '%1%'") % gen.path);
throw SysError("statting '%1%'", gen.path);
gen.creationTime = st.st_mtime;
gens.push_back(gen);
}
@ -117,7 +117,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
static void removeFile(const Path & path)
{
if (remove(path.c_str()) == -1)
throw SysError(format("cannot unlink '%1%'") % path);
throw SysError("cannot unlink '%1%'", path);
}
@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
Generations gens = findGenerations(profile, curGen);
if (gensToDelete.find(curGen) != gensToDelete.end())
throw Error(format("cannot delete current generation of profile %1%'") % profile);
throw Error("cannot delete current generation of profile %1%'", profile);
for (auto & i : gens) {
if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
@ -226,7 +226,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
int days;
if (!string2Int(strDays, days) || days < 1)
throw Error(format("invalid number of days specifier '%1%'") % timeSpec);
throw Error("invalid number of days specifier '%1%'", timeSpec);
time_t oldTime = curTime - days * 24 * 3600;
@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile)
}
Path getDefaultProfile()
{
Path profileLink = getHome() + "/.nix-profile";
try {
if (!pathExists(profileLink)) {
replaceSymlink(
getuid() == 0
? settings.nixStateDir + "/profiles/default"
: fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
profileLink);
}
return absPath(readLink(profileLink), dirOf(profileLink));
} catch (Error &) {
return profileLink;
}
}
}

4
src/libstore/profiles.hh
View file

@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile);
rebuilt. */
string optimisticLockProfile(const Path & profile);
/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create
it. */
Path getDefaultProfile();
}

2
src/libstore/references.cc
View file

@ -92,7 +92,7 @@ PathSet scanForReferences(const string & path,
auto baseName = std::string(baseNameOf(i));
string::size_type pos = baseName.find('-');
if (pos == string::npos)
throw Error(format("bad reference '%1%'") % i);
throw Error("bad reference '%1%'", i);
string s = string(baseName, 0, pos);
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());

4
src/libstore/remote-fs-accessor.cc
View file

@ -19,7 +19,7 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
{
assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
}
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
@ -51,7 +51,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
throw InvalidPath("path '%1%' is not a valid store path", storePath);
auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath};

37
src/libstore/remote-store.cc
View file

@ -116,11 +116,11 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (socketPath.size() + 1 >= sizeof(addr.sun_path))
throw Error(format("socket path '%1%' is too long") % socketPath);
throw Error("socket path '%1%' is too long", socketPath);
strcpy(addr.sun_path, socketPath.c_str());
if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError(format("cannot connect to daemon at '%1%'") % socketPath);
throw SysError("cannot connect to daemon at '%1%'", socketPath);
conn->from.fd = conn->fd.get();
conn->to.fd = conn->fd.get();
@ -268,7 +268,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
StorePathSet res;
for (auto & i : paths)
if (isValidPath(i)) res.insert(i.clone());
if (isValidPath(i)) res.insert(i);
return res;
} else {
conn->to << wopQueryValidPaths;
@ -296,7 +296,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
for (auto & i : paths) {
conn->to << wopHasSubstitutes << printStorePath(i);
conn.processStderr();
if (readInt(conn->from)) res.insert(i.clone());
if (readInt(conn->from)) res.insert(i);
}
return res;
} else {
@ -329,7 +329,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
infos.insert_or_assign(i.clone(), std::move(info));
infos.insert_or_assign(i, std::move(info));
}
} else {
@ -365,14 +365,14 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
} catch (Error & e) {
// Ugly backwards compatibility hack.
if (e.msg().find("is not valid") != std::string::npos)
throw InvalidPath(e.what());
throw InvalidPath(e.info());
throw;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
bool valid; conn->from >> valid;
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
}
info = std::make_shared<ValidPathInfo>(path.clone());
info = std::make_shared<ValidPathInfo>(StorePath(path));
auto deriver = readString(conn->from);
if (deriver != "") info->deriver = parseStorePath(deriver);
info->narHash = Hash(readString(conn->from), htSHA256);
@ -396,7 +396,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
conn->to << wopQueryReferrers << printStorePath(path);
conn.processStderr();
for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
referrers.insert(i.clone());
referrers.insert(i);
}
@ -418,15 +418,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
}
PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopQueryDerivationOutputNames << printStorePath(path);
conn.processStderr();
return readStrings<PathSet>(conn->from);
}
std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
{
auto conn(getConnection());
@ -484,7 +475,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
@ -492,10 +483,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
Path srcPath(absPath(_srcPath));
conn->to << wopAddToStore << name
<< ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */
<< (recursive ? 1 : 0)
<< printHashType(hashAlgo);
conn->to
<< wopAddToStore
<< name
<< ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
<< (method == FileIngestionMethod::Recursive ? 1 : 0)
<< printHashType(hashAlgo);
try {
conn->to.written = 0;

4
src/libstore/remote-store.hh
View file

@ -51,8 +51,6 @@ public:
StorePathSet queryDerivationOutputs(const StorePath & path) override;
StringSet queryDerivationOutputNames(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@ -65,7 +63,7 @@ public:
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
StorePath addTextToStore(const string & name, const string & s,

18
src/libstore/s3-binary-cache-store.cc
View file

@ -6,7 +6,7 @@
#include "nar-info-disk-cache.hh"
#include "globals.hh"
#include "compression.hh"
#include "download.hh"
#include "filetransfer.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h>
@ -32,8 +32,10 @@ namespace nix {
struct S3Error : public Error
{
Aws::S3::S3Errors err;
S3Error(Aws::S3::S3Errors err, const FormatOrString & fs)
: Error(fs), err(err) { };
template<typename... Args>
S3Error(Aws::S3::S3Errors err, const Args & ... args)
: Error(args...), err(err) { };
};
/* Helper: given an Outcome<R, E>, return R in case of success, or
@ -109,7 +111,9 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
if (retry)
printError("AWS error '%s' (%s), will retry in %d ms",
error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries));
error.GetExceptionName(),
error.GetMessage(),
CalculateDelayBeforeNextRetry(error, attemptedRetries));
return retry;
}
};
@ -132,7 +136,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
return res;
}
S3Helper::DownloadResult S3Helper::getObject(
S3Helper::FileTransferResult S3Helper::getObject(
const std::string & bucketName, const std::string & key)
{
debug("fetching 's3://%s/%s'...", bucketName, key);
@ -146,7 +150,7 @@ S3Helper::DownloadResult S3Helper::getObject(
return Aws::New<std::stringstream>("STRINGSTREAM");
});
DownloadResult res;
FileTransferResult res;
auto now1 = std::chrono::steady_clock::now();
@ -249,7 +253,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
// If bucket listing is disabled, 404s turn into 403s
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
return false;
throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
throw Error("AWS error fetching '%s': %s", path, error.GetMessage());
}
return true;

4
src/libstore/s3.hh
View file

@ -18,13 +18,13 @@ struct S3Helper
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
struct DownloadResult
struct FileTransferResult
{
std::shared_ptr<std::string> data;
unsigned int durationMs;
};
DownloadResult getObject(
FileTransferResult getObject(
const std::string & bucketName, const std::string & key);
};

10
src/libstore/sandbox-defaults.sb
View file

@ -71,6 +71,12 @@
(literal "/dev/zero")
(subpath "/dev/fd"))
; Allow pseudo-terminals.
(allow file*
(literal "/dev/ptmx")
(regex #"^/dev/pty[a-z]+")
(regex #"^/dev/ttys[0-9]+"))
; Does nothing, but reduces build noise.
(allow file* (literal "/dev/dtracehelper"))
@ -85,3 +91,7 @@
(literal "/etc")
(literal "/var")
(literal "/private/var/tmp"))
; This is used by /bin/sh on macOS 10.15 and later.
(allow file*
(literal "/private/var/select/sh"))

32
src/libstore/sqlite.cc
View file

@ -25,11 +25,16 @@ namespace nix {
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
}
SQLite::SQLite(const Path & path)
SQLite::SQLite(const Path & path, bool create)
{
if (sqlite3_open_v2(path.c_str(), &db,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
throw Error(format("cannot open SQLite database '%s'") % path);
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error("cannot open SQLite database '%s'", path);
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
exec("pragma foreign_keys = 1");
}
SQLite::~SQLite()
@ -42,6 +47,12 @@ SQLite::~SQLite()
}
}
void SQLite::isCache()
{
exec("pragma synchronous = off");
exec("pragma main.journal_mode = truncate");
}
void SQLite::exec(const std::string & stmt)
{
retrySQLite<void>([&]() {
@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull)
{
if (notNull) {
if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
} else
bind();
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
{
if (notNull) {
@ -183,7 +204,10 @@ void handleSQLiteBusy(const SQLiteBusy & e)
if (now > lastWarned + 10) {
lastWarned = now;
printError("warning: %s", e.what());
logWarning({
.name = "Sqlite busy",
.hint = hintfmt(e.what())
});
}
/* Sleep for a while since retrying the transaction right away

12
src/libstore/sqlite.hh
View file

@ -3,10 +3,10 @@
#include <functional>
#include <string>
#include "types.hh"
#include "error.hh"
class sqlite3;
class sqlite3_stmt;
struct sqlite3;
struct sqlite3_stmt;
namespace nix {
@ -15,13 +15,16 @@ struct SQLite
{
sqlite3 * db = 0;
SQLite() { }
SQLite(const Path & path);
SQLite(const Path & path, bool create = true);
SQLite(const SQLite & from) = delete;
SQLite& operator = (const SQLite & from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
~SQLite();
operator sqlite3 * () { return db; }
/* Disable synchronous mode, set truncate journal mode. */
void isCache();
void exec(const std::string & stmt);
};
@ -52,6 +55,7 @@ struct SQLiteStmt
/* Bind the next parameter. */
Use & operator () (const std::string & value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true);
Use & bind(); // null

137
src/libstore/store-api.cc
View file

@ -6,6 +6,7 @@
#include "thread-pool.hh"
#include "json.hh"
#include "derivations.hh"
#include "url.hh"
#include <future>
@ -22,7 +23,7 @@ bool Store::isInStore(const Path & path) const
Path Store::toStorePath(const Path & path) const
{
if (!isInStore(path))
throw Error(format("path '%1%' is not in the Nix store") % path);
throw Error("path '%1%' is not in the Nix store", path);
Path::size_type slash = path.find('/', storeDir.size() + 1);
if (slash == Path::npos)
return path;
@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
throw Error(format("path '%1%' is not in the Nix store") % path);
throw NotInStore("path '%1%' is not in the Nix store", path);
return path;
}
@ -54,15 +55,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
{
auto [path2, outputs] = nix::parsePathWithOutputs(path);
return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs));
}
string storePathToHash(const Path & path)
{
auto base = baseNameOf(path);
assert(base.size() >= storePathHashLen);
return string(base, 0, storePathHashLen);
return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
}
@ -141,9 +134,9 @@ StorePath Store::makeStorePath(const string & type,
const Hash & hash, std::string_view name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name);
string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
auto h = compressHash(hashString(htSHA256, s), 20);
return StorePath::make(h.hash, name);
return StorePath(h, name);
}
@ -171,19 +164,22 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath(
bool recursive,
FileIngestionMethod recursive,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{
if (hash.type == htSHA256 && recursive) {
if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out", hashString(htSHA256,
"fixed:out:" + (recursive ? (string) "r:" : "") +
hash.to_string(Base16) + ":"), name);
return makeStorePath("output:out",
hashString(htSHA256,
"fixed:out:"
+ (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+ hash.to_string(Base16, true) + ":"),
name);
}
}
@ -200,10 +196,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const
{
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
return std::make_pair(makeFixedOutputPath(recursive, h, name), h);
Hash h = method == FileIngestionMethod::Recursive
? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath);
return std::make_pair(makeFixedOutputPath(method, h, name), h);
}
@ -237,7 +235,7 @@ bool Store::PathInfoCacheValue::isKnownNow()
bool Store::isValidPath(const StorePath & storePath)
{
auto hashPart = storePathToHash(printStorePath(storePath));
std::string hashPart(storePath.hashPart());
{
auto state_(state.lock());
@ -305,7 +303,7 @@ void Store::queryPathInfo(const StorePath & storePath,
std::string hashPart;
try {
hashPart = storePathToHash(printStorePath(storePath));
hashPart = storePath.hashPart();
{
auto res = state.lock()->pathInfoCache.get(hashPart);
@ -441,7 +439,9 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
bool includeImpureInfo, bool showClosureSize,
Base hashBase,
AllowInvalidFlag allowInvalid)
{
auto jsonList = jsonOut.list();
@ -453,7 +453,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
auto info = queryPathInfo(storePath);
jsonPath
.attr("narHash", info->narHash.to_string())
.attr("narHash", info->narHash.to_string(hashBase, true))
.attr("narSize", info->narSize);
{
@ -496,7 +496,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize)
@ -545,7 +545,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod
for (auto & path : paths) {
if (path.path.isDerivation())
unsupported("buildPaths");
paths2.insert(path.path.clone());
paths2.insert(path.path);
}
if (queryValidPaths(paths2).size() != paths2.size())
@ -685,21 +685,6 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
}
ValidPathInfo::ValidPathInfo(const ValidPathInfo & other)
: path(other.path.clone())
, deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{})
, narHash(other.narHash)
, references(cloneStorePathSet(other.references))
, registrationTime(other.registrationTime)
, narSize(other.narSize)
, id(other.id)
, ultimate(other.ultimate)
, sigs(other.sigs)
, ca(other.ca)
{
}
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven)
{
std::string path;
@ -741,12 +726,7 @@ std::string Store::showPaths(const StorePathSet & paths)
string showPaths(const PathSet & paths)
{
string s;
for (auto & i : paths) {
if (s.size() != 0) s += ", ";
s += "'" + i + "'";
}
return s;
return concatStringsSep(", ", quoteStrings(paths));
}
@ -757,7 +737,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32) + ";"
+ narHash.to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(references));
}
@ -772,11 +752,15 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
auto warn = [&]() {
printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
logWarning(
ErrorInfo{
.name = "Path not content-addressed",
.hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path))
});
};
if (hasPrefix(ca, "text:")) {
Hash hash(std::string(ca, 5));
Hash hash(ca.substr(5));
if (store.makeTextPath(path.name(), hash, references) == path)
return true;
else
@ -784,9 +768,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
}
else if (hasPrefix(ca, "fixed:")) {
bool recursive = ca.compare(6, 2, "r:") == 0;
Hash hash(std::string(ca, recursive ? 8 : 6));
auto refs = cloneStorePathSet(references);
FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6));
auto refs = references;
bool hasSelfReference = false;
if (refs.count(path)) {
hasSelfReference = true;
@ -829,27 +813,14 @@ Strings ValidPathInfo::shortRefs() const
}
std::string makeFixedOutputCA(bool recursive, const Hash & hash)
std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
{
return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
return "fixed:"
+ (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
+ hash.to_string(Base32, true);
}
void Store::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor)
{
addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor);
}
void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor)
{
StringSource source(*nar);
addToStore(info, source, repair, checkSigs, accessor);
}
}
@ -869,27 +840,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
Store::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
auto e = s.find('=');
if (e != std::string::npos) {
auto value = s.substr(e + 1);
std::string decoded;
for (size_t i = 0; i < value.size(); ) {
if (value[i] == '%') {
if (i + 2 >= value.size())
throw Error("invalid URI parameter '%s'", value);
try {
decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
i += 3;
} catch (...) {
throw Error("invalid URI parameter '%s'", value);
}
} else
decoded += value[i++];
}
params[s.substr(0, e)] = decoded;
}
}
params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};
@ -964,7 +915,7 @@ std::list<ref<Store>> getDefaultSubstituters()
try {
stores.push_back(openStore(uri));
} catch (Error & e) {
printError("warning: %s", e.what());
logWarning(e.info());
}
};

39
src/libstore/store-api.hh
View file

@ -28,6 +28,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error);
struct BasicDerivation;
@ -43,7 +44,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
/* Magic header of exportPath() output (obsolete). */
const uint32_t exportMagic = 0x4558494e;
@ -189,8 +189,9 @@ struct ValidPathInfo
Strings shortRefs() const;
ValidPathInfo(const StorePath & path) : path(path) { }
ValidPathInfo(StorePath && path) : path(std::move(path)) { }
explicit ValidPathInfo(const ValidPathInfo & other);
virtual ~ValidPathInfo() { }
};
@ -346,7 +347,7 @@ public:
StorePath makeOutputPath(const string & id,
const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(bool recursive,
StorePath makeFixedOutputPath(FileIngestionMethod method,
const Hash & hash, std::string_view name,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
@ -358,7 +359,7 @@ public:
store path to which srcPath is to be copied. Returns the store
path and the cryptographic hash of the contents of srcPath. */
std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
const Path & srcPath, bool recursive = true,
const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
/* Preparatory part of addTextToStore().
@ -430,10 +431,6 @@ public:
virtual StorePathSet queryDerivationOutputs(const StorePath & path)
{ unsupported("queryDerivationOutputs"); }
/* Query the output names of the derivation denoted by `path'. */
virtual StringSet queryDerivationOutputNames(const StorePath & path)
{ unsupported("queryDerivationOutputNames"); }
/* Query the full store path given the hash part of a valid store
path, or empty if the path doesn't exist. */
virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0;
@ -450,24 +447,19 @@ public:
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0);
// FIXME: remove
virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0);
std::shared_ptr<FSAccessor> accessor = 0) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
The function object `filter' can be used to exclude files (see
libutil/archive.hh). */
virtual StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
// FIXME: remove?
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
{
throw Error("addToStoreFromDump() is not supported by this store");
}
@ -561,6 +553,7 @@ public:
each path is included. */
void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize,
Base hashBase = Base32,
AllowInvalidFlag allowInvalid = DisallowInvalid);
/* Return the size of the closure of the specified path, that is,
@ -591,6 +584,9 @@ public:
ensurePath(). */
Derivation derivationFromPath(const StorePath & drvPath);
/* Read a derivation (which must already be valid). */
Derivation readDerivation(const StorePath & drvPath);
/* Place in `out' the set of all store paths in the file system
closure of `storePath'; that is, all paths than can be directly
or indirectly reached from it. `out' is not cleared. If
@ -676,6 +672,11 @@ public:
return storePath;
}
Path toRealPath(const StorePath & storePath)
{
return toRealPath(printStorePath(storePath));
}
virtual void createUser(const std::string & userName, uid_t userId)
{ }
@ -731,10 +732,6 @@ public:
};
/* Extract the hash part of the given store path. */
string storePathToHash(const Path & path);
/* Copy a path from one store to another. */
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
@ -844,7 +841,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(bool recursive, const Hash & hash);
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
/* Split URI into protocol+hierarchy part and its parameter set. */

2
src/libstore/worker-protocol.hh
View file

@ -36,7 +36,7 @@ typedef enum {
wopClearFailedPaths = 25,
wopQueryPathInfo = 26,
wopImportPaths = 27, // obsolete
wopQueryDerivationOutputNames = 28,
wopQueryDerivationOutputNames = 28, // obsolete
wopQueryPathFromHashPart = 29,
wopQuerySubstitutablePathInfos = 30,
wopQueryValidPaths = 31,

19
src/libutil/affinity.cc
View file

@ -12,6 +12,17 @@ namespace nix {
#if __linux__
static bool didSaveAffinity = false;
static cpu_set_t savedAffinity;
std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset)
{
auto count = CPU_COUNT(&cset);
for (int i=0; i < count; ++i)
{
os << (CPU_ISSET(i,&cset) ? "1" : "0");
}
return os;
}
#endif
@ -25,7 +36,7 @@ void setAffinityTo(int cpu)
CPU_ZERO(&newAffinity);
CPU_SET(cpu, &newAffinity);
if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
printError(format("failed to lock thread to CPU %1%") % cpu);
printError("failed to lock thread to CPU %1%", cpu);
#endif
}
@ -47,7 +58,11 @@ void restoreAffinity()
#if __linux__
if (!didSaveAffinity) return;
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
printError("failed to restore affinity %1%");
{
std::ostringstream oss;
oss << savedAffinity;
printError("failed to restore CPU affinity %1%", oss.str());
}
#endif
}

15
src/libutil/ansicolor.hh Normal file
View file

@ -0,0 +1,15 @@
#pragma once
namespace nix {
/* Some ANSI escape sequences. */
#define ANSI_NORMAL "\e[0m"
#define ANSI_BOLD "\e[1m"
#define ANSI_FAINT "\e[2m"
#define ANSI_ITALIC "\e[3m"
#define ANSI_RED "\e[31;1m"
#define ANSI_GREEN "\e[32;1m"
#define ANSI_YELLOW "\e[33;1m"
#define ANSI_BLUE "\e[34;1m"
}

Some files were not shown because too many files have changed in this diff Show more