Merge remote-tracking branch 'upstream/master' into derivation-header-include-order

src/build-remote/build-remote.cc

@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using std::cin;
@@ -200,9 +200,12 @@ static int _main(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
-                        (msg.empty() ? "" : ": " + msg));
+                    logError({
+                        .name = "Remote build",
+                        .hint = hintfmt("cannot build on '%s': %s%s",
+                            bestMachine->storeUri, e.what(),
+                            (msg.empty() ? "" : ": " + msg))
+                    });
                     bestMachine->enabled = false;
                     continue;
                 }
@@ -241,7 +244,7 @@ connected:
 
         uploadLock = -1;
 
-        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
+        auto drv = store->readDerivation(*drvPath);
         drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);

src/libexpr/attr-path.cc

@@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error(format("missing closing quote in selection path '%1%'") % s);
+                    throw Error("missing closing quote in selection path '%1%'", s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -37,9 +37,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 {
     Strings tokens = parseAttrPath(attrPath);
 
-    Error attrError =
-        Error(format("attribute selection path '%1%' does not match expression") % attrPath);
-
     Value * v = &vIn;
     Pos pos = noPos;
 
@@ -63,11 +60,11 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (v->type != tAttrs)
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a set but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a set but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attr.empty())
-                throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
+                throw Error("empty attribute name in selection path '%1%'", attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
@@ -80,9 +77,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (!v->isList())
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a list but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a list but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 

src/libexpr/attr-set.hh

@@ -76,7 +76,11 @@ public:
     {
         auto a = get(name);
         if (!a)
-            throw Error("attribute '%s' missing, at %s", name, pos);
+            throw Error({
+                .hint = hintfmt("attribute '%s' missing", name),
+                .nixCode = NixCode { .errPos = pos }
+            });
+
         return *a;
     }
 

src/libexpr/common-eval-args.cc

@@ -1,31 +1,36 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "util.hh"
 #include "eval.hh"
+#include "fetchers.hh"
+#include "store-api.hh"
 
 namespace nix {
 
 MixEvalArgs::MixEvalArgs()
 {
-    mkFlag()
-        .longName("arg")
-        .description("argument to be passed to Nix functions")
-        .labels({"name", "expr"})
-        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
+    addFlag({
+        .longName = "arg",
+        .description = "argument to be passed to Nix functions",
+        .labels = {"name", "expr"},
+        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
+    });
 
-    mkFlag()
-        .longName("argstr")
-        .description("string-valued argument to be passed to Nix functions")
-        .labels({"name", "string"})
-        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
+    addFlag({
+        .longName = "argstr",
+        .description = "string-valued argument to be passed to Nix functions",
+        .labels = {"name", "string"},
+        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
+    });
 
-    mkFlag()
-        .shortName('I')
-        .longName("include")
-        .description("add a path to the list of locations used to look up <...> file names")
-        .label("path")
-        .handler([&](std::string s) { searchPath.push_back(s); });
+    addFlag({
+        .longName = "include",
+        .shortName = 'I',
+        .description = "add a path to the list of locations used to look up <...> file names",
+        .labels = {"path"},
+        .handler = {[&](std::string s) { searchPath.push_back(s); }}
+    });
 }
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
@@ -46,9 +51,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 Path lookupFileArg(EvalState & state, string s)
 {
     if (isUri(s)) {
-        CachedDownloadRequest request(s);
-        request.unpack = true;
-        return getDownloader()->downloadCached(state.store, request).path;
+        return state.store->toRealPath(
+            fetchers::downloadTarball(
+                state.store, resolveUri(s), "source", false).storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);

src/libexpr/eval-inline.hh

@@ -7,20 +7,26 @@
 
 namespace nix {
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
-    throw EvalError(format(s) % pos);
+    throw EvalError({
+        .hint = hintfmt(s),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v));
+    throw TypeError(s, showType(v));
 }
 
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v) % pos);
+    throw TypeError({
+        .hint = hintfmt(s, showType(v)),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 
@@ -43,7 +49,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v, noPos);
     else if (v.type == tBlackhole)
-        throwEvalError("infinite recursion encountered, at %1%", pos);
+        throwEvalError(pos, "infinite recursion encountered");
 }
 
 
@@ -57,9 +63,9 @@ inline void EvalState::forceAttrs(Value & v)
 
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tAttrs)
-        throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a set was expected", v);
 }
 
 
@@ -73,9 +79,9 @@ inline void EvalState::forceList(Value & v)
 
 inline void EvalState::forceList(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (!v.isList())
-        throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a list was expected", v);
 }
 
 /* Note: Various places expect the allocated memory to be zeroed. */

src/libexpr/eval.cc

@@ -5,7 +5,7 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "eval-inline.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "json.hh"
 #include "function-trace.hh"
 
@@ -22,6 +22,8 @@
 
 #if HAVE_BOEHMGC
 
+#define GC_INCLUDE_NEW
+
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
@@ -56,6 +58,12 @@ static char * dupStringWithLen(const char * s, size_t size)
 }
 
 
+RootValue allocRootValue(Value * v)
+{
+    return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
+}
+
+
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
@@ -493,52 +501,74 @@ Value & EvalState::getBuiltin(const string & name)
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
 {
-    throw EvalError(format(s) % s2);
+    throw EvalError(s, s2);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
 {
-    throw EvalError(format(s) % s2 % pos);
+    throw EvalError({
+        .hint = hintfmt(s, s2),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(format(s) % s2 % s3);
+    throw EvalError(s, s2, s3);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(format(s) % s2 % s3 % pos);
+    throw EvalError({
+        .hint = hintfmt(s, s2, s3),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2))
 {
-    throw EvalError(format(s) % sym % p1 % p2);
+    // p1 is where the error occurred; p2 is a position mentioned in the message.
+    throw EvalError({
+        .hint = hintfmt(s, sym, p2),
+        .nixCode = NixCode { .errPos = p1 }
+    });
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
-    throw TypeError(format(s) % pos);
+    throw TypeError({
+        .hint = hintfmt(s),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
 {
-    throw TypeError(format(s) % s1);
+    throw TypeError(s, s1);
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
 {
-    throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
+    throw TypeError({
+        .hint = hintfmt(s, fun.showNamePos(), s2),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
+LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
 {
-    throw AssertionError(format(s) % s1 % pos);
+    throw AssertionError({
+        .hint = hintfmt(s, s1),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
-LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
+LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
 {
-    throw UndefinedVarError(format(s) % s1 % pos);
+    throw UndefinedVarError({
+        .hint = hintfmt(s, s1),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
@@ -606,7 +636,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
             return j->value;
         }
         if (!env->prevWith)
-            throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
+            throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name);
         for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
@@ -804,7 +834,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
     Value v;
     e->eval(*this, env, v);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
 
@@ -918,7 +948,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Symbol nameSym = state.symbols.create(nameVal.string.s);
         Bindings::iterator j = v.attrs->find(nameSym);
         if (j != v.attrs->end())
-            throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos);
+            throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos);
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
@@ -1006,7 +1036,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             } else {
                 state.forceAttrs(*vAttrs, pos);
                 if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                    throwEvalError("attribute '%1%' missing, at %2%", name, pos);
+                    throwEvalError(pos, "attribute '%1%' missing", name);
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -1132,7 +1162,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
     }
 
     if (fun.type != tLambda)
-        throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos);
+        throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
 
     ExprLambda & lambda(*fun.lambda.fun);
 
@@ -1160,8 +1190,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%",
-                    lambda, i.name, pos);
+                if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
+                    lambda, i.name);
                 env2.values[displ++] = i.def->maybeThunk(*this, env2);
             } else {
                 attrsUsed++;
@@ -1176,7 +1206,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
                user. */
             for (auto & i : *arg.attrs)
                 if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos);
+                    throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
             abort(); // can't happen
         }
     }
@@ -1256,7 +1286,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 
 void ExprIf::eval(EvalState & state, Env & env, Value & v)
 {
-    (state.evalBool(env, cond) ? then : else_)->eval(state, env, v);
+    (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v);
 }
 
 
@@ -1265,7 +1295,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos)) {
         std::ostringstream out;
         cond->show(out);
-        throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
+        throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
     }
     body->eval(state, env, v);
 }
@@ -1417,14 +1447,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos);
+                throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
         } else if (firstType == tFloat) {
             if (vTmp.type == tInt) {
                 nf += vTmp.integer;
             } else if (vTmp.type == tFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos);
+                throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
         } else
             s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
     }
@@ -1435,7 +1465,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         mkFloat(v, nf);
     else if (firstType == tPath) {
         if (!context.empty())
-            throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos);
+            throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
         auto path = canonPath(s.str());
         mkPath(v, path.c_str());
     } else
@@ -1484,7 +1514,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tInt)
-        throwTypeError("value is %1% while an integer was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while an integer was expected", v);
     return v.integer;
 }
 
@@ -1495,16 +1525,16 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
     if (v.type == tInt)
         return v.integer;
     else if (v.type != tFloat)
-        throwTypeError("value is %1% while a float was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a float was expected", v);
     return v.fpoint;
 }
 
 
 bool EvalState::forceBool(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
 
@@ -1517,9 +1547,9 @@ bool EvalState::isFunctor(Value & fun)
 
 void EvalState::forceFunction(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
-        throwTypeError("value is %1% while a function was expected, at %2%", v, pos);
+        throwTypeError(pos, "value is %1% while a function was expected", v);
 }
 
 
@@ -1528,7 +1558,7 @@ string EvalState::forceString(Value & v, const Pos & pos)
     forceValue(v, pos);
     if (v.type != tString) {
         if (pos)
-            throwTypeError("value is %1% while a string was expected, at %2%", v, pos);
+            throwTypeError(pos, "value is %1% while a string was expected", v);
         else
             throwTypeError("value is %1% while a string was expected", v);
     }
@@ -1557,8 +1587,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
     string s = forceString(v, pos);
     if (v.string.context) {
         if (pos)
-            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%",
-                v.string.s, v.string.context[0], pos);
+            throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
+                v.string.s, v.string.context[0]);
         else
             throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
                 v.string.s, v.string.context[0]);
@@ -1594,7 +1624,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
 string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     bool coerceMore, bool copyToStore)
 {
-    forceValue(v);
+    forceValue(v, pos);
 
     string s;
 
@@ -1614,7 +1644,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
             return *maybeString;
         }
         auto i = v.attrs->find(sOutPath);
-        if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos);
+        if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
@@ -1645,7 +1675,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         }
     }
 
-    throwTypeError("cannot coerce %1% to a string, at %2%", v, pos);
+    throwTypeError(pos, "cannot coerce %1% to a string", v);
 }
 
 
@@ -1661,7 +1691,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
     else {
         auto p = settings.readOnlyMode
             ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
-            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
+            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
         dstPath = store->printStorePath(p);
         srcToStore.insert_or_assign(path, std::move(p));
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
@@ -1676,7 +1706,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
 {
     string path = coerceToString(pos, v, context, false, false);
     if (path == "" || path[0] != '/')
-        throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos);
+        throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path);
     return path;
 }
 
@@ -1883,8 +1913,10 @@ void EvalState::printStats()
 
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
-    throw TypeError(format("cannot coerce %1% to a string, at %2%") %
-        showType() % pos);
+    throw TypeError({
+        .hint = hintfmt("cannot coerce %1% to a string", showType()),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 

src/libexpr/json-to-value.cc

@@ -4,7 +4,6 @@
 #include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
-using std::unique_ptr;
 
 namespace nix {
 
@@ -13,69 +12,69 @@ namespace nix {
 class JSONSax : nlohmann::json_sax<json> {
     class JSONState {
     protected:
-        unique_ptr<JSONState> parent;
-        Value * v;
+        std::unique_ptr<JSONState> parent;
+        RootValue v;
     public:
-        virtual unique_ptr<JSONState> resolve(EvalState &)
+        virtual std::unique_ptr<JSONState> resolve(EvalState &)
         {
             throw std::logic_error("tried to close toplevel json parser state");
-        };
-        explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
-        explicit JSONState(Value* v) : v(v) {};
-        JSONState(JSONState& p) = delete;
-        Value& value(EvalState & state)
+        }
+        explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {}
+        explicit JSONState(Value * v) : v(allocRootValue(v)) {}
+        JSONState(JSONState & p) = delete;
+        Value & value(EvalState & state)
         {
-            if (v == nullptr)
-                v = state.allocValue();
-            return *v;
-        };
-        virtual ~JSONState() {};
-        virtual void add() {};
+            if (!v)
+                v = allocRootValue(state.allocValue());
+            return **v;
+        }
+        virtual ~JSONState() {}
+        virtual void add() {}
     };
 
     class JSONObjectState : public JSONState {
         using JSONState::JSONState;
-        ValueMap attrs = ValueMap();
-        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueMap attrs;
+        std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value& v = parent->value(state);
+            Value & v = parent->value(state);
             state.mkAttrs(v, attrs.size());
             for (auto & i : attrs)
                 v.attrs->push_back(Attr(i.first, i.second));
             return std::move(parent);
         }
-        virtual void add() override { v = nullptr; };
+        void add() override { v = nullptr; }
     public:
-        void key(string_t& name, EvalState & state)
+        void key(string_t & name, EvalState & state)
         {
-            attrs[state.symbols.create(name)] = &value(state);
+            attrs.insert_or_assign(state.symbols.create(name), &value(state));
         }
     };
 
     class JSONListState : public JSONState {
-        ValueVector values = ValueVector();
-        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueVector values;
+        std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value& v = parent->value(state);
+            Value & v = parent->value(state);
             state.mkList(v, values.size());
             for (size_t n = 0; n < values.size(); ++n) {
                 v.listElems()[n] = values[n];
             }
             return std::move(parent);
         }
-        virtual void add() override {
-            values.push_back(v);
+        void add() override {
+            values.push_back(*v);
             v = nullptr;
-        };
+        }
     public:
-        JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
+        JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
         {
             values.reserve(reserve);
         }
     };
 
     EvalState & state;
-    unique_ptr<JSONState> rs;
+    std::unique_ptr<JSONState> rs;
 
     template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
     {
@@ -107,12 +106,12 @@ public:
         return handle_value(mkInt, val);
     }
 
-    bool number_float(number_float_t val, const string_t& s)
+    bool number_float(number_float_t val, const string_t & s)
     {
         return handle_value(mkFloat, val);
     }
 
-    bool string(string_t& val)
+    bool string(string_t & val)
     {
         return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
@@ -123,7 +122,7 @@ public:
         return true;
     }
 
-    bool key(string_t& name)
+    bool key(string_t & name)
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;

src/libexpr/lexer.l

@@ -127,14 +127,14 @@ or          { return OR_KW; }
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError(format("invalid integer '%1%'") % yytext);
+                  throw ParseError("invalid integer '%1%'", yytext);
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError(format("invalid float '%1%'") % yytext);
+                  throw ParseError("invalid float '%1%'", yytext);
               return FLOAT;
             }
 
@@ -219,4 +219,3 @@ or          { return OR_KW; }
             }
 
 %%
-

src/libexpr/local.mk

@@ -6,7 +6,9 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore libnixrust
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+
+libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)

src/libexpr/nixexpr.cc

@@ -267,8 +267,11 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
-
+    if (withLevel == -1)
+        throw UndefinedVarError({
+            .hint = hintfmt("undefined variable '%1%'", name),
+            .nixCode = NixCode { .errPos = pos }
+        });
     fromWith = true;
     this->level = withLevel;
 }

src/libexpr/nixexpr.hh

@@ -2,6 +2,7 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
+#include "error.hh"
 
 #include <map>
 
@@ -209,9 +210,10 @@ struct ExprList : Expr
 
 struct Formal
 {
+    Pos pos;
     Symbol name;
     Expr * def;
-    Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
+    Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { };
 };
 
 struct Formals
@@ -234,8 +236,10 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-                % arg % pos);
+            throw ParseError({
+                .hint = hintfmt("duplicate formal function argument '%1%'", arg),
+                .nixCode = NixCode { .errPos = pos }
+            });
     };
     void setName(Symbol & name);
     string showNamePos() const;
@@ -261,8 +265,9 @@ struct ExprWith : Expr
 
 struct ExprIf : Expr
 {
+    Pos pos;
     Expr * cond, * then, * else_;
-    ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
+    ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -31,7 +31,7 @@ namespace nix {
         Expr * result;
         Path basePath;
         Symbol path;
-        string error;
+        ErrorInfo error;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
@@ -64,15 +64,20 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % showAttrPath(attrPath) % pos % prevPos);
+    throw ParseError({
+        .hint = hintfmt("attribute '%1%' already defined at %2%",
+            showAttrPath(attrPath), prevPos),
+        .nixCode = NixCode { .errPos = pos },
+    });
 }
 
 
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % attr % pos % prevPos);
+    throw ParseError({
+        .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
+        .nixCode = NixCode { .errPos = pos },
+    });
 }
 
 
@@ -140,8 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
-        throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-            % formal.name % pos);
+        throw ParseError({
+            .hint = hintfmt("duplicate formal function argument '%1%'",
+                formal.name),
+            .nixCode = NixCode { .errPos = pos },
+        });
     formals->formals.push_front(formal);
 }
 
@@ -249,8 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = (format("%1%, at %2%")
-        % error % makeCurPos(*loc, data)).str();
+    data->error = {
+        .hint = hintfmt(error),
+        .nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
+    };
 }
 
 
@@ -327,15 +337,17 @@ expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
-        throw ParseError(format("dynamic attributes not allowed in let at %1%")
-            % CUR_POS);
+        throw ParseError({
+            .hint = hintfmt("dynamic attributes not allowed in let"),
+            .nixCode = NixCode { .errPos = CUR_POS },
+        });
       $$ = new ExprLet($2, $4);
     }
   | expr_if
   ;
 
 expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
+  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
   | expr_op
   ;
 
@@ -405,7 +417,10 @@ expr_simple
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
       if (noURLLiterals)
-          throw ParseError("URL literals are disabled, at %s", CUR_POS);
+          throw ParseError({
+              .hint = hintfmt("URL literals are disabled"),
+              .nixCode = NixCode { .errPos = CUR_POS }
+          });
       $$ = new ExprString(data->symbols.create($1));
   }
   | '(' expr ')' { $$ = $2; }
@@ -475,8 +490,10 @@ attrs
           $$->push_back(AttrName(str->s));
           delete str;
       } else
-          throw ParseError(format("dynamic attributes not allowed in inherit at %1%")
-              % makeCurPos(@2, data));
+          throw ParseError({
+              .hint = hintfmt("dynamic attributes not allowed in inherit"),
+              .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
+          });
     }
   | { $$ = new AttrPath; }
   ;
@@ -531,8 +548,8 @@ formals
   ;
 
 formal
-  : ID { $$ = new Formal(data->symbols.create($1), 0); }
-  | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
+  : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); }
+  | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); }
   ;
 
 %%
@@ -544,7 +561,8 @@ formal
 #include <unistd.h>
 
 #include "eval.hh"
-#include "download.hh"
+#include "filetransfer.hh"
+#include "fetchers.hh"
 #include "store-api.hh"
 
 
@@ -670,11 +688,13 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-    format f = format(
-        "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
-        + string(pos ? ", at %2%" : ""));
-    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    throw ThrownError(f % path % pos);
+    throw ThrownError({
+        .hint = hintfmt(evalSettings.pureEval
+            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
+            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
+            path),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 
@@ -687,11 +707,13 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            CachedDownloadRequest request(elem.second);
-            request.unpack = true;
-            res = { true, getDownloader()->downloadCached(store, request).path };
-        } catch (DownloadError & e) {
-            printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
+            res = { true, store->toRealPath(fetchers::downloadTarball(
+                        store, resolveUri(elem.second), "source", false).storePath) };
+        } catch (FileTransferError & e) {
+            logWarning({
+                .name = "Entry download",
+                .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     } else {
@@ -699,7 +721,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second);
+            logWarning({
+                .name = "Entry not found",
+                .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     }

src/libexpr/primops.hh

@@ -20,6 +20,7 @@ struct RegisterPrimOp
    them. */
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
 /* Execute a program and parse its output */
 void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);
 

src/libexpr/primops/context.cc

@@ -146,7 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (auto & i : *args[1]->attrs) {
         if (!state.store->isStorePath(i.name))
-            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+            throw EvalError({
+                .hint = hintfmt("Context key '%s' is not a store path", i.name),
+                .nixCode = NixCode { .errPos = *i.pos }
+            });
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
@@ -160,7 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                    throw EvalError({
+                        .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
+                        .nixCode = NixCode { .errPos = *i.pos }
+                    });
                 }
                 context.insert("=" + string(i.name));
             }
@@ -170,7 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                throw EvalError({
+                    .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
+                    .nixCode = NixCode { .errPos = *i.pos }
+                });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

src/libexpr/primops/fetchGit.cc

@@ -1,203 +1,19 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "download.hh"
 #include "store-api.hh"
-#include "pathlocks.hh"
 #include "hash.hh"
-#include "tarfile.hh"
-
-#include <sys/time.h>
-
-#include <regex>
-
-#include <nlohmann/json.hpp>
-
-using namespace std::string_literals;
+#include "fetchers.hh"
+#include "url.hh"
 
 namespace nix {
 
-struct GitInfo
-{
-    Path storePath;
-    std::string rev;
-    std::string shortRev;
-    uint64_t revCount = 0;
-};
-
-std::regex revRegex("^[0-9a-fA-F]{40}$");
-
-GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::optional<std::string> ref, std::string rev,
-    const std::string & name)
-{
-    if (evalSettings.pureEval && rev == "")
-        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
-    if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
-
-        bool clean = true;
-
-        try {
-            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
-        } catch (ExecError & e) {
-            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-            clean = false;
-        }
-
-        if (!clean) {
-
-            /* This is an unclean working tree. So copy all tracked files. */
-            GitInfo gitInfo;
-            gitInfo.rev = "0000000000000000000000000000000000000000";
-            gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
-
-            auto files = tokenizeString<std::set<std::string>>(
-                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
-
-            PathFilter filter = [&](const Path & p) -> bool {
-                assert(hasPrefix(p, uri));
-                std::string file(p, uri.size() + 1);
-
-                auto st = lstat(p);
-
-                if (S_ISDIR(st.st_mode)) {
-                    auto prefix = file + "/";
-                    auto i = files.lower_bound(prefix);
-                    return i != files.end() && hasPrefix(*i, prefix);
-                }
-
-                return files.count(file);
-            };
-
-            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
-
-            return gitInfo;
-        }
-
-        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
-        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
-        ref = "HEAD"s;
-    }
-
-    if (!ref) ref = "HEAD"s;
-
-    if (rev != "" && !std::regex_match(rev, revRegex))
-        throw Error("invalid Git revision '%s'", rev);
-
-    deletePath(getCacheDir() + "/nix/git");
-
-    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
-
-    if (!pathExists(cacheDir)) {
-        createDirs(dirOf(cacheDir));
-        runProgram("git", true, { "init", "--bare", cacheDir });
-    }
-
-    Path localRefFile;
-    if (ref->compare(0, 5, "refs/") == 0)
-        localRefFile = cacheDir + "/" + *ref;
-    else
-        localRefFile = cacheDir + "/refs/heads/" + *ref;
-
-    bool doFetch;
-    time_t now = time(0);
-    /* If a rev was specified, we need to fetch if it's not in the
-       repo. */
-    if (rev != "") {
-        try {
-            runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
-            doFetch = false;
-        } catch (ExecError & e) {
-            if (WIFEXITED(e.status)) {
-                doFetch = true;
-            } else {
-                throw;
-            }
-        }
-    } else {
-        /* If the local ref is older than ‘tarball-ttl’ seconds, do a
-           git fetch to update the local ref to the remote ref. */
-        struct stat st;
-        doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
-    }
-    if (doFetch)
-    {
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
-
-        // FIXME: git stderr messes up our progress indicator, so
-        // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
-
-        struct timeval times[2];
-        times[0].tv_sec = now;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = now;
-        times[1].tv_usec = 0;
-
-        utimes(localRefFile.c_str(), times);
-    }
-
-    // FIXME: check whether rev is an ancestor of ref.
-    GitInfo gitInfo;
-    gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
-    gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
-
-    printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
-
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
-    Path storeLink = cacheDir + "/" + storeLinkName + ".link";
-    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
-
-    try {
-        auto json = nlohmann::json::parse(readFile(storeLink));
-
-        assert(json["name"] == name && json["rev"] == gitInfo.rev);
-
-        gitInfo.storePath = json["storePath"];
-
-        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
-            gitInfo.revCount = json["revCount"];
-            return gitInfo;
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    auto source = sinkToSource([&](Sink & sink) {
-        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
-        gitOptions.standardOut = &sink;
-        runProgram2(gitOptions);
-    });
-
-    Path tmpDir = createTempDir();
-    AutoDelete delTmpDir(tmpDir, true);
-
-    unpackTarfile(*source, tmpDir);
-
-    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
-
-    gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
-
-    nlohmann::json json;
-    json["storePath"] = gitInfo.storePath;
-    json["uri"] = uri;
-    json["name"] = name;
-    json["rev"] = gitInfo.rev;
-    json["revCount"] = gitInfo.revCount;
-
-    writeFile(storeLink, json.dump());
-
-    return gitInfo;
-}
-
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
     std::optional<std::string> ref;
-    std::string rev;
+    std::optional<Hash> rev;
     std::string name = "source";
+    bool fetchSubmodules = false;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -213,15 +29,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "ref")
                 ref = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "rev")
-                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+                rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "submodules")
+                fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
+                    .nixCode = NixCode { .errPos = *attr.pos }
+                });
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
@@ -230,17 +54,36 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto gitInfo = exportGit(state.store, url, ref, rev, name);
+    if (evalSettings.pureEval && !rev)
+        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
+
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "git");
+    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    if (ref) attrs.insert_or_assign("ref", *ref);
+    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
+    if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
+    auto input = fetchers::inputFromAttrs(attrs);
+
+    // FIXME: use name?
+    auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);
-    mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
+    auto storePath = state.store->printStorePath(tree.storePath);
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    // Backward compatibility: set 'rev' to
+    // 0000000000000000000000000000000000000000 for a dirty tree.
+    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
+    // Backward compatibility: set 'revCount' to 0 for a dirty tree.
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
+        tree.info.revCount.value_or(0));
+    mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
+        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

src/libexpr/primops/fetchMercurial.cc

@@ -1,174 +1,18 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "download.hh"
 #include "store-api.hh"
-#include "pathlocks.hh"
-
-#include <sys/time.h>
+#include "fetchers.hh"
+#include "url.hh"
 
 #include <regex>
 
-#include <nlohmann/json.hpp>
-
-using namespace std::string_literals;
-
 namespace nix {
 
-struct HgInfo
-{
-    Path storePath;
-    std::string branch;
-    std::string rev;
-    uint64_t revCount = 0;
-};
-
-std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
-
-HgInfo exportMercurial(ref<Store> store, const std::string & uri,
-    std::string rev, const std::string & name)
-{
-    if (evalSettings.pureEval && rev == "")
-        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
-
-    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
-
-        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
-
-        if (!clean) {
-
-            /* This is an unclean working tree. So copy all tracked
-               files. */
-
-            printTalkative("copying unclean Mercurial working tree '%s'", uri);
-
-            HgInfo hgInfo;
-            hgInfo.rev = "0000000000000000000000000000000000000000";
-            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
-
-            auto files = tokenizeString<std::set<std::string>>(
-                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
-
-            PathFilter filter = [&](const Path & p) -> bool {
-                assert(hasPrefix(p, uri));
-                std::string file(p, uri.size() + 1);
-
-                auto st = lstat(p);
-
-                if (S_ISDIR(st.st_mode)) {
-                    auto prefix = file + "/";
-                    auto i = files.lower_bound(prefix);
-                    return i != files.end() && hasPrefix(*i, prefix);
-                }
-
-                return files.count(file);
-            };
-
-            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
-
-            return hgInfo;
-        }
-    }
-
-    if (rev == "") rev = "default";
-
-    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
-
-    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
-
-    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
-       do so now. */
-    time_t now = time(0);
-    struct stat st;
-    if (stat(stampFile.c_str(), &st) != 0 ||
-        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
-    {
-        /* Except that if this is a commit hash that we already have,
-           we don't have to pull again. */
-        if (!(std::regex_match(rev, commitHashRegex)
-                && pathExists(cacheDir)
-                && runProgram(
-                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
-                    .killStderr(true)).second == "1"))
-        {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
-
-            if (pathExists(cacheDir)) {
-                try {
-                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                }
-                catch (ExecError & e) {
-                    string transJournal = cacheDir + "/.hg/store/journal";
-                    /* hg throws "abandoned transaction" error only if this file exists */
-                    if (pathExists(transJournal)) {
-                        runProgram("hg", true, { "recover", "-R", cacheDir });
-                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                    } else {
-                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
-                    }
-                }
-            } else {
-                createDirs(dirOf(cacheDir));
-                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
-            }
-        }
-
-        writeFile(stampFile, "");
-    }
-
-    auto tokens = tokenizeString<std::vector<std::string>>(
-        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
-    assert(tokens.size() == 3);
-
-    HgInfo hgInfo;
-    hgInfo.rev = tokens[0];
-    hgInfo.revCount = std::stoull(tokens[1]);
-    hgInfo.branch = tokens[2];
-
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
-    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
-
-    try {
-        auto json = nlohmann::json::parse(readFile(storeLink));
-
-        assert(json["name"] == name && json["rev"] == hgInfo.rev);
-
-        hgInfo.storePath = json["storePath"];
-
-        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
-            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
-            return hgInfo;
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    Path tmpDir = createTempDir();
-    AutoDelete delTmpDir(tmpDir, true);
-
-    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
-
-    deletePath(tmpDir + "/.hg_archival.txt");
-
-    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
-
-    nlohmann::json json;
-    json["storePath"] = hgInfo.storePath;
-    json["uri"] = uri;
-    json["name"] = name;
-    json["branch"] = hgInfo.branch;
-    json["rev"] = hgInfo.rev;
-    json["revCount"] = hgInfo.revCount;
-
-    writeFile(storeLink, json.dump());
-
-    return hgInfo;
-}
-
 static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::string rev;
+    std::optional<Hash> rev;
+    std::optional<std::string> ref;
     std::string name = "source";
     PathSet context;
 
@@ -182,16 +26,29 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             string n(attr.name);
             if (n == "url")
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-            else if (n == "rev")
-                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "rev") {
+                // Ugly: unlike fetchGit, here the "rev" attribute can
+                // be both a revision or a branch/tag name.
+                auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
+                if (std::regex_match(value, revRegex))
+                    rev = Hash(value, htSHA1);
+                else
+                    ref = value;
+            }
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
+                    .nixCode = NixCode { .errPos = *attr.pos }
+                });
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
@@ -200,18 +57,35 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto hgInfo = exportMercurial(state.store, url, rev, name);
+    if (evalSettings.pureEval && !rev)
+        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
+
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "hg");
+    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    if (ref) attrs.insert_or_assign("ref", *ref);
+    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
+    auto input = fetchers::inputFromAttrs(attrs);
+
+    // FIXME: use name
+    auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);
-    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
-    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
+    auto storePath = state.store->printStorePath(tree.storePath);
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    if (input2->getRef())
+        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
+    // Backward compatibility: set 'rev' to
+    // 0000000000000000000000000000000000000000 for a dirty tree.
+    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
+    if (tree.info.revCount)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
+        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fetchTree.cc

@@ -0,0 +1,172 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+#include "fetchers.hh"
+#include "filetransfer.hh"
+
+#include <ctime>
+#include <iomanip>
+
+namespace nix {
+
+void emitTreeAttrs(
+    EvalState & state,
+    const fetchers::Tree & tree,
+    std::shared_ptr<const fetchers::Input> input,
+    Value & v)
+{
+    state.mkAttrs(v, 8);
+
+    auto storePath = state.store->printStorePath(tree.storePath);
+
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+
+    assert(tree.info.narHash);
+    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
+        tree.info.narHash.to_string(SRI, true));
+
+    if (input->getRev()) {
+        mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
+        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
+    }
+
+    if (tree.info.revCount)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+
+    if (tree.info.lastModified)
+        mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
+            fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
+
+    v.attrs->sort();
+}
+
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    settings.requireExperimentalFeature("flakes");
+
+    std::shared_ptr<const fetchers::Input> input;
+    PathSet context;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+        state.forceAttrs(*args[0], pos);
+
+        fetchers::Attrs attrs;
+
+        for (auto & attr : *args[0]->attrs) {
+            state.forceValue(*attr.value);
+            if (attr.value->type == tString)
+                attrs.emplace(attr.name, attr.value->string.s);
+            else if (attr.value->type == tBool)
+                attrs.emplace(attr.name, attr.value->boolean);
+            else
+                throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
+                    attr.name, showType(*attr.value));
+        }
+
+        if (!attrs.count("type"))
+            throw Error({
+                .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .nixCode = NixCode { .errPos = pos }
+            });
+
+        input = fetchers::inputFromAttrs(attrs);
+    } else
+        input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
+
+    if (evalSettings.pureEval && !input->isImmutable())
+        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
+
+    // FIXME: use fetchOrSubstituteTree
+    auto [tree, input2] = input->fetchTree(state.store);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(tree.actualPath);
+
+    emitTreeAttrs(state, tree, input2, v);
+}
+
+static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
+
+static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
+    const string & who, bool unpack, std::string name)
+{
+    std::optional<std::string> url;
+    std::optional<Hash> expectedHash;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+
+        state.forceAttrs(*args[0], pos);
+
+        for (auto & attr : *args[0]->attrs) {
+            string n(attr.name);
+            if (n == "url")
+                url = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "sha256")
+                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+            else if (n == "name")
+                name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
+                    .nixCode = NixCode { .errPos = *attr.pos }
+                });
+            }
+
+        if (!url)
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
+    } else
+        url = state.forceStringNoCtx(*args[0], pos);
+
+    url = resolveUri(*url);
+
+    state.checkURI(*url);
+
+    if (name == "")
+        name = baseNameOf(*url);
+
+    if (evalSettings.pureEval && !expectedHash)
+        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
+
+    auto storePath =
+        unpack
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
+
+    auto path = state.store->toRealPath(storePath);
+
+    if (expectedHash) {
+        auto hash = unpack
+            ? state.store->queryPathInfo(storePath)->narHash
+            : hashFile(htSHA256, path);
+        if (hash != *expectedHash)
+            throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
+                *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
+    }
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(path);
+
+    mkString(v, path, PathSet({path}));
+}
+
+static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchurl", false, "");
+}
+
+static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchTarball", true, "source");
+}
+
+static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
+static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+
+}

src/libexpr/primops/fromTOML.cc

@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "cpptoml/cpptoml.h"
+#include "../../cpptoml/cpptoml.h"
 
 namespace nix {
 
@@ -81,7 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         visit(v, parser(tomlStream).parse());
     } catch (std::runtime_error & e) {
-        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
+        throw EvalError({
+            .hint = hintfmt("while parsing a TOML string: %s", e.what()),
+            .nixCode = NixCode { .errPos = pos }
+        });
     }
 }
 

src/libexpr/value-to-json.cc

@@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         default:
-            throw TypeError(format("cannot convert %1% to JSON") % showType(v));
+            throw TypeError("cannot convert %1% to JSON", showType(v));
     }
 }
 
@@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict,
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
     JSONPlaceholder & out, PathSet & context) const
 {
-    throw TypeError(format("cannot convert %1% to JSON") % showType());
+    throw TypeError("cannot convert %1% to JSON", showType());
 }
 
 

src/libexpr/value.hh

@@ -253,12 +253,17 @@ void mkPath(Value & v, const char * s);
 
 
 #if HAVE_BOEHMGC
-typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
-typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
+typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
+typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
 #else
 typedef std::vector<Value *> ValueVector;
 typedef std::map<Symbol, Value *> ValueMap;
 #endif
 
 
+/* A value allocated in traceable memory. */
+typedef std::shared_ptr<Value *> RootValue;
+
+RootValue allocRootValue(Value * v);
+
 }

src/libfetchers/attrs.cc

@@ -0,0 +1,107 @@
+#include "attrs.hh"
+#include "fetchers.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+Attrs jsonToAttrs(const nlohmann::json & json)
+{
+    Attrs attrs;
+
+    for (auto & i : json.items()) {
+        if (i.value().is_number())
+            attrs.emplace(i.key(), i.value().get<int64_t>());
+        else if (i.value().is_string())
+            attrs.emplace(i.key(), i.value().get<std::string>());
+        else if (i.value().is_boolean())
+            attrs.emplace(i.key(), i.value().get<bool>());
+        else
+            throw Error("unsupported input attribute type in lock file");
+    }
+
+    return attrs;
+}
+
+nlohmann::json attrsToJson(const Attrs & attrs)
+{
+    nlohmann::json json;
+    for (auto & attr : attrs) {
+        if (auto v = std::get_if<int64_t>(&attr.second)) {
+            json[attr.first] = *v;
+        } else if (auto v = std::get_if<std::string>(&attr.second)) {
+            json[attr.first] = *v;
+        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
+            json[attr.first] = v->t;
+        } else abort();
+    }
+    return json;
+}
+
+std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<std::string>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
+}
+
+std::string getStrAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetStrAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<int64_t>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not an integer", name);
+}
+
+int64_t getIntAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetIntAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<int64_t>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not a Boolean", name);
+}
+
+bool getBoolAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetBoolAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
+{
+    std::map<std::string, std::string> query;
+    for (auto & attr : attrs) {
+        if (auto v = std::get_if<int64_t>(&attr.second)) {
+            query.insert_or_assign(attr.first, fmt("%d", *v));
+        } else if (auto v = std::get_if<std::string>(&attr.second)) {
+            query.insert_or_assign(attr.first, *v);
+        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
+            query.insert_or_assign(attr.first, v->t ? "1" : "0");
+        } else abort();
+    }
+    return query;
+}
+
+}

src/libfetchers/attrs.hh

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "types.hh"
+
+#include <variant>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix::fetchers {
+
+/* Wrap bools to prevent string literals (i.e. 'char *') from being
+   cast to a bool in Attr. */
+template<typename T>
+struct Explicit {
+    T t;
+};
+
+typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
+typedef std::map<std::string, Attr> Attrs;
+
+Attrs jsonToAttrs(const nlohmann::json & json);
+
+nlohmann::json attrsToJson(const Attrs & attrs);
+
+std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
+
+std::string getStrAttr(const Attrs & attrs, const std::string & name);
+
+std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
+
+int64_t getIntAttr(const Attrs & attrs, const std::string & name);
+
+std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
+
+bool getBoolAttr(const Attrs & attrs, const std::string & name);
+
+std::map<std::string, std::string> attrsToQuery(const Attrs & attrs);
+
+}

src/libfetchers/cache.cc

@@ -0,0 +1,121 @@
+#include "cache.hh"
+#include "sqlite.hh"
+#include "sync.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+static const char * schema = R"sql(
+
+create table if not exists Cache (
+    input     text not null,
+    info      text not null,
+    path      text not null,
+    immutable integer not null,
+    timestamp integer not null,
+    primary key (input)
+);
+)sql";
+
+struct CacheImpl : Cache
+{
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt add, lookup;
+    };
+
+    Sync<State> _state;
+
+    CacheImpl()
+    {
+        auto state(_state.lock());
+
+        auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
+        createDirs(dirOf(dbPath));
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->add.create(state->db,
+            "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
+
+        state->lookup.create(state->db,
+            "select info, path, immutable, timestamp from Cache where input = ?");
+    }
+
+    void add(
+        ref<Store> store,
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs,
+        const StorePath & storePath,
+        bool immutable) override
+    {
+        _state.lock()->add.use()
+            (attrsToJson(inAttrs).dump())
+            (attrsToJson(infoAttrs).dump())
+            (store->printStorePath(storePath))
+            (immutable)
+            (time(0)).exec();
+    }
+
+    std::optional<std::pair<Attrs, StorePath>> lookup(
+        ref<Store> store,
+        const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(store, inAttrs)) {
+            if (!res->expired)
+                return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
+            debug("ignoring expired cache entry '%s'",
+                attrsToJson(inAttrs).dump());
+        }
+        return {};
+    }
+
+    std::optional<Result> lookupExpired(
+        ref<Store> store,
+        const Attrs & inAttrs) override
+    {
+        auto state(_state.lock());
+
+        auto inAttrsJson = attrsToJson(inAttrs).dump();
+
+        auto stmt(state->lookup.use()(inAttrsJson));
+        if (!stmt.next()) {
+            debug("did not find cache entry for '%s'", inAttrsJson);
+            return {};
+        }
+
+        auto infoJson = stmt.getStr(0);
+        auto storePath = store->parseStorePath(stmt.getStr(1));
+        auto immutable = stmt.getInt(2) != 0;
+        auto timestamp = stmt.getInt(3);
+
+        store->addTempRoot(storePath);
+        if (!store->isValidPath(storePath)) {
+            // FIXME: we could try to substitute 'storePath'.
+            debug("ignoring disappeared cache entry '%s'", inAttrsJson);
+            return {};
+        }
+
+        debug("using cache entry '%s' -> '%s', '%s'",
+            inAttrsJson, infoJson, store->printStorePath(storePath));
+
+        return Result {
+            .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
+            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
+            .storePath = std::move(storePath)
+        };
+    }
+};
+
+ref<Cache> getCache()
+{
+    static auto cache = std::make_shared<CacheImpl>();
+    return ref<Cache>(cache);
+}
+
+}

src/libfetchers/cache.hh

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "fetchers.hh"
+
+namespace nix::fetchers {
+
+struct Cache
+{
+    virtual void add(
+        ref<Store> store,
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs,
+        const StorePath & storePath,
+        bool immutable) = 0;
+
+    virtual std::optional<std::pair<Attrs, StorePath>> lookup(
+        ref<Store> store,
+        const Attrs & inAttrs) = 0;
+
+    struct Result
+    {
+        bool expired = false;
+        Attrs infoAttrs;
+        StorePath storePath;
+    };
+
+    virtual std::optional<Result> lookupExpired(
+        ref<Store> store,
+        const Attrs & inAttrs) = 0;
+};
+
+ref<Cache> getCache();
+
+}

src/libfetchers/fetchers.cc

@@ -0,0 +1,75 @@
+#include "fetchers.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
+
+void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
+{
+    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
+    inputSchemes->push_back(std::move(inputScheme));
+}
+
+std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
+{
+    for (auto & inputScheme : *inputSchemes) {
+        auto res = inputScheme->inputFromURL(url);
+        if (res) return res;
+    }
+    throw Error("input '%s' is unsupported", url.url);
+}
+
+std::unique_ptr<Input> inputFromURL(const std::string & url)
+{
+    return inputFromURL(parseURL(url));
+}
+
+std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
+{
+    auto attrs2(attrs);
+    attrs2.erase("narHash");
+    for (auto & inputScheme : *inputSchemes) {
+        auto res = inputScheme->inputFromAttrs(attrs2);
+        if (res) {
+            if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
+                // FIXME: require SRI hash.
+                res->narHash = newHashAllowEmpty(*narHash, htUnknown);
+            return res;
+        }
+    }
+    throw Error("input '%s' is unsupported", attrsToJson(attrs));
+}
+
+Attrs Input::toAttrs() const
+{
+    auto attrs = toAttrsInternal();
+    if (narHash)
+        attrs.emplace("narHash", narHash->to_string(SRI, true));
+    attrs.emplace("type", type());
+    return attrs;
+}
+
+std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
+{
+    auto [tree, input] = fetchTreeInternal(store);
+
+    if (tree.actualPath == "")
+        tree.actualPath = store->toRealPath(tree.storePath);
+
+    if (!tree.info.narHash)
+        tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
+
+    if (input->narHash)
+        assert(input->narHash == tree.info.narHash);
+
+    if (narHash && narHash != input->narHash)
+        throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+            to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
+
+    return {std::move(tree), input};
+}
+
+}

src/libfetchers/fetchers.hh

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+#include "path.hh"
+#include "tree-info.hh"
+#include "attrs.hh"
+#include "url.hh"
+
+#include <memory>
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct Input;
+
+struct Tree
+{
+    Path actualPath;
+    StorePath storePath;
+    TreeInfo info;
+};
+
+struct Input : std::enable_shared_from_this<Input>
+{
+    std::optional<Hash> narHash; // FIXME: implement
+
+    virtual std::string type() const = 0;
+
+    virtual ~Input() { }
+
+    virtual bool operator ==(const Input & other) const { return false; }
+
+    /* Check whether this is a "direct" input, that is, not
+       one that goes through a registry. */
+    virtual bool isDirect() const { return true; }
+
+    /* Check whether this is an "immutable" input, that is,
+       one that contains a commit hash or content hash. */
+    virtual bool isImmutable() const { return (bool) narHash; }
+
+    virtual bool contains(const Input & other) const { return false; }
+
+    virtual std::optional<std::string> getRef() const { return {}; }
+
+    virtual std::optional<Hash> getRev() const { return {}; }
+
+    virtual ParsedURL toURL() const = 0;
+
+    std::string to_string() const
+    {
+        return toURL().to_string();
+    }
+
+    Attrs toAttrs() const;
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
+
+private:
+
+    virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
+
+    virtual Attrs toAttrsInternal() const = 0;
+};
+
+struct InputScheme
+{
+    virtual ~InputScheme() { }
+
+    virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
+
+    virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
+};
+
+std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
+
+std::unique_ptr<Input> inputFromURL(const std::string & url);
+
+std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
+
+void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
+
+struct DownloadFileResult
+{
+    StorePath storePath;
+    std::string etag;
+    std::string effectiveUrl;
+};
+
+DownloadFileResult downloadFile(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable);
+
+Tree downloadTarball(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable);
+
+}

src/libfetchers/git.cc

@@ -0,0 +1,441 @@
+#include "fetchers.hh"
+#include "cache.hh"
+#include "globals.hh"
+#include "tarfile.hh"
+#include "store-api.hh"
+
+#include <sys/time.h>
+
+using namespace std::string_literals;
+
+namespace nix::fetchers {
+
+static std::string readHead(const Path & path)
+{
+    return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
+}
+
+static bool isNotDotGitDirectory(const Path & path)
+{
+    static const std::regex gitDirRegex("^(?:.*/)?\\.git$");
+
+    return not std::regex_match(path, gitDirRegex);
+}
+
+struct GitInput : Input
+{
+    ParsedURL url;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+    bool shallow = false;
+    bool submodules = false;
+
+    GitInput(const ParsedURL & url) : url(url)
+    { }
+
+    std::string type() const override { return "git"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const GitInput *>(&other);
+        return
+            other2
+            && url == other2->url
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev || narHash;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        ParsedURL url2(url);
+        if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
+        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
+        if (ref) url2.query.insert_or_assign("ref", *ref);
+        if (shallow) url2.query.insert_or_assign("shallow", "1");
+        return url2;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("url", url.to_string());
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        if (shallow)
+            attrs.emplace("shallow", true);
+        if (submodules)
+            attrs.emplace("submodules", true);
+        return attrs;
+    }
+
+    std::pair<bool, std::string> getActualUrl() const
+    {
+        // Don't clone file:// URIs (but otherwise treat them the
+        // same as remote URIs, i.e. don't use the working tree or
+        // HEAD).
+        static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
+        bool isLocal = url.scheme == "file" && !forceHttp;
+        return {isLocal, isLocal ? url.path : url.base};
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto name = "source";
+
+        auto input = std::make_shared<GitInput>(*this);
+
+        assert(!rev || rev->type == htSHA1);
+
+        std::string cacheType = "git";
+        if (shallow) cacheType += "-shallow";
+        if (submodules) cacheType += "-submodules";
+
+        auto getImmutableAttrs = [&]()
+        {
+            return Attrs({
+                {"type", cacheType},
+                {"name", name},
+                {"rev", input->rev->gitRev()},
+            });
+        };
+
+        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
+            -> std::pair<Tree, std::shared_ptr<const Input>>
+        {
+            assert(input->rev);
+            assert(!rev || rev == input->rev);
+            return {
+                Tree {
+                    .actualPath = store->toRealPath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
+                        .lastModified = getIntAttr(infoAttrs, "lastModified"),
+                    },
+                },
+                input
+            };
+        };
+
+        if (rev) {
+            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+                return makeResult(res->first, std::move(res->second));
+        }
+
+        auto [isLocal, actualUrl_] = getActualUrl();
+        auto actualUrl = actualUrl_; // work around clang bug
+
+        // If this is a local directory and no ref or revision is
+        // given, then allow the use of an unclean working tree.
+        if (!input->ref && !input->rev && isLocal) {
+            bool clean = false;
+
+            /* Check whether this repo has any commits. There are
+               probably better ways to do this. */
+            auto gitDir = actualUrl + "/.git";
+            auto commonGitDir = chomp(runProgram(
+                "git",
+                true,
+                { "-C", actualUrl, "rev-parse", "--git-common-dir" }
+            ));
+            if (commonGitDir != ".git")
+                    gitDir = commonGitDir;
+
+            bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
+
+            try {
+                if (haveCommits) {
+                    runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
+                    clean = true;
+                }
+            } catch (ExecError & e) {
+                if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
+            }
+
+            if (!clean) {
+
+                /* This is an unclean working tree. So copy all tracked files. */
+
+                if (!settings.allowDirty)
+                    throw Error("Git tree '%s' is dirty", actualUrl);
+
+                if (settings.warnDirty)
+                    warn("Git tree '%s' is dirty", actualUrl);
+
+                auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" });
+                if (submodules)
+                    gitOpts.emplace_back("--recurse-submodules");
+
+                auto files = tokenizeString<std::set<std::string>>(
+                    runProgram("git", true, gitOpts), "\0"s);
+
+                PathFilter filter = [&](const Path & p) -> bool {
+                    assert(hasPrefix(p, actualUrl));
+                    std::string file(p, actualUrl.size() + 1);
+
+                    auto st = lstat(p);
+
+                    if (S_ISDIR(st.st_mode)) {
+                        auto prefix = file + "/";
+                        auto i = files.lower_bound(prefix);
+                        return i != files.end() && hasPrefix(*i, prefix);
+                    }
+
+                    return files.count(file);
+                };
+
+                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+
+                auto tree = Tree {
+                    .actualPath = store->printStorePath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        // FIXME: maybe we should use the timestamp of the last
+                        // modified dirty file?
+                        .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
+                    }
+                };
+
+                return {std::move(tree), input};
+            }
+        }
+
+        if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
+
+        Attrs mutableAttrs({
+            {"type", cacheType},
+            {"name", name},
+            {"url", actualUrl},
+            {"ref", *input->ref},
+        });
+
+        Path repoDir;
+
+        if (isLocal) {
+
+            if (!input->rev)
+                input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
+
+            repoDir = actualUrl;
+
+        } else {
+
+            if (auto res = getCache()->lookup(store, mutableAttrs)) {
+                auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+                if (!rev || rev == rev2) {
+                    input->rev = rev2;
+                    return makeResult(res->first, std::move(res->second));
+                }
+            }
+
+            Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
+            repoDir = cacheDir;
+
+            if (!pathExists(cacheDir)) {
+                createDirs(dirOf(cacheDir));
+                runProgram("git", true, { "init", "--bare", repoDir });
+            }
+
+            Path localRefFile =
+                input->ref->compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + *input->ref
+                : cacheDir + "/refs/heads/" + *input->ref;
+
+            bool doFetch;
+            time_t now = time(0);
+
+            /* If a rev was specified, we need to fetch if it's not in the
+               repo. */
+            if (input->rev) {
+                try {
+                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
+                    doFetch = false;
+                } catch (ExecError & e) {
+                    if (WIFEXITED(e.status)) {
+                        doFetch = true;
+                    } else {
+                        throw;
+                    }
+                }
+            } else {
+                /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+                   git fetch to update the local ref to the remote ref. */
+                struct stat st;
+                doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+                    (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+            }
+
+            if (doFetch) {
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
+
+                // FIXME: git stderr messes up our progress indicator, so
+                // we're using --quiet for now. Should process its stderr.
+                try {
+                    auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
+                        ? *input->ref
+                        : "refs/heads/" + *input->ref;
+                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
+                } catch (Error & e) {
+                    if (!pathExists(localRefFile)) throw;
+                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
+                }
+
+                struct timeval times[2];
+                times[0].tv_sec = now;
+                times[0].tv_usec = 0;
+                times[1].tv_sec = now;
+                times[1].tv_usec = 0;
+
+                utimes(localRefFile.c_str(), times);
+            }
+
+            if (!input->rev)
+                input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
+        }
+
+        bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
+
+        if (isShallow && !shallow)
+            throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
+
+        // FIXME: check whether rev is an ancestor of ref.
+
+        printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
+
+        /* Now that we know the ref, check again whether we have it in
+           the store. */
+        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            return makeResult(res->first, std::move(res->second));
+
+        Path tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+        PathFilter filter = defaultPathFilter;
+
+        if (submodules) {
+            Path tmpGitDir = createTempDir();
+            AutoDelete delTmpGitDir(tmpGitDir, true);
+
+            runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+            // TODO: repoDir might lack the ref (it only checks if rev
+            // exists, see FIXME above) so use a big hammer and fetch
+            // everything to ensure we get the rev.
+            runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
+                                      "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
+
+            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
+            runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
+            runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
+
+            filter = isNotDotGitDirectory;
+        } else {
+            // FIXME: should pipe this, or find some better way to extract a
+            // revision.
+            auto source = sinkToSource([&](Sink & sink) {
+                RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
+                gitOptions.standardOut = &sink;
+                runProgram2(gitOptions);
+            });
+
+            unpackTarfile(*source, tmpDir);
+        }
+
+        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
+
+        Attrs infoAttrs({
+            {"rev", input->rev->gitRev()},
+            {"lastModified", lastModified},
+        });
+
+        if (!shallow)
+            infoAttrs.insert_or_assign("revCount",
+                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
+
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                storePath,
+                false);
+
+        getCache()->add(
+            store,
+            getImmutableAttrs(),
+            infoAttrs,
+            storePath,
+            true);
+
+        return makeResult(infoAttrs, std::move(storePath));
+    }
+};
+
+struct GitInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "git" &&
+            url.scheme != "git+http" &&
+            url.scheme != "git+https" &&
+            url.scheme != "git+ssh" &&
+            url.scheme != "git+file") return nullptr;
+
+        auto url2(url);
+        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
+        url2.query.clear();
+
+        Attrs attrs;
+        attrs.emplace("type", "git");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "git") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
+                throw Error("unsupported Git input attribute '%s'", name);
+
+        auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (std::regex_search(*ref, badGitRefRegex))
+                throw BadURL("invalid Git branch/tag name '%s'", *ref);
+            input->ref = *ref;
+        }
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+
+        input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
+
+        input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
+
+}

src/libfetchers/github.cc

@@ -0,0 +1,195 @@
+#include "filetransfer.hh"
+#include "cache.hh"
+#include "fetchers.hh"
+#include "globals.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct GitHubInput : Input
+{
+    std::string owner;
+    std::string repo;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    std::string type() const override { return "github"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const GitHubInput *>(&other);
+        return
+            other2
+            && owner == other2->owner
+            && repo == other2->repo
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev || narHash;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        auto path = owner + "/" + repo;
+        assert(!(ref && rev));
+        if (ref) path += "/" + *ref;
+        if (rev) path += "/" + rev->to_string(Base16, false);
+        return ParsedURL {
+            .scheme = "github",
+            .path = path,
+        };
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("owner", owner);
+        attrs.emplace("repo", repo);
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        return attrs;
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto rev = this->rev;
+        auto ref = this->ref.value_or("master");
+
+        if (!rev) {
+            auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
+                owner, repo, ref);
+            auto json = nlohmann::json::parse(
+                readFile(
+                    store->toRealPath(
+                        downloadFile(store, url, "source", false).storePath)));
+            rev = Hash(std::string { json["sha"] }, htSHA1);
+            debug("HEAD revision for '%s' is %s", url, rev->gitRev());
+        }
+
+        auto input = std::make_shared<GitHubInput>(*this);
+        input->ref = {};
+        input->rev = *rev;
+
+        Attrs immutableAttrs({
+            {"type", "git-tarball"},
+            {"rev", rev->gitRev()},
+        });
+
+        if (auto res = getCache()->lookup(store, immutableAttrs)) {
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(res->second),
+                    .storePath = std::move(res->second),
+                    .info = TreeInfo {
+                        .lastModified = getIntAttr(res->first, "lastModified"),
+                    },
+                },
+                input
+            };
+        }
+
+        // FIXME: use regular /archive URLs instead? api.github.com
+        // might have stricter rate limits.
+
+        auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
+            owner, repo, rev->to_string(Base16, false));
+
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            url += "?access_token=" + accessToken;
+
+        auto tree = downloadTarball(store, url, "source", true);
+
+        getCache()->add(
+            store,
+            immutableAttrs,
+            {
+                {"rev", rev->gitRev()},
+                {"lastModified", *tree.info.lastModified}
+            },
+            tree.storePath,
+            true);
+
+        return {std::move(tree), input};
+    }
+};
+
+struct GitHubInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "github") return nullptr;
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+        auto input = std::make_unique<GitHubInput>();
+
+        if (path.size() == 2) {
+        } else if (path.size() == 3) {
+            if (std::regex_match(path[2], revRegex))
+                input->rev = Hash(path[2], htSHA1);
+            else if (std::regex_match(path[2], refRegex))
+                input->ref = path[2];
+            else
+                throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+        } else
+            throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev") {
+                if (input->rev)
+                    throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
+                input->rev = Hash(value, htSHA1);
+            }
+            else if (name == "ref") {
+                if (!std::regex_match(value, refRegex))
+                    throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
+                if (input->ref)
+                    throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
+                input->ref = value;
+            }
+        }
+
+        if (input->ref && input->rev)
+            throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
+
+        input->owner = path[0];
+        input->repo = path[1];
+
+        return input;
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "github") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
+                throw Error("unsupported GitHub input attribute '%s'", name);
+
+        auto input = std::make_unique<GitHubInput>();
+        input->owner = getStrAttr(attrs, "owner");
+        input->repo = getStrAttr(attrs, "repo");
+        input->ref = maybeGetStrAttr(attrs, "ref");
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+
+}

src/libfetchers/local.mk

@@ -0,0 +1,11 @@
+libraries += libfetchers
+
+libfetchers_NAME = libnixfetchers
+
+libfetchers_DIR := $(d)
+
+libfetchers_SOURCES := $(wildcard $(d)/*.cc)
+
+libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
+
+libfetchers_LIBS = libutil libstore

src/libfetchers/mercurial.cc

@@ -0,0 +1,303 @@
+#include "fetchers.hh"
+#include "cache.hh"
+#include "globals.hh"
+#include "tarfile.hh"
+#include "store-api.hh"
+
+#include <sys/time.h>
+
+using namespace std::string_literals;
+
+namespace nix::fetchers {
+
+struct MercurialInput : Input
+{
+    ParsedURL url;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    MercurialInput(const ParsedURL & url) : url(url)
+    { }
+
+    std::string type() const override { return "hg"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const MercurialInput *>(&other);
+        return
+            other2
+            && url == other2->url
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev || narHash;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        ParsedURL url2(url);
+        url2.scheme = "hg+" + url2.scheme;
+        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
+        if (ref) url2.query.insert_or_assign("ref", *ref);
+        return url;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("url", url.to_string());
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        return attrs;
+    }
+
+    std::pair<bool, std::string> getActualUrl() const
+    {
+        bool isLocal = url.scheme == "file";
+        return {isLocal, isLocal ? url.path : url.base};
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto name = "source";
+
+        auto input = std::make_shared<MercurialInput>(*this);
+
+        auto [isLocal, actualUrl_] = getActualUrl();
+        auto actualUrl = actualUrl_; // work around clang bug
+
+        // FIXME: return lastModified.
+
+        // FIXME: don't clone local repositories.
+
+        if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
+
+            bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
+
+            if (!clean) {
+
+                /* This is an unclean working tree. So copy all tracked
+                   files. */
+
+                if (!settings.allowDirty)
+                    throw Error("Mercurial tree '%s' is unclean", actualUrl);
+
+                if (settings.warnDirty)
+                    warn("Mercurial tree '%s' is unclean", actualUrl);
+
+                input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
+
+                auto files = tokenizeString<std::set<std::string>>(
+                    runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+
+                PathFilter filter = [&](const Path & p) -> bool {
+                    assert(hasPrefix(p, actualUrl));
+                    std::string file(p, actualUrl.size() + 1);
+
+                    auto st = lstat(p);
+
+                    if (S_ISDIR(st.st_mode)) {
+                        auto prefix = file + "/";
+                        auto i = files.lower_bound(prefix);
+                        return i != files.end() && hasPrefix(*i, prefix);
+                    }
+
+                    return files.count(file);
+                };
+
+                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+
+                return {Tree {
+                    .actualPath = store->printStorePath(storePath),
+                    .storePath = std::move(storePath),
+                }, input};
+            }
+        }
+
+        if (!input->ref) input->ref = "default";
+
+        auto getImmutableAttrs = [&]()
+        {
+            return Attrs({
+                {"type", "hg"},
+                {"name", name},
+                {"rev", input->rev->gitRev()},
+            });
+        };
+
+        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
+            -> std::pair<Tree, std::shared_ptr<const Input>>
+        {
+            assert(input->rev);
+            assert(!rev || rev == input->rev);
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        .revCount = getIntAttr(infoAttrs, "revCount"),
+                    },
+                },
+                input
+            };
+        };
+
+        if (input->rev) {
+            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+                return makeResult(res->first, std::move(res->second));
+        }
+
+        assert(input->rev || input->ref);
+        auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
+
+        Attrs mutableAttrs({
+            {"type", "hg"},
+            {"name", name},
+            {"url", actualUrl},
+            {"ref", *input->ref},
+        });
+
+        if (auto res = getCache()->lookup(store, mutableAttrs)) {
+            auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+            if (!rev || rev == rev2) {
+                input->rev = rev2;
+                return makeResult(res->first, std::move(res->second));
+            }
+        }
+
+        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
+
+        /* If this is a commit hash that we already have, we don't
+           have to pull again. */
+        if (!(input->rev
+                && pathExists(cacheDir)
+                && runProgram(
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
+                    .killStderr(true)).second == "1"))
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
+
+            if (pathExists(cacheDir)) {
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                }
+                catch (ExecError & e) {
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal)) {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                    } else {
+                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
+                    }
+                }
+            } else {
+                createDirs(dirOf(cacheDir));
+                runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
+            }
+        }
+
+        auto tokens = tokenizeString<std::vector<std::string>>(
+            runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
+        assert(tokens.size() == 3);
+
+        input->rev = Hash(tokens[0], htSHA1);
+        auto revCount = std::stoull(tokens[1]);
+        input->ref = tokens[2];
+
+        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            return makeResult(res->first, std::move(res->second));
+
+        Path tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+
+        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
+
+        deletePath(tmpDir + "/.hg_archival.txt");
+
+        auto storePath = store->addToStore(name, tmpDir);
+
+        Attrs infoAttrs({
+            {"rev", input->rev->gitRev()},
+            {"revCount", (int64_t) revCount},
+        });
+
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                storePath,
+                false);
+
+        getCache()->add(
+            store,
+            getImmutableAttrs(),
+            infoAttrs,
+            storePath,
+            true);
+
+        return makeResult(infoAttrs, std::move(storePath));
+    }
+};
+
+struct MercurialInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "hg+http" &&
+            url.scheme != "hg+https" &&
+            url.scheme != "hg+ssh" &&
+            url.scheme != "hg+file") return nullptr;
+
+        auto url2(url);
+        url2.scheme = std::string(url2.scheme, 3);
+        url2.query.clear();
+
+        Attrs attrs;
+        attrs.emplace("type", "hg");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev")
+                throw Error("unsupported Mercurial input attribute '%s'", name);
+
+        auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (!std::regex_match(*ref, refRegex))
+                throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
+            input->ref = *ref;
+        }
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
+
+}

src/libfetchers/path.cc

@@ -0,0 +1,148 @@
+#include "fetchers.hh"
+#include "store-api.hh"
+
+namespace nix::fetchers {
+
+struct PathInput : Input
+{
+    Path path;
+
+    /* Allow the user to pass in "fake" tree info attributes. This is
+       useful for making a pinned tree work the same as the repository
+       from which is exported
+       (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
+    std::optional<Hash> rev;
+    std::optional<uint64_t> revCount;
+    std::optional<time_t> lastModified;
+
+    std::string type() const override { return "path"; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const PathInput *>(&other);
+        return
+            other2
+            && path == other2->path
+            && rev == other2->rev
+            && revCount == other2->revCount
+            && lastModified == other2->lastModified;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) narHash;
+    }
+
+    ParsedURL toURL() const override
+    {
+        auto query = attrsToQuery(toAttrsInternal());
+        query.erase("path");
+        return ParsedURL {
+            .scheme = "path",
+            .path = path,
+            .query = query,
+        };
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("path", path);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        if (revCount)
+            attrs.emplace("revCount", *revCount);
+        if (lastModified)
+            attrs.emplace("lastModified", *lastModified);
+        return attrs;
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto input = std::make_shared<PathInput>(*this);
+
+        // FIXME: check whether access to 'path' is allowed.
+
+        auto storePath = store->maybeParseStorePath(path);
+
+        if (storePath)
+            store->addTempRoot(*storePath);
+
+        if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
+            // FIXME: try to substitute storePath.
+            storePath = store->addToStore("source", path);
+
+        return
+            {
+                Tree {
+                    .actualPath = store->toRealPath(*storePath),
+                    .storePath = std::move(*storePath),
+                    .info = TreeInfo {
+                        .revCount = revCount,
+                        .lastModified = lastModified
+                    }
+                },
+                input
+            };
+    }
+
+};
+
+struct PathInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "path") return nullptr;
+
+        auto input = std::make_unique<PathInput>();
+        input->path = url.path;
+
+        for (auto & [name, value] : url.query)
+            if (name == "rev")
+                input->rev = Hash(value, htSHA1);
+            else if (name == "revCount") {
+                uint64_t revCount;
+                if (!string2Int(value, revCount))
+                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+                input->revCount = revCount;
+            }
+            else if (name == "lastModified") {
+                time_t lastModified;
+                if (!string2Int(value, lastModified))
+                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+                input->lastModified = lastModified;
+            }
+            else
+                throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
+
+        return input;
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "path") return {};
+
+        auto input = std::make_unique<PathInput>();
+        input->path = getStrAttr(attrs, "path");
+
+        for (auto & [name, value] : attrs)
+            if (name == "rev")
+                input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
+            else if (name == "revCount")
+                input->revCount = getIntAttr(attrs, "revCount");
+            else if (name == "lastModified")
+                input->lastModified = getIntAttr(attrs, "lastModified");
+            else if (name == "type" || name == "path")
+                ;
+            else
+                throw Error("unsupported path input attribute '%s'", name);
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
+
+}

src/libfetchers/tarball.cc

@@ -0,0 +1,275 @@
+#include "fetchers.hh"
+#include "cache.hh"
+#include "filetransfer.hh"
+#include "globals.hh"
+#include "store-api.hh"
+#include "archive.hh"
+#include "tarfile.hh"
+
+namespace nix::fetchers {
+
+DownloadFileResult downloadFile(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable)
+{
+    // FIXME: check store
+
+    Attrs inAttrs({
+        {"type", "file"},
+        {"url", url},
+        {"name", name},
+    });
+
+    auto cached = getCache()->lookupExpired(store, inAttrs);
+
+    auto useCached = [&]() -> DownloadFileResult
+    {
+        return {
+            .storePath = std::move(cached->storePath),
+            .etag = getStrAttr(cached->infoAttrs, "etag"),
+            .effectiveUrl = getStrAttr(cached->infoAttrs, "url")
+        };
+    };
+
+    if (cached && !cached->expired)
+        return useCached();
+
+    FileTransferRequest request(url);
+    if (cached)
+        request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
+    FileTransferResult res;
+    try {
+        res = getFileTransfer()->download(request);
+    } catch (FileTransferError & e) {
+        if (cached) {
+            warn("%s; using cached version", e.msg());
+            return useCached();
+        } else
+            throw;
+    }
+
+    // FIXME: write to temporary file.
+
+    Attrs infoAttrs({
+        {"etag", res.etag},
+        {"url", res.effectiveUri},
+    });
+
+    std::optional<StorePath> storePath;
+
+    if (res.cached) {
+        assert(cached);
+        assert(request.expectedETag == res.etag);
+        storePath = std::move(cached->storePath);
+    } else {
+        StringSink sink;
+        dumpString(*res.data, sink);
+        auto hash = hashString(htSHA256, *res.data);
+        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
+        info.narHash = hashString(htSHA256, *sink.s);
+        info.narSize = sink.s->size();
+        info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
+        auto source = StringSource { *sink.s };
+        store->addToStore(info, source, NoRepair, NoCheckSigs);
+        storePath = std::move(info.path);
+    }
+
+    getCache()->add(
+        store,
+        inAttrs,
+        infoAttrs,
+        *storePath,
+        immutable);
+
+    if (url != res.effectiveUri)
+        getCache()->add(
+            store,
+            {
+                {"type", "file"},
+                {"url", res.effectiveUri},
+                {"name", name},
+            },
+            infoAttrs,
+            *storePath,
+            immutable);
+
+    return {
+        .storePath = std::move(*storePath),
+        .etag = res.etag,
+        .effectiveUrl = res.effectiveUri,
+    };
+}
+
+Tree downloadTarball(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable)
+{
+    Attrs inAttrs({
+        {"type", "tarball"},
+        {"url", url},
+        {"name", name},
+    });
+
+    auto cached = getCache()->lookupExpired(store, inAttrs);
+
+    if (cached && !cached->expired)
+        return Tree {
+            .actualPath = store->toRealPath(cached->storePath),
+            .storePath = std::move(cached->storePath),
+            .info = TreeInfo {
+                .lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
+            },
+        };
+
+    auto res = downloadFile(store, url, name, immutable);
+
+    std::optional<StorePath> unpackedStorePath;
+    time_t lastModified;
+
+    if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) {
+        unpackedStorePath = std::move(cached->storePath);
+        lastModified = getIntAttr(cached->infoAttrs, "lastModified");
+    } else {
+        Path tmpDir = createTempDir();
+        AutoDelete autoDelete(tmpDir, true);
+        unpackTarfile(store->toRealPath(res.storePath), tmpDir);
+        auto members = readDirectory(tmpDir);
+        if (members.size() != 1)
+            throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
+        auto topDir = tmpDir + "/" + members.begin()->name;
+        lastModified = lstat(topDir).st_mtime;
+        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
+    }
+
+    Attrs infoAttrs({
+        {"lastModified", lastModified},
+        {"etag", res.etag},
+    });
+
+    getCache()->add(
+        store,
+        inAttrs,
+        infoAttrs,
+        *unpackedStorePath,
+        immutable);
+
+    return Tree {
+        .actualPath = store->toRealPath(*unpackedStorePath),
+        .storePath = std::move(*unpackedStorePath),
+        .info = TreeInfo {
+            .lastModified = lastModified,
+        },
+    };
+}
+
+struct TarballInput : Input
+{
+    ParsedURL url;
+    std::optional<Hash> hash;
+
+    TarballInput(const ParsedURL & url) : url(url)
+    { }
+
+    std::string type() const override { return "tarball"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const TarballInput *>(&other);
+        return
+            other2
+            && to_string() == other2->to_string()
+            && hash == other2->hash;
+    }
+
+    bool isImmutable() const override
+    {
+        return hash || narHash;
+    }
+
+    ParsedURL toURL() const override
+    {
+        auto url2(url);
+        // NAR hashes are preferred over file hashes since tar/zip files
+        // don't have a canonical representation.
+        if (narHash)
+            url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
+        else if (hash)
+            url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
+        return url2;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("url", url.to_string());
+        if (hash)
+            attrs.emplace("hash", hash->to_string(SRI, true));
+        return attrs;
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto tree = downloadTarball(store, url.to_string(), "source", false);
+
+        auto input = std::make_shared<TarballInput>(*this);
+        input->narHash = store->queryPathInfo(tree.storePath)->narHash;
+
+        return {std::move(tree), input};
+    }
+};
+
+struct TarballInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
+
+        if (!hasSuffix(url.path, ".zip")
+            && !hasSuffix(url.path, ".tar")
+            && !hasSuffix(url.path, ".tar.gz")
+            && !hasSuffix(url.path, ".tar.xz")
+            && !hasSuffix(url.path, ".tar.bz2"))
+            return nullptr;
+
+        auto input = std::make_unique<TarballInput>(url);
+
+        auto hash = input->url.query.find("hash");
+        if (hash != input->url.query.end()) {
+            // FIXME: require SRI hash.
+            input->hash = Hash(hash->second);
+            input->url.query.erase(hash);
+        }
+
+        auto narHash = input->url.query.find("narHash");
+        if (narHash != input->url.query.end()) {
+            // FIXME: require SRI hash.
+            input->narHash = Hash(narHash->second);
+            input->url.query.erase(narHash);
+        }
+
+        return input;
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "hash")
+                throw Error("unsupported tarball input attribute '%s'", name);
+
+        auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
+        if (auto hash = maybeGetStrAttr(attrs, "hash"))
+            input->hash = newHashAllowEmpty(*hash, htUnknown);
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
+
+}

src/libfetchers/tree-info.cc

@@ -0,0 +1,14 @@
+#include "tree-info.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+StorePath TreeInfo::computeStorePath(Store & store) const
+{
+    assert(narHash);
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
+}
+
+}

src/libfetchers/tree-info.hh

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "path.hh"
+#include "hash.hh"
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct TreeInfo
+{
+    Hash narHash;
+    std::optional<uint64_t> revCount;
+    std::optional<time_t> lastModified;
+
+    bool operator ==(const TreeInfo & other) const
+    {
+        return
+            narHash == other.narHash
+            && revCount == other.revCount
+            && lastModified == other.lastModified;
+    }
+
+    StorePath computeStorePath(Store & store) const;
+};
+
+}

src/libmain/common-args.cc

@@ -1,48 +1,61 @@
 #include "common-args.hh"
 #include "globals.hh"
+#include "loggers.hh"
 
 namespace nix {
 
 MixCommonArgs::MixCommonArgs(const string & programName)
     : programName(programName)
 {
-    mkFlag()
-        .longName("verbose")
-        .shortName('v')
-        .description("increase verbosity level")
-        .handler([]() { verbosity = (Verbosity) (verbosity + 1); });
+    addFlag({
+        .longName = "verbose",
+        .shortName = 'v',
+        .description = "increase verbosity level",
+        .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
+    });
 
-    mkFlag()
-        .longName("quiet")
-        .description("decrease verbosity level")
-        .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; });
+    addFlag({
+        .longName = "quiet",
+        .description = "decrease verbosity level",
+        .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
+    });
 
-    mkFlag()
-        .longName("debug")
-        .description("enable debug output")
-        .handler([]() { verbosity = lvlDebug; });
+    addFlag({
+        .longName = "debug",
+        .description = "enable debug output",
+        .handler = {[]() { verbosity = lvlDebug; }},
+    });
 
-    mkFlag()
-        .longName("option")
-        .labels({"name", "value"})
-        .description("set a Nix configuration option (overriding nix.conf)")
-        .arity(2)
-        .handler([](std::vector<std::string> ss) {
+    addFlag({
+        .longName = "option",
+        .description = "set a Nix configuration option (overriding nix.conf)",
+        .labels = {"name", "value"},
+        .handler = {[](std::string name, std::string value) {
             try {
-                globalConfig.set(ss[0], ss[1]);
+                globalConfig.set(name, value);
             } catch (UsageError & e) {
                 warn(e.what());
             }
-        });
+        }},
+    });
 
-    mkFlag()
-        .longName("max-jobs")
-        .shortName('j')
-        .label("jobs")
-        .description("maximum number of parallel builds")
-        .handler([=](std::string s) {
+    addFlag({
+        .longName = "log-format",
+        .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
+                        "or \"bar-with-logs\"",
+        .labels = {"format"},
+        .handler = {[](std::string format) { setLogFormat(format); }},
+    });
+
+    addFlag({
+        .longName = "max-jobs",
+        .shortName = 'j',
+        .description = "maximum number of parallel builds",
+        .labels = Strings{"jobs"},
+        .handler = {[=](std::string s) {
             settings.set("max-jobs", s);
-        });
+        }}
+    });
 
     std::string cat = "config";
     globalConfig.convertToArgs(*this, cat);

src/libmain/local.mk

@@ -6,6 +6,8 @@ libmain_DIR := $(d)
 
 libmain_SOURCES := $(wildcard $(d)/*.cc)
 
+libmain_CXXFLAGS += -I src/libutil -I src/libstore
+
 libmain_LDFLAGS = $(OPENSSL_LIBS)
 
 libmain_LIBS = libstore libutil

src/libmain/loggers.cc

@@ -0,0 +1,52 @@
+#include "loggers.hh"
+#include "progress-bar.hh"
+
+namespace nix {
+
+LogFormat defaultLogFormat = LogFormat::raw;
+
+LogFormat parseLogFormat(const std::string & logFormatStr) {
+    if (logFormatStr == "raw")
+        return LogFormat::raw;
+    else if (logFormatStr == "raw-with-logs")
+        return LogFormat::rawWithLogs;
+    else if (logFormatStr == "internal-json")
+        return LogFormat::internalJson;
+    else if (logFormatStr == "bar")
+        return LogFormat::bar;
+    else if (logFormatStr == "bar-with-logs")
+        return LogFormat::barWithLogs;
+    throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
+}
+
+Logger * makeDefaultLogger() {
+    switch (defaultLogFormat) {
+    case LogFormat::raw:
+        return makeSimpleLogger(false);
+    case LogFormat::rawWithLogs:
+        return makeSimpleLogger(true);
+    case LogFormat::internalJson:
+        return makeJSONLogger(*makeSimpleLogger());
+    case LogFormat::bar:
+        return makeProgressBar();
+    case LogFormat::barWithLogs:
+        return makeProgressBar(true);
+    default:
+        abort();
+    }
+}
+
+void setLogFormat(const std::string & logFormatStr) {
+    setLogFormat(parseLogFormat(logFormatStr));
+}
+
+void setLogFormat(const LogFormat & logFormat) {
+    defaultLogFormat = logFormat;
+    createDefaultLogger();
+}
+
+void createDefaultLogger() {
+    logger = makeDefaultLogger();
+}
+
+}

src/libmain/loggers.hh

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+enum class LogFormat {
+  raw,
+  rawWithLogs,
+  internalJson,
+  bar,
+  barWithLogs,
+};
+
+void setLogFormat(const std::string & logFormatStr);
+void setLogFormat(const LogFormat & logFormat);
+
+void createDefaultLogger();
+
+}

src/libmain/progress-bar.cc

@@ -7,6 +7,7 @@
 #include <atomic>
 #include <map>
 #include <thread>
+#include <iostream>
 
 namespace nix {
 
@@ -105,25 +106,36 @@ public:
         updateThread.join();
     }
 
-    void stop()
+    void stop() override
     {
         auto state(state_.lock());
         if (!state->active) return;
         state->active = false;
-        std::string status = getStatus(*state);
         writeToStderr("\r\e[K");
-        if (status != "")
-            writeToStderr("[" + status + "]\n");
         updateCV.notify_one();
         quitCV.notify_one();
     }
 
+    bool isVerbose() override {
+        return printBuildLogs;
+    }
+
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
         auto state(state_.lock());
         log(*state, lvl, fs.s);
     }
 
+    void logEI(const ErrorInfo &ei) override
+    {
+        auto state(state_.lock());
+
+        std::stringstream oss;
+        oss << ei;
+
+        log(*state, ei.level, oss.str());
+    }
+
     void log(State & state, Verbosity lvl, const std::string & s)
     {
         if (state.active) {
@@ -141,7 +153,7 @@ public:
     {
         auto state(state_.lock());
 
-        if (lvl <= verbosity && !s.empty())
+        if (lvl <= verbosity && !s.empty() && type != actBuildWaiting)
             log(*state, lvl, s + "...");
 
         state->activities.emplace_back(ActInfo());
@@ -153,7 +165,7 @@ public:
         state->activitiesByType[type].its.emplace(act, i);
 
         if (type == actBuild) {
-            auto name = storePathToName(getS(fields, 0));
+            std::string name(storePathToName(getS(fields, 0)));
             if (hasSuffix(name, ".drv"))
                 name = name.substr(0, name.size() - 4);
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
@@ -190,8 +202,8 @@ public:
             i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
         }
 
-        if ((type == actDownload && hasAncestor(*state, actCopyPath, parent))
-            || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent))
+        if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent))
+            || (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent))
             || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
             i->visible = false;
 
@@ -416,7 +428,7 @@ public:
             if (!s2.empty()) { res += " ("; res += s2; res += ')'; }
         }
 
-        showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
+        showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB);
 
         {
             auto s = renderActivity(actOptimiseStore, "%s paths optimised");
@@ -442,13 +454,31 @@ public:
 
         return res;
     }
+
+    void writeToStdout(std::string_view s) override
+    {
+        auto state(state_.lock());
+        if (state->active) {
+            std::cerr << "\r\e[K";
+            Logger::writeToStdout(s);
+            draw(*state);
+        } else {
+            Logger::writeToStdout(s);
+        }
+    }
 };
 
+Logger * makeProgressBar(bool printBuildLogs)
+{
+    return new ProgressBar(
+        printBuildLogs,
+        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
+    );
+}
+
 void startProgressBar(bool printBuildLogs)
 {
-    logger = new ProgressBar(
-        printBuildLogs,
-        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb");
+    logger = makeProgressBar(printBuildLogs);
 }
 
 void stopProgressBar()

src/libmain/progress-bar.hh

@@ -4,6 +4,8 @@
 
 namespace nix {
 
+Logger * makeProgressBar(bool printBuildLogs = false);
+
 void startProgressBar(bool printBuildLogs = false);
 
 void stopProgressBar();

src/libmain/shared.cc

@@ -2,6 +2,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "loggers.hh"
 
 #include <algorithm>
 #include <cctype>
@@ -75,7 +76,7 @@ string getArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
-    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
+    if (i == end) throw UsageError("'%1%' requires an argument", opt);
     return *i;
 }
 
@@ -155,7 +156,7 @@ void initNix()
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
 #if __APPLE__
-    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
+    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 }
@@ -165,28 +166,32 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
     : MixCommonArgs(programName), parseArg(parseArg)
 {
-    mkFlag()
-        .longName("no-build-output")
-        .shortName('Q')
-        .description("do not show build output")
-        .set(&settings.verboseBuild, false);
+    addFlag({
+        .longName = "no-build-output",
+        .shortName = 'Q',
+        .description = "do not show build output",
+        .handler = {[&]() {setLogFormat(LogFormat::raw); }},
+    });
 
-    mkFlag()
-        .longName("keep-failed")
-        .shortName('K')
-        .description("keep temporary directories of failed builds")
-        .set(&(bool&) settings.keepFailed, true);
+    addFlag({
+        .longName = "keep-failed",
+        .shortName ='K',
+        .description = "keep temporary directories of failed builds",
+        .handler = {&(bool&) settings.keepFailed, true},
+    });
 
-    mkFlag()
-        .longName("keep-going")
-        .shortName('k')
-        .description("keep going after a build fails")
-        .set(&(bool&) settings.keepGoing, true);
+    addFlag({
+        .longName = "keep-going",
+        .shortName ='k',
+        .description = "keep going after a build fails",
+        .handler = {&(bool&) settings.keepGoing, true},
+    });
 
-    mkFlag()
-        .longName("fallback")
-        .description("build from source if substitution fails")
-        .set(&(bool&) settings.tryFallback, true);
+    addFlag({
+        .longName = "fallback",
+        .description = "build from source if substitution fails",
+        .handler = {&(bool&) settings.tryFallback, true},
+    });
 
     auto intSettingAlias = [&](char shortName, const std::string & longName,
         const std::string & description, const std::string & dest) {
@@ -205,11 +210,12 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
         &gcWarning, false);
 
-    mkFlag()
-        .longName("store")
-        .label("store-uri")
-        .description("URI of the Nix store to use")
-        .dest(&(std::string&) settings.storeUri);
+    addFlag({
+        .longName = "store",
+        .description = "URI of the Nix store to use",
+        .labels = {"store-uri"},
+        .handler = {&(std::string&) settings.storeUri},
+    });
 }
 
 
@@ -229,7 +235,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
     Strings ss(args);
     auto pos = ss.begin();
     if (!parseArg(pos, ss.end()))
-        throw UsageError(format("unexpected argument '%1%'") % args.front());
+        throw UsageError("unexpected argument '%1%'", args.front());
     return true;
 }
 
@@ -260,7 +266,10 @@ void printVersion(const string & programName)
         cfg.push_back("signed-caches");
 #endif
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
-        std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
+        std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
+        std::cout << "User configuration files: " <<
+            concatStringsSep(":", settings.nixUserConfFiles)
+            << "\n";
         std::cout << "Store directory: " << settings.nixStore << "\n";
         std::cout << "State directory: " << settings.nixStateDir << "\n";
     }
@@ -273,7 +282,7 @@ void showManPage(const string & name)
     restoreSignals();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), nullptr);
-    throw SysError(format("command 'man %1%' failed") % name.c_str());
+    throw SysError("command 'man %1%' failed", name.c_str());
 }
 
 
@@ -281,6 +290,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
 {
     ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
 
+    ErrorInfo::programName = baseNameOf(programName);
+
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {
         try {
@@ -296,12 +307,13 @@ int handleExceptions(const string & programName, std::function<void()> fun)
     } catch (Exit & e) {
         return e.status;
     } catch (UsageError & e) {
-        printError(
-            format(error + "%1%\nTry '%2% --help' for more information.")
-            % e.what() % programName);
+        logError(e.info());
+        printError("Try '%1% --help' for more information.", programName);
         return 1;
     } catch (BaseError & e) {
-        printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
+        if (settings.showTrace && e.prefix() != "")
+            printError(e.prefix());
+        logError(e.info());
         if (e.prefix() != "" && !settings.showTrace)
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
@@ -338,7 +350,7 @@ RunPager::RunPager()
         execlp("pager", "pager", nullptr);
         execlp("less", "less", nullptr);
         execlp("more", "more", nullptr);
-        throw SysError(format("executing '%1%'") % pager);
+        throw SysError("executing '%1%'", pager);
     });
 
     pid.setKillSignal(SIGINT);

src/libmain/shared.hh

@@ -56,7 +56,7 @@ template<class N> N getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
 {
     ++i;
-    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
+    if (i == end) throw UsageError("'%1%' requires an argument", opt);
     string s = *i;
     N multiplier = 1;
     if (allowUnit && !s.empty()) {
@@ -66,13 +66,13 @@ template<class N> N getIntArg(const string & opt,
             else if (u == 'M') multiplier = 1ULL << 20;
             else if (u == 'G') multiplier = 1ULL << 30;
             else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError(format("invalid unit specifier '%1%'") % u);
+            else throw UsageError("invalid unit specifier '%1%'", u);
             s.resize(s.size() - 1);
         }
     }
     N n;
     if (!string2Int(s, n))
-        throw UsageError(format("'%1%' requires an integer argument") % opt);
+        throw UsageError("'%1%' requires an integer argument", opt);
     return n * multiplier;
 }
 

src/libmain/stack.cc

@@ -1,4 +1,4 @@
-#include "types.hh"
+#include "error.hh"
 
 #include <cstring>
 #include <cstddef>

src/libstore/binary-cache-store.cc

@@ -40,14 +40,14 @@ void BinaryCacheStore::init()
         upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
-            size_t colon = line.find(':');
-            if (colon == std::string::npos) continue;
+            size_t colon= line.find(':');
+            if (colon ==std::string::npos) continue;
             auto name = line.substr(0, colon);
             auto value = trim(line.substr(colon + 1, std::string::npos));
             if (name == "StoreDir") {
                 if (value != storeDir)
-                    throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
-                        % getUri() % value % storeDir);
+                    throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
+                        getUri(), value, storeDir);
             } else if (name == "WantMassQuery") {
                 wantMassQuery.setDefault(value == "1" ? "true" : "false");
             } else if (name == "Priority") {
@@ -93,7 +93,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
 {
-    return storePathToHash(printStorePath(storePath)) + ".narinfo";
+    return std::string(storePath.hashPart()) + ".narinfo";
 }
 
 void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
@@ -102,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    auto hashPart = storePathToHash(printStorePath(narInfo->path));
+    std::string hashPart(narInfo->path.hashPart());
 
     {
         auto state_(state.lock());
@@ -113,9 +113,12 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 }
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
+    // FIXME: See if we can use the original source to reduce memory usage.
+    auto nar = make_ref<std::string>(narSource.drain());
+
     if (!repair && isValidPath(info.path)) return;
 
     /* Verify that all references are valid. This may do some .narinfo
@@ -161,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             }
         }
 
-        upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
     }
 
     /* Compress the NAR. */
@@ -284,7 +287,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
     try {
         getFile(info->url, *decompressor);
     } catch (NoSuchBinaryCacheFile & e) {
-        throw SubstituteGone(e.what());
+        throw SubstituteGone(e.info());
     }
 
     decompressor->finish();
@@ -327,7 +330,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
 
@@ -336,7 +339,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
        small files. */
     StringSink sink;
     Hash h;
-    if (recursive) {
+    if (method == FileIngestionMethod::Recursive) {
         dumpPath(srcPath, sink, filter);
         h = hashString(hashAlgo, *sink.s);
     } else {
@@ -345,9 +348,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
+    ValidPathInfo info(makeFixedOutputPath(method, h, name));
 
-    addToStore(info, sink.s, repair, CheckSigs, nullptr);
+    auto source = StringSource { *sink.s };
+    addToStore(info, source, repair, CheckSigs, nullptr);
 
     return std::move(info.path);
 }
@@ -356,12 +360,13 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     const StorePathSet & references, RepairFlag repair)
 {
     ValidPathInfo info(computeStorePathForText(name, s, references));
-    info.references = cloneStorePathSet(references);
+    info.references = references;
 
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToStore(info, sink.s, repair, CheckSigs, nullptr);
+        auto source = StringSource { *sink.s };
+        addToStore(info, source, repair, CheckSigs, nullptr);
     }
 
     return std::move(info.path);
@@ -390,14 +395,14 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
 
 std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
 {
-    auto drvPath = path.clone();
+    auto drvPath = path;
 
     if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
             // FIXME: add a "Log" field to .narinfo
             if (!info->deriver) return nullptr;
-            drvPath = info->deriver->clone();
+            drvPath = *info->deriver;
         } catch (InvalidPath &) {
             return nullptr;
         }

src/libstore/binary-cache-store.hh

@@ -74,12 +74,12 @@ public:
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
-    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+    void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive, HashType hashAlgo,
+        FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
     StorePath addTextToStore(const string & name, const string & s,

src/libstore/builtins/buildenv.cc

@@ -22,7 +22,10 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
         srcFiles = readDirectory(srcDir);
     } catch (SysError & e) {
         if (e.errNo == ENOTDIR) {
-            printError("warning: not including '%s' in the user environment because it's not a directory", srcDir);
+            logWarning({
+                .name = "Create links - directory",
+                .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
+            });
             return;
         }
         throw;
@@ -41,7 +44,10 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                 throw SysError("getting status of '%1%'", srcFile);
         } catch (SysError & e) {
             if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
-                printError("warning: skipping dangling symlink '%s'", dstFile);
+                logWarning({
+                    .name = "Create links - skipping symlink",
+                    .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
+                });
                 continue;
             }
             throw;
@@ -72,15 +78,15 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     if (!S_ISDIR(lstat(target).st_mode))
                         throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking '%1%'") % dstFile);
+                        throw SysError("unlinking '%1%'", dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
-                        throw SysError(format("creating directory '%1%'"));
+                        throw SysError("creating directory '%1%'", dstFile);
                     createLinks(state, target, dstFile, state.priorities[dstFile]);
                     createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
-                throw SysError(format("getting status of '%1%'") % dstFile);
+                throw SysError("getting status of '%1%'", dstFile);
         }
 
         else {
@@ -99,11 +105,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     if (prevPriority < priority)
                         continue;
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking '%1%'") % dstFile);
+                        throw SysError("unlinking '%1%'", dstFile);
                 } else if (S_ISDIR(dstSt.st_mode))
                     throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
             } else if (errno != ENOENT)
-                throw SysError(format("getting status of '%1%'") % dstFile);
+                throw SysError("getting status of '%1%'", dstFile);
         }
 
         createSymlink(srcFile, dstFile);

src/libstore/builtins/fetchurl.cc

@@ -1,5 +1,5 @@
 #include "builtins.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "compression.hh"
@@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     auto getAttr = [&](const string & name) {
         auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name);
+        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;
     };
 
@@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
     auto mainUrl = getAttr("url");
     bool unpack = get(drv.env, "unpack").value_or("") == "1";
 
-    /* Note: have to use a fresh downloader here because we're in
+    /* Note: have to use a fresh fileTransfer here because we're in
        a forked process. */
-    auto downloader = makeDownloader();
+    auto fileTransfer = makeFileTransfer();
 
     auto fetch = [&](const std::string & url) {
 
@@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
             /* No need to do TLS verification, because we check the hash of
                the result anyway. */
-            DownloadRequest request(url);
+            FileTransferRequest request(url);
             request.verifyTLS = false;
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(
                 unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
-            downloader->download(std::move(request), *decompressor);
+            fileTransfer->download(std::move(request), *decompressor);
             decompressor->finish();
         });
 
@@ -54,7 +54,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         auto executable = drv.env.find("executable");
         if (executable != drv.env.end() && executable->second == "1") {
             if (chmod(storePath.c_str(), 0755) == -1)
-                throw SysError(format("making '%1%' executable") % storePath);
+                throw SysError("making '%1%' executable", storePath);
         }
     };
 

src/libstore/daemon.cc

@@ -73,6 +73,18 @@ struct TunnelLogger : public Logger
         enqueueMsg(*buf.s);
     }
 
+    void logEI(const ErrorInfo & ei) override
+    {
+        if (ei.level > verbosity) return;
+
+        std::stringstream oss;
+        oss << ei;
+
+        StringSink buf;
+        buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
+        enqueueMsg(*buf.s);
+    }
+
     /* startWork() means that we're starting an operation for which we
       want to send out stderr to the client. */
     void startWork()
@@ -281,7 +293,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         StorePathSet paths; // FIXME
-        paths.insert(path.clone());
+        paths.insert(path);
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
         to << (res.count(path) != 0);
@@ -315,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         StorePathSet paths;
         if (op == wopQueryReferences)
             for (auto & i : store->queryPathInfo(path)->references)
-                paths.insert(i.clone());
+                paths.insert(i);
         else if (op == wopQueryReferrers)
             store->queryReferrers(path, paths);
         else if (op == wopQueryValidDerivers)
@@ -329,8 +341,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputNames: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        StringSet names;
-        names = store->queryDerivationOutputNames(path);
+        auto names = store->readDerivation(path).outputNames();
         logger->stopWork();
         to << names;
         break;
@@ -355,20 +366,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddToStore: {
-        bool fixed, recursive;
         std::string s, baseName;
-        from >> baseName >> fixed /* obsolete */ >> recursive >> s;
-        /* Compatibility hack. */
-        if (!fixed) {
-            s = "sha256";
-            recursive = true;
+        FileIngestionMethod method;
+        {
+            bool fixed; uint8_t recursive;
+            from >> baseName >> fixed /* obsolete */ >> recursive >> s;
+            if (recursive > (uint8_t) FileIngestionMethod::Recursive)
+                throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
+            method = FileIngestionMethod { recursive };
+            /* Compatibility hack. */
+            if (!fixed) {
+                s = "sha256";
+                method = FileIngestionMethod::Recursive;
+            }
         }
         HashType hashAlgo = parseHashType(s);
 
         TeeSource savedNAR(from);
         RetrieveRegularNARSink savedRegular;
 
-        if (recursive) {
+        if (method == FileIngestionMethod::Recursive) {
             /* Get the entire NAR dump from the client and save it to
                a string so that we can pass it to
                addToStoreFromDump(). */
@@ -380,7 +397,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
 
-        auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
+        auto path = store->addToStoreFromDump(
+            method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
+            baseName,
+            method,
+            hashAlgo);
         logger->stopWork();
 
         to << store->printStorePath(path);
@@ -572,9 +593,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         SubstitutablePathInfos infos;
-        StorePathSet paths;
-        paths.insert(path.clone()); // FIXME
-        store->querySubstitutablePathInfos(paths, infos);
+        store->querySubstitutablePathInfos({path}, infos);
         logger->stopWork();
         auto i = infos.find(path);
         if (i == infos.end())
@@ -735,7 +754,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     default:
-        throw Error(format("invalid operation %1%") % op);
+        throw Error("invalid operation %1%", op);
     }
 }
 

src/libstore/derivations.cc

@@ -9,13 +9,13 @@
 namespace nix {
 
 
-void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
+void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
 {
-    recursive = false;
+    recursive = FileIngestionMethod::Flat;
     string algo = hashAlgo;
 
     if (string(algo, 0, 2) == "r:") {
-        recursive = true;
+        recursive = FileIngestionMethod::Recursive;
         algo = string(algo, 2);
     }
 
@@ -27,28 +27,6 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
 }
 
 
-BasicDerivation::BasicDerivation(const BasicDerivation & other)
-    : platform(other.platform)
-    , builder(other.builder)
-    , args(other.args)
-    , env(other.env)
-{
-    for (auto & i : other.outputs)
-        outputs.insert_or_assign(i.first,
-            DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash)));
-    for (auto & i : other.inputSrcs)
-        inputSrcs.insert(i.clone());
-}
-
-
-Derivation::Derivation(const Derivation & other)
-    : BasicDerivation(other)
-{
-    for (auto & i : other.inputDrvs)
-        inputDrvs.insert_or_assign(i.first.clone(), i.second);
-}
-
-
 const StorePath & BasicDerivation::findOutput(const string & id) const
 {
     auto i = outputs.find(id);
@@ -67,9 +45,9 @@ bool BasicDerivation::isBuiltin() const
 StorePath writeDerivation(ref<Store> store,
     const Derivation & drv, std::string_view name, RepairFlag repair)
 {
-    auto references = cloneStorePathSet(drv.inputSrcs);
+    auto references = drv.inputSrcs;
     for (auto & i : drv.inputDrvs)
-        references.insert(i.first.clone());
+        references.insert(i.first);
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
@@ -87,7 +65,7 @@ static void expect(std::istream & str, const string & s)
     char s2[s.size()];
     str.read(s2, s.size());
     if (string(s2, s.size()) != s)
-        throw FormatError(format("expected string '%1%'") % s);
+        throw FormatError("expected string '%1%'", s);
 }
 
 
@@ -114,7 +92,7 @@ static Path parsePath(std::istream & str)
 {
     string s = parseString(str);
     if (s.size() == 0 || s[0] != '/')
-        throw FormatError(format("bad path '%1%' in derivation") % s);
+        throw FormatError("bad path '%1%' in derivation", s);
     return s;
 }
 
@@ -155,7 +133,11 @@ static Derivation parseDerivation(const Store & store, const string & s)
         expect(str, ","); auto hashAlgo = parseString(str);
         expect(str, ","); auto hash = parseString(str);
         expect(str, ")");
-        drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
+        drv.outputs.emplace(id, DerivationOutput {
+            .path = std::move(path),
+            .hashAlgo = std::move(hashAlgo),
+            .hash = std::move(hash)
+        });
     }
 
     /* Parse the list of input derivations. */
@@ -196,7 +178,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
     try {
         return parseDerivation(store, readFile(drvPath));
     } catch (FormatError & e) {
-        throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
+        throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
     }
 }
 
@@ -204,6 +186,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
 Derivation Store::derivationFromPath(const StorePath & drvPath)
 {
     ensurePath(drvPath);
+    return readDerivation(drvPath);
+}
+
+
+Derivation Store::readDerivation(const StorePath & drvPath)
+{
     auto accessor = getFSAccessor();
     try {
         return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
@@ -377,8 +365,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
         auto h = drvHashes.find(i.first);
         if (h == drvHashes.end()) {
             assert(store.isValidPath(i.first));
-            h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
-                readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first;
+            h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store,
+                store.readDerivation(i.first), false)).first;
         }
         inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
     }
@@ -405,11 +393,20 @@ StorePathSet BasicDerivation::outputPaths() const
 {
     StorePathSet paths;
     for (auto & i : outputs)
-        paths.insert(i.second.path.clone());
+        paths.insert(i.second.path);
     return paths;
 }
 
 
+StringSet BasicDerivation::outputNames() const
+{
+    StringSet names;
+    for (auto & i : outputs)
+        names.insert(i.first);
+    return names;
+}
+
+
 Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
 {
     drv.outputs.clear();
@@ -419,7 +416,11 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
         auto path = store.parseStorePath(readString(in));
         auto hashAlgo = readString(in);
         auto hash = readString(in);
-        drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
+        drv.outputs.emplace(name, DerivationOutput {
+            .path = std::move(path),
+            .hashAlgo = std::move(hashAlgo),
+            .hash = std::move(hash)
+        });
     }
 
     drv.inputSrcs = readStorePaths<StorePathSet>(store, in);

src/libstore/derivations.hh

@@ -17,12 +17,7 @@ struct DerivationOutput
     StorePath path;
     std::string hashAlgo; /* hash used for expected hash computation */
     std::string hash; /* expected hash, may be null */
-    DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash)
-        : path(std::move(path))
-        , hashAlgo(std::move(hashAlgo))
-        , hash(std::move(hash))
-    { }
-    void parseHashInfo(bool & recursive, Hash & hash) const;
+    void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -43,7 +38,6 @@ struct BasicDerivation
     StringPairs env;
 
     BasicDerivation() { }
-    explicit BasicDerivation(const BasicDerivation & other);
     virtual ~BasicDerivation() { };
 
     /* Return the path corresponding to the output identifier `id' in
@@ -58,6 +52,8 @@ struct BasicDerivation
     /* Return the output paths of a derivation. */
     StorePathSet outputPaths() const;
 
+    /* Return the output names of a derivation. */
+    StringSet outputNames() const;
 };
 
 struct Derivation : BasicDerivation
@@ -69,8 +65,6 @@ struct Derivation : BasicDerivation
         std::map<std::string, StringSet> * actualInputs = nullptr) const;
 
     Derivation() { }
-    Derivation(Derivation && other) = default;
-    explicit Derivation(const Derivation & other);
 };
 
 

src/libstore/export-import.cc

@@ -1,3 +1,4 @@
+#include "serialise.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "worker-protocol.hh"
@@ -56,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     Hash hash = hashAndWriteSink.currentHash();
     if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
-            printStorePath(path), info->narHash.to_string(), hash.to_string());
+            printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 
     hashAndWriteSink
         << exportMagic
@@ -100,9 +101,11 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (readInt(source) == 1)
             readString(source);
 
-        addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
+        // Can't use underlying source, which would have been exhausted
+        auto source = StringSource { *tee.source.data };
+        addToStore(info, source, NoRepair, checkSigs, accessor);
 
-        res.push_back(info.path.clone());
+        res.push_back(info.path);
     }
 
     return res;

src/libstore/filetransfer.cc

@@ -1,14 +1,10 @@
-#include "download.hh"
+#include "filetransfer.hh"
 #include "util.hh"
 #include "globals.hh"
-#include "hash.hh"
 #include "store-api.hh"
-#include "archive.hh"
 #include "s3.hh"
 #include "compression.hh"
-#include "pathlocks.hh"
 #include "finally.hh"
-#include "tarfile.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -31,13 +27,9 @@ using namespace std::string_literals;
 
 namespace nix {
 
-DownloadSettings downloadSettings;
+FileTransferSettings fileTransferSettings;
 
-static GlobalConfig::Register r1(&downloadSettings);
-
-CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
-    : uri(uri), ttl(settings.tarballTtl)
-{ }
+static GlobalConfig::Register r1(&fileTransferSettings);
 
 std::string resolveUri(const std::string & uri)
 {
@@ -47,21 +39,21 @@ std::string resolveUri(const std::string & uri)
         return uri;
 }
 
-struct CurlDownloader : public Downloader
+struct curlFileTransfer : public FileTransfer
 {
     CURLM * curlm = 0;
 
     std::random_device rd;
     std::mt19937 mt19937;
 
-    struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
+    struct TransferItem : public std::enable_shared_from_this<TransferItem>
     {
-        CurlDownloader & downloader;
-        DownloadRequest request;
-        DownloadResult result;
+        curlFileTransfer & fileTransfer;
+        FileTransferRequest request;
+        FileTransferResult result;
         Activity act;
         bool done = false; // whether either the success or failure function has been called
-        Callback<DownloadResult> callback;
+        Callback<FileTransferResult> callback;
         CURL * req = 0;
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
@@ -80,19 +72,36 @@ struct CurlDownloader : public Downloader
 
         curl_off_t writtenToSink = 0;
 
-        DownloadItem(CurlDownloader & downloader,
-            const DownloadRequest & request,
-            Callback<DownloadResult> && callback)
-            : downloader(downloader)
+        /* Get the HTTP status code, or 0 for other protocols. */
+        long getHTTPStatus()
+        {
+            long httpStatus = 0;
+            long protocol = 0;
+            curl_easy_getinfo(req, CURLINFO_PROTOCOL, &protocol);
+            if (protocol == CURLPROTO_HTTP || protocol == CURLPROTO_HTTPS)
+                curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+            return httpStatus;
+        }
+
+        TransferItem(curlFileTransfer & fileTransfer,
+            const FileTransferRequest & request,
+            Callback<FileTransferResult> && callback)
+            : fileTransfer(fileTransfer)
             , request(request)
-            , act(*logger, lvlTalkative, actDownload,
+            , act(*logger, lvlTalkative, actFileTransfer,
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
             , callback(std::move(callback))
             , finalSink([this](const unsigned char * data, size_t len) {
                 if (this->request.dataCallback) {
-                    writtenToSink += len;
-                    this->request.dataCallback((char *) data, len);
+                    auto httpStatus = getHTTPStatus();
+
+                    /* Only write data to the sink if this is a
+                       successful response. */
+                    if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
+                        writtenToSink += len;
+                        this->request.dataCallback((char *) data, len);
+                    }
                 } else
                     this->result.data->append((char *) data, len);
               })
@@ -103,17 +112,17 @@ struct CurlDownloader : public Downloader
                 requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
         }
 
-        ~DownloadItem()
+        ~TransferItem()
         {
             if (req) {
                 if (active)
-                    curl_multi_remove_handle(downloader.curlm, req);
+                    curl_multi_remove_handle(fileTransfer.curlm, req);
                 curl_easy_cleanup(req);
             }
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
+                    fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -157,7 +166,7 @@ struct CurlDownloader : public Downloader
 
         static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
         {
-            return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb);
+            return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
         }
 
         size_t headerCallback(void * contents, size_t size, size_t nmemb)
@@ -199,7 +208,7 @@ struct CurlDownloader : public Downloader
 
         static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
         {
-            return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb);
+            return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
         }
 
         int progressCallback(double dltotal, double dlnow)
@@ -214,7 +223,7 @@ struct CurlDownloader : public Downloader
 
         static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
         {
-            return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow);
+            return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
         }
 
         static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
@@ -238,7 +247,7 @@ struct CurlDownloader : public Downloader
 
         static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
         {
-            return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
+            return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
         }
 
         void init()
@@ -249,7 +258,7 @@ struct CurlDownloader : public Downloader
 
             if (verbosity >= lvlVomit) {
                 curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
-                curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback);
+                curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback);
             }
 
             curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
@@ -258,19 +267,19 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
             curl_easy_setopt(req, CURLOPT_USERAGENT,
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
-                    (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
+                    (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str());
             #if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
             #endif
             #if LIBCURL_VERSION_NUM >= 0x072f00
-            if (downloadSettings.enableHttp2)
+            if (fileTransferSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
             else
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
             #endif
-            curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
-            curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
 
             curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
@@ -298,10 +307,10 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
-            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
+            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
-            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
 
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
@@ -317,8 +326,7 @@ struct CurlDownloader : public Downloader
 
         void finish(CURLcode code)
         {
-            long httpStatus = 0;
-            curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+            auto httpStatus = getHTTPStatus();
 
             char * effectiveUriCStr;
             curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
@@ -345,7 +353,7 @@ struct CurlDownloader : public Downloader
                 failEx(writeException);
 
             else if (code == CURLE_OK &&
-                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
+                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
                 act.progress(result.bodySize, result.bodySize);
@@ -390,6 +398,7 @@ struct CurlDownloader : public Downloader
                         case CURLE_SSL_CACERT_BADFILE:
                         case CURLE_TOO_MANY_REDIRECTS:
                         case CURLE_WRITE_ERROR:
+                        case CURLE_UNSUPPORTED_PROTOCOL:
                             err = Misc;
                             break;
                         default: // Shut up warnings
@@ -401,14 +410,14 @@ struct CurlDownloader : public Downloader
 
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
+                    ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
                     : httpStatus != 0
-                    ? DownloadError(err,
+                    ? FileTransferError(err,
                         fmt("unable to %s '%s': HTTP error %d",
                             request.verb(), request.uri, httpStatus)
                         + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
                         )
-                    : DownloadError(err,
+                    : FileTransferError(err,
                         fmt("unable to %s '%s': %s (%d)",
                             request.verb(), request.uri, curl_easy_strerror(code), code));
 
@@ -422,13 +431,13 @@ struct CurlDownloader : public Downloader
                         || writtenToSink == 0
                         || (acceptRanges && encoding.empty())))
                 {
-                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
+                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
                     if (writtenToSink)
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else
                         warn("%s; retrying in %d ms", exc.what(), ms);
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
-                    downloader.enqueueItem(shared_from_this());
+                    fileTransfer.enqueueItem(shared_from_this());
                 }
                 else
                     fail(exc);
@@ -439,12 +448,12 @@ struct CurlDownloader : public Downloader
     struct State
     {
         struct EmbargoComparator {
-            bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
+            bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) {
                 return i1->embargo > i2->embargo;
             }
         };
         bool quit = false;
-        std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
+        std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming;
     };
 
     Sync<State> state_;
@@ -456,7 +465,7 @@ struct CurlDownloader : public Downloader
 
     std::thread workerThread;
 
-    CurlDownloader()
+    curlFileTransfer()
         : mt19937(rd())
     {
         static std::once_flag globalInit;
@@ -469,7 +478,7 @@ struct CurlDownloader : public Downloader
         #endif
         #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
-            downloadSettings.httpConnections.get());
+            fileTransferSettings.httpConnections.get());
         #endif
 
         wakeupPipe.create();
@@ -478,7 +487,7 @@ struct CurlDownloader : public Downloader
         workerThread = std::thread([&]() { workerThreadEntry(); });
     }
 
-    ~CurlDownloader()
+    ~curlFileTransfer()
     {
         stopWorkerThread();
 
@@ -504,7 +513,7 @@ struct CurlDownloader : public Downloader
             stopWorkerThread();
         });
 
-        std::map<CURL *, std::shared_ptr<DownloadItem>> items;
+        std::map<CURL *, std::shared_ptr<TransferItem>> items;
 
         bool quit = false;
 
@@ -517,7 +526,7 @@ struct CurlDownloader : public Downloader
             int running;
             CURLMcode mc = curl_multi_perform(curlm, &running);
             if (mc != CURLM_OK)
-                throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc));
+                throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
 
             /* Set the promises of any finished requests. */
             CURLMsg * msg;
@@ -547,7 +556,7 @@ struct CurlDownloader : public Downloader
             vomit("download thread waiting for %d ms", sleepTimeMs);
             mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)
-                throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc));
+                throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
 
             nextWakeup = std::chrono::steady_clock::time_point();
 
@@ -561,7 +570,7 @@ struct CurlDownloader : public Downloader
                     throw SysError("reading curl wakeup socket");
             }
 
-            std::vector<std::shared_ptr<DownloadItem>> incoming;
+            std::vector<std::shared_ptr<TransferItem>> incoming;
             auto now = std::chrono::steady_clock::now();
 
             {
@@ -599,7 +608,11 @@ struct CurlDownloader : public Downloader
             workerThreadMain();
         } catch (nix::Interrupted & e) {
         } catch (std::exception & e) {
-            printError("unexpected error in download thread: %s", e.what());
+            logError({
+                .name = "File transfer",
+                .hint = hintfmt("unexpected error in download thread: %s",
+                                e.what())
+            });
         }
 
         {
@@ -609,7 +622,7 @@ struct CurlDownloader : public Downloader
         }
     }
 
-    void enqueueItem(std::shared_ptr<DownloadItem> item)
+    void enqueueItem(std::shared_ptr<TransferItem> item)
     {
         if (item->request.data
             && !hasPrefix(item->request.uri, "http://")
@@ -641,8 +654,8 @@ struct CurlDownloader : public Downloader
     }
 #endif
 
-    void enqueueDownload(const DownloadRequest & request,
-        Callback<DownloadResult> callback) override
+    void enqueueFileTransfer(const FileTransferRequest & request,
+        Callback<FileTransferResult> callback) override
     {
         /* Ugly hack to support s3:// URIs. */
         if (hasPrefix(request.uri, "s3://")) {
@@ -660,9 +673,9 @@ struct CurlDownloader : public Downloader
 
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
-                DownloadResult res;
+                FileTransferResult res;
                 if (!s3Res.data)
-                    throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
+                    throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
                 res.data = s3Res.data;
                 callback(std::move(res));
 #else
@@ -672,26 +685,26 @@ struct CurlDownloader : public Downloader
             return;
         }
 
-        enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
+        enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
     }
 };
 
-ref<Downloader> getDownloader()
+ref<FileTransfer> getFileTransfer()
 {
-    static ref<Downloader> downloader = makeDownloader();
-    return downloader;
+    static ref<FileTransfer> fileTransfer = makeFileTransfer();
+    return fileTransfer;
 }
 
-ref<Downloader> makeDownloader()
+ref<FileTransfer> makeFileTransfer()
 {
-    return make_ref<CurlDownloader>();
+    return make_ref<curlFileTransfer>();
 }
 
-std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request)
+std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
 {
-    auto promise = std::make_shared<std::promise<DownloadResult>>();
-    enqueueDownload(request,
-        {[promise](std::future<DownloadResult> fut) {
+    auto promise = std::make_shared<std::promise<FileTransferResult>>();
+    enqueueFileTransfer(request,
+        {[promise](std::future<FileTransferResult> fut) {
             try {
                 promise->set_value(fut.get());
             } catch (...) {
@@ -701,15 +714,21 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
     return promise->get_future();
 }
 
-DownloadResult Downloader::download(const DownloadRequest & request)
+FileTransferResult FileTransfer::download(const FileTransferRequest & request)
 {
-    return enqueueDownload(request).get();
+    return enqueueFileTransfer(request).get();
 }
 
-void Downloader::download(DownloadRequest && request, Sink & sink)
+FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
+{
+    /* Note: this method is the same as download, but helps in readability */
+    return enqueueFileTransfer(request).get();
+}
+
+void FileTransfer::download(FileTransferRequest && request, Sink & sink)
 {
     /* Note: we can't call 'sink' via request.dataCallback, because
-       that would cause the sink to execute on the downloader
+       that would cause the sink to execute on the fileTransfer
        thread. If 'sink' is a coroutine, this will fail. Also, if the
        sink is expensive (e.g. one that does decompression and writing
        to the Nix store), it would stall the download thread too much.
@@ -755,8 +774,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
         state->avail.notify_one();
     };
 
-    enqueueDownload(request,
-        {[_state](std::future<DownloadResult> fut) {
+    enqueueFileTransfer(request,
+        {[_state](std::future<FileTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
             try {
@@ -801,140 +820,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     }
 }
 
-CachedDownloadResult Downloader::downloadCached(
-    ref<Store> store, const CachedDownloadRequest & request)
-{
-    auto url = resolveUri(request.uri);
-
-    auto name = request.name;
-    if (name == "") {
-        auto p = url.rfind('/');
-        if (p != string::npos) name = string(url, p + 1);
-    }
-
-    std::optional<StorePath> expectedStorePath;
-    if (request.expectedHash) {
-        expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
-        if (store->isValidPath(*expectedStorePath)) {
-            CachedDownloadResult result;
-            result.storePath = store->printStorePath(*expectedStorePath);
-            result.path = store->toRealPath(result.storePath);
-            return result;
-        }
-    }
-
-    Path cacheDir = getCacheDir() + "/nix/tarballs";
-    createDirs(cacheDir);
-
-    string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
-
-    Path dataFile = cacheDir + "/" + urlHash + ".info";
-    Path fileLink = cacheDir + "/" + urlHash + "-file";
-
-    PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
-
-    std::optional<StorePath> storePath;
-
-    string expectedETag;
-
-    bool skip = false;
-
-    CachedDownloadResult result;
-
-    if (pathExists(fileLink) && pathExists(dataFile)) {
-        storePath = store->parseStorePath(readLink(fileLink));
-        // FIXME
-        store->addTempRoot(*storePath);
-        if (store->isValidPath(*storePath)) {
-            auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
-            if (ss.size() >= 3 && ss[0] == url) {
-                time_t lastChecked;
-                if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
-                    skip = true;
-                    result.effectiveUri = request.uri;
-                    result.etag = ss[1];
-                } else if (!ss[1].empty()) {
-                    debug(format("verifying previous ETag '%1%'") % ss[1]);
-                    expectedETag = ss[1];
-                }
-            }
-        } else
-            storePath.reset();
-    }
-
-    if (!skip) {
-
-        try {
-            DownloadRequest request2(url);
-            request2.expectedETag = expectedETag;
-            auto res = download(request2);
-            result.effectiveUri = res.effectiveUri;
-            result.etag = res.etag;
-
-            if (!res.cached) {
-                StringSink sink;
-                dumpString(*res.data, sink);
-                Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
-                ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
-                info.narHash = hashString(htSHA256, *sink.s);
-                info.narSize = sink.s->size();
-                info.ca = makeFixedOutputCA(false, hash);
-                store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
-                storePath = info.path.clone();
-            }
-
-            assert(storePath);
-            replaceSymlink(store->printStorePath(*storePath), fileLink);
-
-            writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
-        } catch (DownloadError & e) {
-            if (!storePath) throw;
-            warn("warning: %s; using cached result", e.msg());
-            result.etag = expectedETag;
-        }
-    }
-
-    if (request.unpack) {
-        Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
-        PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
-        std::optional<StorePath> unpackedStorePath;
-        if (pathExists(unpackedLink)) {
-            unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
-            // FIXME
-            store->addTempRoot(*unpackedStorePath);
-            if (!store->isValidPath(*unpackedStorePath))
-                unpackedStorePath.reset();
-        }
-        if (!unpackedStorePath) {
-            printInfo("unpacking '%s'...", url);
-            Path tmpDir = createTempDir();
-            AutoDelete autoDelete(tmpDir, true);
-            unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir);
-            auto members = readDirectory(tmpDir);
-            if (members.size() != 1)
-                throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
-            auto topDir = tmpDir + "/" + members.begin()->name;
-            unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
-        }
-        replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
-        storePath = std::move(*unpackedStorePath);
-    }
-
-    if (expectedStorePath && *storePath != *expectedStorePath) {
-        unsigned int statusCode = 102;
-        Hash gotHash = request.unpack
-            ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
-            : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
-        throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-            url, request.expectedHash.to_string(), gotHash.to_string());
-    }
-
-    result.storePath = store->printStorePath(*storePath);
-    result.path = store->toRealPath(result.storePath);
-    return result;
-}
-
-
 bool isUri(const string & s)
 {
     if (s.compare(0, 8, "channel:") == 0) return true;

src/libstore/filetransfer.hh

@@ -9,7 +9,7 @@
 
 namespace nix {
 
-struct DownloadSettings : Config
+struct FileTransferSettings : Config
 {
     Setting<bool> enableHttp2{this, true, "http2",
         "Whether to enable HTTP/2 support."};
@@ -31,15 +31,15 @@ struct DownloadSettings : Config
         "How often Nix will attempt to download a file before giving up."};
 };
 
-extern DownloadSettings downloadSettings;
+extern FileTransferSettings fileTransferSettings;
 
-struct DownloadRequest
+struct FileTransferRequest
 {
     std::string uri;
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
-    size_t tries = downloadSettings.tries;
+    size_t tries = fileTransferSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
@@ -47,7 +47,7 @@ struct DownloadRequest
     std::string mimeType;
     std::function<void(char *, size_t)> dataCallback;
 
-    DownloadRequest(const std::string & uri)
+    FileTransferRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
     std::string verb()
@@ -56,7 +56,7 @@ struct DownloadRequest
     }
 };
 
-struct DownloadResult
+struct FileTransferResult
 {
     bool cached = false;
     std::string etag;
@@ -65,74 +65,53 @@ struct DownloadResult
     uint64_t bodySize = 0;
 };
 
-struct CachedDownloadRequest
-{
-    std::string uri;
-    bool unpack = false;
-    std::string name;
-    Hash expectedHash;
-    unsigned int ttl;
-
-    CachedDownloadRequest(const std::string & uri);
-    CachedDownloadRequest() = delete;
-};
-
-struct CachedDownloadResult
-{
-    // Note: 'storePath' may be different from 'path' when using a
-    // chroot store.
-    Path storePath;
-    Path path;
-    std::optional<std::string> etag;
-    std::string effectiveUri;
-};
-
 class Store;
 
-struct Downloader
+struct FileTransfer
 {
-    virtual ~Downloader() { }
+    virtual ~FileTransfer() { }
 
-    /* Enqueue a download request, returning a future to the result of
-       the download. The future may throw a DownloadError
+    /* Enqueue a data transfer request, returning a future to the result of
+       the download. The future may throw a FileTransferError
        exception. */
-    virtual void enqueueDownload(const DownloadRequest & request,
-        Callback<DownloadResult> callback) = 0;
+    virtual void enqueueFileTransfer(const FileTransferRequest & request,
+        Callback<FileTransferResult> callback) = 0;
 
-    std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
+    std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
 
     /* Synchronously download a file. */
-    DownloadResult download(const DownloadRequest & request);
+    FileTransferResult download(const FileTransferRequest & request);
+
+    /* Synchronously upload a file. */
+    FileTransferResult upload(const FileTransferRequest & request);
 
     /* Download a file, writing its data to a sink. The sink will be
        invoked on the thread of the caller. */
-    void download(DownloadRequest && request, Sink & sink);
-
-    /* Check if the specified file is already in ~/.cache/nix/tarballs
-       and is more recent than ‘tarball-ttl’ seconds. Otherwise,
-       use the recorded ETag to verify if the server has a more
-       recent version, and if so, download it to the Nix store. */
-    CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
+    void download(FileTransferRequest && request, Sink & sink);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
 
-/* Return a shared Downloader object. Using this object is preferred
+/* Return a shared FileTransfer object. Using this object is preferred
    because it enables connection reuse and HTTP/2 multiplexing. */
-ref<Downloader> getDownloader();
+ref<FileTransfer> getFileTransfer();
 
-/* Return a new Downloader object. */
-ref<Downloader> makeDownloader();
+/* Return a new FileTransfer object. */
+ref<FileTransfer> makeFileTransfer();
 
-class DownloadError : public Error
+class FileTransferError : public Error
 {
 public:
-    Downloader::Error error;
-    DownloadError(Downloader::Error error, const FormatOrString & fs)
-        : Error(fs), error(error)
+    FileTransfer::Error error;
+    template<typename... Args>
+    FileTransferError(FileTransfer::Error error, const Args & ... args)
+        : Error(args...), error(error)
     { }
 };
 
 bool isUri(const string & s);
 
+/* Resolve deprecated 'channel:<foo>' URLs. */
+std::string resolveUri(const std::string & uri);
+
 }

src/libstore/gc.cc

@@ -38,10 +38,10 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
 
     AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
     if (!fdGCLock)
-        throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
+        throw SysError("opening global GC lock '%1%'", fnGCLock);
 
     if (!lockFile(fdGCLock.get(), lockType, false)) {
-        printError(format("waiting for the big garbage collector lock..."));
+        printInfo("waiting for the big garbage collector lock...");
         lockFile(fdGCLock.get(), lockType, true);
     }
 
@@ -65,8 +65,8 @@ static void makeSymlink(const Path & link, const Path & target)
 
     /* Atomically replace the old one. */
     if (rename(tempLink.c_str(), link.c_str()) == -1)
-        throw SysError(format("cannot rename '%1%' to '%2%'")
-            % tempLink % link);
+        throw SysError("cannot rename '%1%' to '%2%'",
+            tempLink , link);
 }
 
 
@@ -91,15 +91,15 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
     Path gcRoot(canonPath(_gcRoot));
 
     if (isInStore(gcRoot))
-        throw Error(format(
+        throw Error(
                 "creating a garbage collector root (%1%) in the Nix store is forbidden "
-                "(are you running nix-build inside the store?)") % gcRoot);
+                "(are you running nix-build inside the store?)", gcRoot);
 
     if (indirect) {
         /* Don't clobber the link if it already exists and doesn't
            point to the Nix store. */
         if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
-            throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
+            throw Error("cannot create symlink '%1%'; already exists", gcRoot);
         makeSymlink(gcRoot, printStorePath(storePath));
         addIndirectRoot(gcRoot);
     }
@@ -109,10 +109,10 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
             Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
 
             if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
-                throw Error(format(
+                throw Error(
                     "path '%1%' is not a valid garbage collector root; "
-                    "it's not in the directory '%2%'")
-                    % gcRoot % rootsDir);
+                    "it's not in the directory '%2%'",
+                    gcRoot, rootsDir);
         }
 
         if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
@@ -128,11 +128,13 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
        gcroots directory. */
     if (settings.checkRootReachability) {
         auto roots = findRoots(false);
-        if (roots[storePath.clone()].count(gcRoot) == 0)
-            printError(
-                "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
-                "therefore, '%2%' might be removed by the garbage collector",
-                gcRoot, printStorePath(storePath));
+        if (roots[storePath].count(gcRoot) == 0)
+            logWarning({
+                .name = "GC root",
+                .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
+                    "therefore, '%2%' might be removed by the garbage collector",
+                    gcRoot, printStorePath(storePath))
+            });
     }
 
     /* Grab the global GC root, causing us to block while a GC is in
@@ -170,7 +172,7 @@ void LocalStore::addTempRoot(const StorePath & path)
                way. */
             struct stat st;
             if (fstat(state->fdTempRoots.get(), &st) == -1)
-                throw SysError(format("statting '%1%'") % fnTempRoots);
+                throw SysError("statting '%1%'", fnTempRoots);
             if (st.st_size == 0) break;
 
             /* The garbage collector deleted this file before we could
@@ -202,6 +204,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
     /* Read the `temproots' directory for per-process temporary root
        files. */
     for (auto & i : readDirectory(tempRootsDir)) {
+        if (i.name[0] == '.') {
+            // Ignore hidden files. Some package managers (notably portage) create
+            // those to keep the directory alive.
+            continue;
+        }
         Path path = tempRootsDir + "/" + i.name;
 
         pid_t pid = std::stoi(i.name);
@@ -211,7 +218,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
         if (!*fd) {
             /* It's okay if the file has disappeared. */
             if (errno == ENOENT) continue;
-            throw SysError(format("opening temporary roots file '%1%'") % path);
+            throw SysError("opening temporary roots file '%1%'", path);
         }
 
         /* This should work, but doesn't, for some reason. */
@@ -222,7 +229,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
         if (lockFile(fd->get(), ltWrite, false)) {
-            printError(format("removing stale temporary roots file '%1%'") % path);
+            printInfo("removing stale temporary roots file '%1%'", path);
             unlink(path.c_str());
             writeFull(fd->get(), "d");
             continue;
@@ -398,7 +405,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                 if (!fdDir) {
                     if (errno == ENOENT || errno == EACCES)
                         continue;
-                    throw SysError(format("opening %1%") % fdStr);
+                    throw SysError("opening %1%", fdStr);
                 }
                 struct dirent * fd_ent;
                 while (errno = 0, fd_ent = readdir(fdDir.get())) {
@@ -408,13 +415,13 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                 if (errno) {
                     if (errno == ESRCH)
                         continue;
-                    throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
+                    throw SysError("iterating /proc/%1%/fd", ent->d_name);
                 }
                 fdDir.reset();
 
                 try {
                     auto mapFile = fmt("/proc/%s/maps", ent->d_name);
-                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
+                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n");
                     for (const auto & line : mapLines) {
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
@@ -422,7 +429,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                     }
 
                     auto envFile = fmt("/proc/%s/environ", ent->d_name);
-                    auto envString = readFile(envFile, true);
+                    auto envString = readFile(envFile);
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
                         unchecked[i->str()].emplace(envFile);
@@ -471,9 +478,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
         if (!isValidPath(path)) continue;
         debug("got additional root '%1%'", pathS);
         if (censor)
-            roots[path.clone()].insert(censored);
+            roots[path].insert(censored);
         else
-            roots[path.clone()].insert(links.begin(), links.end());
+            roots[path].insert(links.begin(), links.end());
     }
 }
 
@@ -536,7 +543,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
     struct stat st;
     if (lstat(realPath.c_str(), &st)) {
         if (errno == ENOENT) return;
-        throw SysError(format("getting status of %1%") % realPath);
+        throw SysError("getting status of %1%", realPath);
     }
 
     printInfo(format("deleting '%1%'") % path);
@@ -554,10 +561,10 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         // size.
         try {
             if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
-                throw SysError(format("making '%1%' writable") % realPath);
+                throw SysError("making '%1%' writable", realPath);
             Path tmp = trashDir + "/" + std::string(baseNameOf(path));
             if (rename(realPath.c_str(), tmp.c_str()))
-                throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
+                throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
             state.bytesInvalidated += size;
         } catch (SysError & e) {
             if (e.errNo == ENOSPC) {
@@ -585,11 +592,11 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
 
     if (state.roots.count(path)) {
         debug("cannot delete '%1%' because it's a root", printStorePath(path));
-        state.alive.insert(path.clone());
+        state.alive.insert(path);
         return true;
     }
 
-    visited.insert(path.clone());
+    visited.insert(path);
 
     if (!isValidPath(path)) return false;
 
@@ -603,7 +610,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
     if (state.gcKeepDerivations && path.isDerivation()) {
         for (auto & i : queryDerivationOutputs(path))
             if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                incoming.insert(i.clone());
+                incoming.insert(i);
     }
 
     /* If keep-outputs is set, then don't delete this path if there
@@ -611,13 +618,13 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
     if (state.gcKeepOutputs) {
         auto derivers = queryValidDerivers(path);
         for (auto & i : derivers)
-            incoming.insert(i.clone());
+            incoming.insert(i);
     }
 
     for (auto & i : incoming)
         if (i != path)
             if (canReachRoot(state, visited, i)) {
-                state.alive.insert(path.clone());
+                state.alive.insert(path);
                 return true;
             }
 
@@ -661,7 +668,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
            ‘nix-store --delete’ doesn't have the unexpected effect of
            recursing into derivations and outputs. */
         for (auto & i : visited)
-            state.dead.insert(i.clone());
+            state.dead.insert(i);
         if (state.shouldDelete)
             deletePathRecursive(state, path);
     }
@@ -676,7 +683,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 void LocalStore::removeUnusedLinks(const GCState & state)
 {
     AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
+    if (!dir) throw SysError("opening directory '%1%'", linksDir);
 
     long long actualSize = 0, unsharedSize = 0;
 
@@ -689,7 +696,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
 
         struct stat st;
         if (lstat(path.c_str(), &st) == -1)
-            throw SysError(format("statting '%1%'") % path);
+            throw SysError("statting '%1%'", path);
 
         if (st.st_nlink != 1) {
             actualSize += st.st_size;
@@ -700,14 +707,14 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
 
         if (unlink(path.c_str()) == -1)
-            throw SysError(format("deleting '%1%'") % path);
+            throw SysError("deleting '%1%'", path);
 
         state.results.bytesFreed += st.st_size;
     }
 
     struct stat st;
     if (stat(linksDir.c_str(), &st) == -1)
-        throw SysError(format("statting '%1%'") % linksDir);
+        throw SysError("statting '%1%'", linksDir);
     long long overhead = st.st_blocks * 512ULL;
 
     printInfo(format("note: currently hard linking saves %.2f MiB")
@@ -742,12 +749,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    printError("finding garbage collector roots...");
+    printInfo("finding garbage collector roots...");
     Roots rootMap;
     if (!options.ignoreLiveness)
         findRootsNoTemp(rootMap, true);
 
-    for (auto & i : rootMap) state.roots.insert(i.first.clone());
+    for (auto & i : rootMap) state.roots.insert(i.first);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
@@ -756,8 +763,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     Roots tempRoots;
     findTempRoots(fds, tempRoots, true);
     for (auto & root : tempRoots) {
-        state.tempRoots.insert(root.first.clone());
-        state.roots.insert(root.first.clone());
+        state.tempRoots.insert(root.first);
+        state.roots.insert(root.first);
     }
 
     /* After this point the set of roots or temporary roots cannot
@@ -794,14 +801,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     } else if (options.maxFreed > 0) {
 
         if (state.shouldDelete)
-            printError("deleting garbage...");
+            printInfo("deleting garbage...");
         else
-            printError("determining live/dead paths...");
+            printInfo("determining live/dead paths...");
 
         try {
 
             AutoCloseDir dir(opendir(realStoreDir.c_str()));
-            if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir);
+            if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
 
             /* Read the store and immediately delete all paths that
                aren't valid.  When using --max-freed etc., deleting
@@ -863,7 +870,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Clean up the links directory. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
-        printError("deleting unused links...");
+        printInfo("deleting unused links...");
         removeUnusedLinks(state);
     }
 
@@ -884,7 +891,7 @@ void LocalStore::autoGC(bool sync)
         if (statvfs(realStoreDir.c_str(), &st))
             throw SysError("getting filesystem info about '%s'", realStoreDir);
 
-        return (uint64_t) st.f_bavail * st.f_bsize;
+        return (uint64_t) st.f_bavail * st.f_frsize;
     };
 
     std::shared_future<void> future;

src/libstore/globals.cc

@@ -31,6 +31,7 @@ Settings::Settings()
     , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
     , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
     , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
+    , nixUserConfFiles(getUserConfigFiles())
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
@@ -77,13 +78,29 @@ void loadConfFile()
        ~/.nix/nix.conf or the command line. */
     globalConfig.resetOverriden();
 
-    auto dirs = getConfigDirs();
-    // Iterate over them in reverse so that the ones appearing first in the path take priority
-    for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
-        globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
+    auto files = settings.nixUserConfFiles;
+    for (auto file = files.rbegin(); file != files.rend(); file++) {
+        globalConfig.applyConfigFile(*file);
     }
 }
 
+std::vector<Path> getUserConfigFiles()
+{
+    // Use the paths specified in NIX_USER_CONF_FILES if it has been defined
+    auto nixConfFiles = getEnv("NIX_USER_CONF_FILES");
+    if (nixConfFiles.has_value()) {
+        return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":");
+    }
+
+    // Use the paths specified by the XDG spec
+    std::vector<Path> files;
+    auto dirs = getConfigDirs();
+    for (auto & dir : dirs) {
+        files.insert(files.end(), dir + "/nix/nix.conf");
+    }
+    return files;
+}
+
 unsigned int Settings::getDefaultCores()
 {
     return std::max(1U, std::thread::hardware_concurrency());
@@ -113,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
 void Settings::requireExperimentalFeature(const std::string & name)
 {
     if (!isExperimentalFeatureEnabled(name))
-        throw Error("experimental Nix feature '%s' is disabled", name);
+        throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
 }
 
 bool Settings::isWSL1()
@@ -150,21 +167,24 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
 
 template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description("Enable sandboxing.")
-        .handler([=](std::vector<std::string> ss) { override(smEnabled); })
-        .category(category);
-    args.mkFlag()
-        .longName("no-" + name)
-        .description("Disable sandboxing.")
-        .handler([=](std::vector<std::string> ss) { override(smDisabled); })
-        .category(category);
-    args.mkFlag()
-        .longName("relaxed-" + name)
-        .description("Enable sandboxing, but allow builds to disable it.")
-        .handler([=](std::vector<std::string> ss) { override(smRelaxed); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = "Enable sandboxing.",
+        .category = category,
+        .handler = {[=]() { override(smEnabled); }}
+    });
+    args.addFlag({
+        .longName = "no-" + name,
+        .description = "Disable sandboxing.",
+        .category = category,
+        .handler = {[=]() { override(smDisabled); }}
+    });
+    args.addFlag({
+        .longName = "relaxed-" + name,
+        .description = "Enable sandboxing, but allow builds to disable it.",
+        .category = category,
+        .handler = {[=]() { override(smRelaxed); }}
+    });
 }
 
 void MaxBuildJobsSetting::set(const std::string & str)

src/libstore/globals.hh

@@ -53,9 +53,12 @@ public:
     /* The directory where state is stored. */
     Path nixStateDir;
 
-    /* The directory where configuration files are stored. */
+    /* The directory where system configuration files are stored. */
     Path nixConfDir;
 
+    /* A list of user configuration files to load. */
+    std::vector<Path> nixUserConfFiles;
+
     /* The directory where internal helper programs are stored. */
     Path nixLibexecDir;
 
@@ -268,7 +271,7 @@ public:
         "listed in 'trusted-public-keys'."};
 
     Setting<StringSet> extraPlatforms{this,
-        std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
+        std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
         "extra-platforms",
         "Additional platforms that can be built on the local system. "
         "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
@@ -351,12 +354,21 @@ public:
     Setting<Paths> pluginFiles{this, {}, "plugin-files",
         "Plugins to dynamically load at nix initialization time."};
 
+    Setting<std::string> githubAccessToken{this, "", "github-access-token",
+        "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
+
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
     bool isExperimentalFeatureEnabled(const std::string & name);
 
     void requireExperimentalFeature(const std::string & name);
+
+    Setting<bool> allowDirty{this, true, "allow-dirty",
+        "Whether to allow dirty Git/Mercurial trees."};
+
+    Setting<bool> warnDirty{this, true, "warn-dirty",
+        "Whether to warn about dirty Git/Mercurial trees."};
 };
 
 
@@ -369,6 +381,9 @@ void initPlugins();
 
 void loadConfFile();
 
+// Used by the Settings constructor
+std::vector<Path> getUserConfigFiles();
+
 extern const string nixVersion;
 
 }

src/libstore/http-binary-cache-store.cc

@@ -1,5 +1,5 @@
 #include "binary-cache-store.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
 
@@ -85,14 +85,14 @@ protected:
         checkEnabled();
 
         try {
-            DownloadRequest request(cacheUri + "/" + path);
+            FileTransferRequest request(cacheUri + "/" + path);
             request.head = true;
-            getDownloader()->download(request);
+            getFileTransfer()->download(request);
             return true;
-        } catch (DownloadError & e) {
+        } catch (FileTransferError & e) {
             /* S3 buckets return 403 if a file doesn't exist and the
                bucket is unlistable, so treat 403 as 404. */
-            if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+            if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                 return false;
             maybeDisable();
             throw;
@@ -103,19 +103,19 @@ protected:
         const std::string & data,
         const std::string & mimeType) override
     {
-        auto req = DownloadRequest(cacheUri + "/" + path);
+        auto req = FileTransferRequest(cacheUri + "/" + path);
         req.data = std::make_shared<string>(data); // FIXME: inefficient
         req.mimeType = mimeType;
         try {
-            getDownloader()->download(req);
-        } catch (DownloadError & e) {
+            getFileTransfer()->upload(req);
+        } catch (FileTransferError & e) {
             throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
         }
     }
 
-    DownloadRequest makeRequest(const std::string & path)
+    FileTransferRequest makeRequest(const std::string & path)
     {
-        DownloadRequest request(cacheUri + "/" + path);
+        FileTransferRequest request(cacheUri + "/" + path);
         return request;
     }
 
@@ -124,9 +124,9 @@ protected:
         checkEnabled();
         auto request(makeRequest(path));
         try {
-            getDownloader()->download(std::move(request), sink);
-        } catch (DownloadError & e) {
-            if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+            getFileTransfer()->download(std::move(request), sink);
+        } catch (FileTransferError & e) {
+            if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
             maybeDisable();
             throw;
@@ -142,12 +142,12 @@ protected:
 
         auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
-        getDownloader()->enqueueDownload(request,
-            {[callbackPtr, this](std::future<DownloadResult> result) {
+        getFileTransfer()->enqueueFileTransfer(request,
+            {[callbackPtr, this](std::future<FileTransferResult> result) {
                 try {
                     (*callbackPtr)(result.get().data);
-                } catch (DownloadError & e) {
-                    if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+                } catch (FileTransferError & e) {
+                    if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                         return (*callbackPtr)(std::shared_ptr<std::string>());
                     maybeDisable();
                     callbackPtr->rethrow();
@@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([](
 });
 
 }
-

src/libstore/legacy-ssh-store.cc

@@ -195,7 +195,7 @@ struct LegacySSHStore : public Store
     { unsupported("queryPathFromHashPart"); }
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive, HashType hashAlgo,
+        FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override
     { unsupported("addToStore"); }
 
@@ -256,7 +256,7 @@ struct LegacySSHStore : public Store
         conn->to.flush();
 
         for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
-            out.insert(i.clone());
+            out.insert(i);
     }
 
     StorePathSet queryValidPaths(const StorePathSet & paths,

src/libstore/local-binary-cache-store.cc

@@ -74,7 +74,7 @@ static void atomicWrite(const Path & path, const std::string & s)
     AutoDelete del(tmp, false);
     writeFile(tmp, s);
     if (rename(tmp.c_str(), path.c_str()))
-        throw SysError(format("renaming '%1%' to '%2%'") % tmp % path);
+        throw SysError("renaming '%1%' to '%2%'", tmp, path);
     del.cancel();
 }
 

src/libstore/local-fs-store.cc

@@ -22,7 +22,7 @@ struct LocalStoreAccessor : public FSAccessor
     {
         Path storePath = store->toStorePath(path);
         if (!store->isValidPath(store->parseStorePath(storePath)))
-            throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
+            throw InvalidPath("path '%1%' is not a valid store path", storePath);
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 
@@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor
         struct stat st;
         if (lstat(realPath.c_str(), &st)) {
             if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
-            throw SysError(format("getting status of '%1%'") % path);
+            throw SysError("getting status of '%1%'", path);
         }
 
         if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
-            throw Error(format("file '%1%' has unsupported type") % path);
+            throw Error("file '%1%' has unsupported type", path);
 
         return {
             S_ISREG(st.st_mode) ? Type::tRegular :
@@ -90,13 +90,13 @@ const string LocalFSStore::drvsLogDir = "drvs";
 
 std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
 {
-    auto path = path_.clone();
+    auto path = path_;
 
     if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
             if (!info->deriver) return nullptr;
-            path = info->deriver->clone();
+            path = *info->deriver;
         } catch (InvalidPath &) {
             return nullptr;
         }

src/libstore/local-store.cc

@@ -87,18 +87,22 @@ LocalStore::LocalStore(const Params & params)
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
         if (!gr)
-            printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist")
-                % settings.buildUsersGroup);
+            logError({
+                .name = "'build-users-group' not found",
+                .hint = hintfmt(
+                    "warning: the group '%1%' specified in 'build-users-group' does not exist",
+                    settings.buildUsersGroup)
+            });
         else {
             struct stat st;
             if (stat(realStoreDir.c_str(), &st))
-                throw SysError(format("getting attributes of path '%1%'") % realStoreDir);
+                throw SysError("getting attributes of path '%1%'", realStoreDir);
 
             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
                 if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
-                    throw SysError(format("changing ownership of path '%1%'") % realStoreDir);
+                    throw SysError("changing ownership of path '%1%'", realStoreDir);
                 if (chmod(realStoreDir.c_str(), perm) == -1)
-                    throw SysError(format("changing permissions on path '%1%'") % realStoreDir);
+                    throw SysError("changing permissions on path '%1%'", realStoreDir);
             }
         }
     }
@@ -109,12 +113,12 @@ LocalStore::LocalStore(const Params & params)
         struct stat st;
         while (path != "/") {
             if (lstat(path.c_str(), &st))
-                throw SysError(format("getting status of '%1%'") % path);
+                throw SysError("getting status of '%1%'", path);
             if (S_ISLNK(st.st_mode))
-                throw Error(format(
+                throw Error(
                         "the path '%1%' is a symlink; "
-                        "this is not allowed for the Nix store and its parent directories")
-                    % path);
+                        "this is not allowed for the Nix store and its parent directories",
+                        path);
             path = dirOf(path);
         }
     }
@@ -147,7 +151,7 @@ LocalStore::LocalStore(const Params & params)
     globalLock = openLockFile(globalLockPath.c_str(), true);
 
     if (!lockFile(globalLock.get(), ltRead, false)) {
-        printError("waiting for the big Nix store lock...");
+        printInfo("waiting for the big Nix store lock...");
         lockFile(globalLock.get(), ltRead, true);
     }
 
@@ -155,8 +159,8 @@ LocalStore::LocalStore(const Params & params)
        upgrade.  */
     int curSchema = getSchema();
     if (curSchema > nixSchemaVersion)
-        throw Error(format("current Nix store schema is version %1%, but I only support %2%")
-            % curSchema % nixSchemaVersion);
+        throw Error("current Nix store schema is version %1%, but I only support %2%",
+             curSchema, nixSchemaVersion);
 
     else if (curSchema == 0) { /* new store */
         curSchema = nixSchemaVersion;
@@ -178,7 +182,7 @@ LocalStore::LocalStore(const Params & params)
                 "please upgrade Nix to version 1.11 first.");
 
         if (!lockFile(globalLock.get(), ltWrite, false)) {
-            printError("waiting for exclusive access to the Nix store...");
+            printInfo("waiting for exclusive access to the Nix store...");
             lockFile(globalLock.get(), ltWrite, true);
         }
 
@@ -256,7 +260,7 @@ LocalStore::~LocalStore()
     }
 
     if (future.valid()) {
-        printError("waiting for auto-GC to finish on exit...");
+        printInfo("waiting for auto-GC to finish on exit...");
         future.get();
     }
 
@@ -284,7 +288,7 @@ int LocalStore::getSchema()
     if (pathExists(schemaPath)) {
         string s = readFile(schemaPath);
         if (!string2Int(s, curSchema))
-            throw Error(format("'%1%' is corrupt") % schemaPath);
+            throw Error("'%1%' is corrupt", schemaPath);
     }
     return curSchema;
 }
@@ -293,7 +297,7 @@ int LocalStore::getSchema()
 void LocalStore::openDB(State & state, bool create)
 {
     if (access(dbDir.c_str(), R_OK | W_OK))
-        throw SysError(format("Nix database directory '%1%' is not writable") % dbDir);
+        throw SysError("Nix database directory '%1%' is not writable", dbDir);
 
     /* Open the Nix database. */
     string dbPath = dbDir + "/db.sqlite";
@@ -367,7 +371,7 @@ void LocalStore::makeStoreWritable()
             throw SysError("setting up a private mount namespace");
 
         if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
-            throw SysError(format("remounting %1% writable") % realStoreDir);
+            throw SysError("remounting %1% writable", realStoreDir);
     }
 #endif
 }
@@ -388,7 +392,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
                  | 0444
                  | (st.st_mode & S_IXUSR ? 0111 : 0);
             if (chmod(path.c_str(), mode) == -1)
-                throw SysError(format("changing mode of '%1%' to %2$o") % path % mode);
+                throw SysError("changing mode of '%1%' to %2$o", path, mode);
         }
 
     }
@@ -406,7 +410,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
 #else
         if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
 #endif
-            throw SysError(format("changing modification time of '%1%'") % path);
+            throw SysError("changing modification time of '%1%'", path);
     }
 }
 
@@ -415,7 +419,7 @@ void canonicaliseTimestampAndPermissions(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
     canonicaliseTimestampAndPermissions(path, st);
 }
 
@@ -430,17 +434,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
        setattrlist() to remove other attributes as well. */
     if (lchflags(path.c_str(), 0)) {
         if (errno != ENOTSUP)
-            throw SysError(format("clearing flags of path '%1%'") % path);
+            throw SysError("clearing flags of path '%1%'", path);
     }
 #endif
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     /* Really make sure that the path is of a supported type. */
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
-        throw Error(format("file '%1%' has an unsupported type") % path);
+        throw Error("file '%1%' has an unsupported type", path);
 
 #if __linux__
     /* Remove extended attributes / ACLs. */
@@ -474,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
         assert(!S_ISDIR(st.st_mode));
         if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
-            throw BuildError(format("invalid ownership on file '%1%'") % path);
+            throw BuildError("invalid ownership on file '%1%'", path);
         mode_t mode = st.st_mode & ~S_IFMT;
         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
         return;
@@ -498,8 +502,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
         if (!S_ISLNK(st.st_mode) &&
             chown(path.c_str(), geteuid(), getegid()) == -1)
 #endif
-            throw SysError(format("changing owner of '%1%' to %2%")
-                % path % geteuid());
+            throw SysError("changing owner of '%1%' to %2%",
+                path, geteuid());
     }
 
     if (S_ISDIR(st.st_mode)) {
@@ -518,11 +522,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
        be a symlink, since we can't change its ownership. */
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     if (st.st_uid != geteuid()) {
         assert(S_ISLNK(st.st_mode));
-        throw Error(format("wrong ownership of top-level store path '%1%'") % path);
+        throw Error("wrong ownership of top-level store path '%1%'", path);
     }
 }
 
@@ -557,10 +561,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
         if (out == drv.outputs.end())
             throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
 
-        bool recursive; Hash h;
-        out->second.parseHashInfo(recursive, h);
+        FileIngestionMethod method; Hash h;
+        out->second.parseHashInfo(method, h);
 
-        check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
+        check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
     }
 
     else {
@@ -580,7 +584,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     state.stmtRegisterValidPath.use()
         (printStorePath(info.path))
-        (info.narHash.to_string(Base16))
+        (info.narHash.to_string(Base16, true))
         (info.registrationTime == 0 ? time(0) : info.registrationTime)
         (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
         (info.narSize, info.narSize != 0)
@@ -595,7 +599,7 @@ uint64_t LocalStore::addValidPath(State & state,
        efficiently query whether a path is an output of some
        derivation. */
     if (info.path.isDerivation()) {
-        auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string()));
+        auto drv = readDerivation(info.path);
 
         /* Verify that the output paths in the derivation are correct
            (i.e., follow the scheme for computing output paths from
@@ -615,7 +619,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)),
+        state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
             PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
     }
 
@@ -627,7 +631,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        auto info = std::make_shared<ValidPathInfo>(path.clone());
+        auto info = std::make_shared<ValidPathInfo>(path);
 
         callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
             auto state(_state.lock());
@@ -680,7 +684,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 {
     state.stmtUpdatePathInfo.use()
         (info.narSize, info.narSize != 0)
-        (info.narHash.to_string(Base16))
+        (info.narHash.to_string(Base16, true))
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
         (info.ca, !info.ca.empty())
@@ -717,7 +721,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF
 {
     StorePathSet res;
     for (auto & i : paths)
-        if (isValidPath(i)) res.insert(i.clone());
+        if (isValidPath(i)) res.insert(i);
     return res;
 }
 
@@ -785,26 +789,9 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
 }
 
 
-StringSet LocalStore::queryDerivationOutputNames(const StorePath & path)
-{
-    return retrySQLite<StringSet>([&]() {
-        auto state(_state.lock());
-
-        auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
-            (queryValidPathId(*state, path)));
-
-        StringSet outputNames;
-        while (useQueryDerivationOutputs.next())
-            outputNames.insert(useQueryDerivationOutputs.getStr(0));
-
-        return outputNames;
-    });
-}
-
-
 std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
 {
-    if (hashPart.size() != storePathHashLen) throw Error("invalid hash part");
+    if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part");
 
     Path prefix = storeDir + "/" + hashPart;
 
@@ -829,7 +816,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
 
     StorePathSet remaining;
     for (auto & i : paths)
-        remaining.insert(i.clone());
+        remaining.insert(i);
 
     StorePathSet res;
 
@@ -843,9 +830,9 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
         StorePathSet remaining2;
         for (auto & path : remaining)
             if (valid.count(path))
-                res.insert(path.clone());
+                res.insert(path);
             else
-                remaining2.insert(path.clone());
+                remaining2.insert(path);
 
         std::swap(remaining, remaining2);
     }
@@ -867,16 +854,16 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
                 auto info = sub->queryPathInfo(path);
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));
-                infos.insert_or_assign(path.clone(), SubstitutablePathInfo{
-                    info->deriver ? info->deriver->clone() : std::optional<StorePath>(),
-                    cloneStorePathSet(info->references),
+                infos.insert_or_assign(path, SubstitutablePathInfo{
+                    info->deriver,
+                    info->references,
                     narInfo ? narInfo->fileSize : 0,
                     info->narSize});
             } catch (InvalidPath &) {
             } catch (SubstituterDisabled &) {
             } catch (Error & e) {
                 if (settings.tryFallback)
-                    printError(e.what());
+                    logError(e.info());
                 else
                     throw;
             }
@@ -913,7 +900,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
                 updatePathInfo(*state, i);
             else
                 addValidPath(*state, i, false);
-            paths.insert(i.path.clone());
+            paths.insert(i.path);
         }
 
         for (auto & i : infos) {
@@ -928,8 +915,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         for (auto & i : infos)
             if (i.path.isDerivation()) {
                 // FIXME: inefficient; we already loaded the derivation in addValidPath().
-                checkDerivationOutputs(i.path,
-                    readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string())));
+                checkDerivationOutputs(i.path, readDerivation(i.path));
             }
 
         /* Do a topological sort of the paths.  This will throw an
@@ -956,7 +942,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.erase(storePathToHash(printStorePath(path)));
+        state_->pathInfoCache.erase(std::string(path.hashPart()));
     }
 }
 
@@ -1008,7 +994,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             if (info.ca == "" || !info.references.count(info.path))
                 hashSink = std::make_unique<HashSink>(htSHA256);
             else
-                hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
+                hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
@@ -1022,7 +1008,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
-                    printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string());
+                    printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
 
             if (hashResult.second != info.narSize)
                 throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
@@ -1043,11 +1029,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
-    bool recursive, HashType hashAlgo, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
 {
     Hash h = hashString(hashAlgo, dump);
 
-    auto dstPath = makeFixedOutputPath(recursive, h, name);
+    auto dstPath = makeFixedOutputPath(method, h, name);
 
     addTempRoot(dstPath);
 
@@ -1067,7 +1053,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
             autoGC();
 
-            if (recursive) {
+            if (method == FileIngestionMethod::Recursive) {
                 StringSource source(dump);
                 restorePath(realPath, source);
             } else
@@ -1080,7 +1066,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
                above (if called with recursive == true and hashAlgo ==
                sha256); otherwise, compute it here. */
             HashResult hash;
-            if (recursive) {
+            if (method == FileIngestionMethod::Recursive) {
                 hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
                 hash.second = dump.size();
             } else
@@ -1088,10 +1074,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
             optimisePath(realPath); // FIXME: combine with hashPath()
 
-            ValidPathInfo info(dstPath.clone());
+            ValidPathInfo info(dstPath);
             info.narHash = hash.first;
             info.narSize = hash.second;
-            info.ca = makeFixedOutputCA(recursive, h);
+            info.ca = makeFixedOutputCA(method, h);
             registerValidPath(info);
         }
 
@@ -1103,7 +1089,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
 
 StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     Path srcPath(absPath(_srcPath));
 
@@ -1111,12 +1097,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
        method for very large paths, but `copyPath' is mainly used for
        small files. */
     StringSink sink;
-    if (recursive)
+    if (method == FileIngestionMethod::Recursive)
         dumpPath(srcPath, sink, filter);
     else
         sink.s = make_ref<std::string>(readFile(srcPath));
 
-    return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair);
+    return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
 }
 
 
@@ -1151,11 +1137,11 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
             optimisePath(realPath);
 
-            ValidPathInfo info(dstPath.clone());
+            ValidPathInfo info(dstPath);
             info.narHash = narHash;
             info.narSize = sink.s->size();
-            info.references = cloneStorePathSet(references);
-            info.ca = "text:" + hash.to_string();
+            info.references = references;
+            info.ca = "text:" + hash.to_string(Base32, true);
             registerValidPath(info);
         }
 
@@ -1205,7 +1191,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path)
 
 bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 {
-    printError(format("reading the Nix store..."));
+    printInfo(format("reading the Nix store..."));
 
     bool errors = false;
 
@@ -1237,12 +1223,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             Path linkPath = linksDir + "/" + link.name;
             string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
             if (hash != link.name) {
-                printError(
-                    "link '%s' was modified! expected hash '%s', got '%s'",
-                    linkPath, link.name, hash);
+                logError({
+                    .name = "Invalid hash",
+                    .hint = hintfmt(
+                        "link '%s' was modified! expected hash '%s', got '%s'",
+                        linkPath, link.name, hash)
+                });
                 if (repair) {
                     if (unlink(linkPath.c_str()) == 0)
-                        printError("removed link '%s'", linkPath);
+                        printInfo("removed link '%s'", linkPath);
                     else
                         throw SysError("removing corrupt link '%s'", linkPath);
                 } else {
@@ -1266,14 +1255,17 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 if (info->ca == "" || !info->references.count(info->path))
                     hashSink = std::make_unique<HashSink>(info->narHash.type);
                 else
-                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
+                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
 
                 dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
-                    printError("path '%s' was modified! expected hash '%s', got '%s'",
-                        printStorePath(i), info->narHash.to_string(), current.first.to_string());
+                    logError({
+                        .name = "Invalid hash - path modified",
+                        .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
+                        printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
+                    });
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
@@ -1281,14 +1273,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                     /* Fill in missing hashes. */
                     if (info->narHash == nullHash) {
-                        printError("fixing missing hash on '%s'", printStorePath(i));
+                        printInfo("fixing missing hash on '%s'", printStorePath(i));
                         info->narHash = current.first;
                         update = true;
                     }
 
                     /* Fill in missing narSize fields (from old stores). */
                     if (info->narSize == 0) {
-                        printError("updating size field on '%s' to %s", printStorePath(i), current.second);
+                        printInfo("updating size field on '%s' to %s", printStorePath(i), current.second);
                         info->narSize = current.second;
                         update = true;
                     }
@@ -1304,7 +1296,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 /* It's possible that the path got GC'ed, so ignore
                    errors on invalid paths. */
                 if (isValidPath(i))
-                    printError("error: %s", e.msg());
+                    logError(e.info());
                 else
                     warn(e.msg());
                 errors = true;
@@ -1324,7 +1316,10 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
     if (!done.insert(pathS).second) return;
 
     if (!isStorePath(pathS)) {
-        printError("path '%s' is not in the Nix store", pathS);
+        logError({
+            .name = "Nix path not found",
+            .hint = hintfmt("path '%s' is not in the Nix store", pathS)
+        });
         return;
     }
 
@@ -1343,16 +1338,19 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
             }
 
         if (canInvalidate) {
-            printError("path '%s' disappeared, removing from database...", pathS);
+            printInfo("path '%s' disappeared, removing from database...", pathS);
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            printError("path '%s' disappeared, but it still has valid referrers!", pathS);
+            logError({
+                .name = "Missing path with referrers",
+                .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
+            });
             if (repair)
                 try {
                     repairPath(path);
                 } catch (Error & e) {
-                    warn(e.msg());
+                    logWarning(e.info());
                     errors = true;
                 }
             else errors = true;
@@ -1392,7 +1390,7 @@ static void makeMutable(const Path & path)
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
     if (fd == -1) {
         if (errno == ELOOP) return; // it's a symlink
-        throw SysError(format("opening file '%1%'") % path);
+        throw SysError("opening file '%1%'", path);
     }
 
     unsigned int flags = 0, old;
@@ -1410,7 +1408,7 @@ static void makeMutable(const Path & path)
 void LocalStore::upgradeStore7()
 {
     if (getuid() != 0) return;
-    printError("removing immutable bits from the Nix store (this may take a while)...");
+    printInfo("removing immutable bits from the Nix store (this may take a while)...");
     makeMutable(realStoreDir);
 }
 

src/libstore/local-store.hh

@@ -135,8 +135,6 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    StringSet queryDerivationOutputNames(const StorePath & path) override;
-
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@@ -149,7 +147,7 @@ public:
         std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive, HashType hashAlgo,
+        FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
     /* Like addToStore(), but the contents of the path are contained
@@ -157,7 +155,7 @@ public:
        true) or simply the contents of a regular file (if recursive ==
        false). */
     StorePath addToStoreFromDump(const string & dump, const string & name,
-        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
 
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;

src/libstore/local.mk

@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
-libstore_LIBS = libutil libnixrust
+libstore_LIBS = libutil
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifneq ($(OS), FreeBSD)
@@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1)
 	libstore_LDFLAGS += -lseccomp
 endif
 
-libstore_CXXFLAGS = \
+libstore_CXXFLAGS += \
+ -I src/libutil -I src/libstore \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \

src/libstore/misc.cc

@@ -103,7 +103,7 @@ void Store::computeFSClosure(const StorePath & startPath,
     StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
 {
     StorePathSet paths;
-    paths.insert(startPath.clone());
+    paths.insert(startPath);
     computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers);
 }
 
@@ -141,11 +141,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
     auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
         {
             auto state(state_.lock());
-            state->willBuild.insert(drvPath.clone());
+            state->willBuild.insert(drvPath);
         }
 
         for (auto & i : drv.inputDrvs)
-            pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second)));
+            pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second }));
     };
 
     auto checkOutput = [&](
@@ -157,9 +157,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
         auto outPath = parseStorePath(outPathS);
 
         SubstitutablePathInfos infos;
-        StorePathSet paths; // FIXME
-        paths.insert(outPath.clone());
-        querySubstitutablePathInfos(paths, infos);
+        querySubstitutablePathInfos({outPath}, infos);
 
         if (infos.empty()) {
             drvState_->lock()->done = true;
@@ -170,10 +168,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                 if (drvState->done) return;
                 assert(drvState->left);
                 drvState->left--;
-                drvState->outPaths.insert(outPath.clone());
+                drvState->outPaths.insert(outPath);
                 if (!drvState->left) {
                     for (auto & path : drvState->outPaths)
-                        pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone())));
+                        pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } ));
                 }
             }
         }
@@ -190,12 +188,12 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             if (!isValidPath(path.path)) {
                 // FIXME: we could try to substitute the derivation.
                 auto state(state_.lock());
-                state->unknown.insert(path.path.clone());
+                state->unknown.insert(path.path);
                 return;
             }
 
             auto drv = make_ref<Derivation>(derivationFromPath(path.path));
-            ParsedDerivation parsedDrv(path.path.clone(), *drv);
+            ParsedDerivation parsedDrv(StorePath(path.path), *drv);
 
             PathSet invalid;
             for (auto & j : drv->outputs)
@@ -216,13 +214,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             if (isValidPath(path.path)) return;
 
             SubstitutablePathInfos infos;
-            StorePathSet paths; // FIXME
-            paths.insert(path.path.clone());
-            querySubstitutablePathInfos(paths, infos);
+            querySubstitutablePathInfos({path.path}, infos);
 
             if (infos.empty()) {
                 auto state(state_.lock());
-                state->unknown.insert(path.path.clone());
+                state->unknown.insert(path.path);
                 return;
             }
 
@@ -231,13 +227,13 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
 
             {
                 auto state(state_.lock());
-                state->willSubstitute.insert(path.path.clone());
+                state->willSubstitute.insert(path.path);
                 state->downloadSize += info->second.downloadSize;
                 state->narSize += info->second.narSize;
             }
 
             for (auto & ref : info->second.references)
-                pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref)));
+                pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref }));
         }
     };
 
@@ -260,12 +256,12 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
             throw BuildError("cycle detected in the references of '%s' from '%s'",
                 printStorePath(path), printStorePath(*parent));
 
-        if (!visited.insert(path.clone()).second) return;
-        parents.insert(path.clone());
+        if (!visited.insert(path).second) return;
+        parents.insert(path);
 
         StorePathSet references;
         try {
-            references = cloneStorePathSet(queryPathInfo(path)->references);
+            references = queryPathInfo(path)->references;
         } catch (InvalidPath &) {
         }
 
@@ -275,7 +271,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
             if (i != path && paths.count(i))
                 dfsVisit(i, &path);
 
-        sorted.push_back(path.clone());
+        sorted.push_back(path);
         parents.erase(path);
     };
 

src/libstore/nar-accessor.cc

@@ -184,7 +184,7 @@ struct NarAccessor : public FSAccessor
         auto i = get(path);
 
         if (i.type != FSAccessor::Type::tDirectory)
-            throw Error(format("path '%1%' inside NAR file is not a directory") % path);
+            throw Error("path '%1%' inside NAR file is not a directory", path);
 
         StringSet res;
         for (auto & child : i.children)
@@ -197,7 +197,7 @@ struct NarAccessor : public FSAccessor
     {
         auto i = get(path);
         if (i.type != FSAccessor::Type::tRegular)
-            throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
+            throw Error("path '%1%' inside NAR file is not a regular file", path);
 
         if (getNarBytes) return getNarBytes(i.start, i.size);
 
@@ -209,7 +209,7 @@ struct NarAccessor : public FSAccessor
     {
         auto i = get(path);
         if (i.type != FSAccessor::Type::tSymlink)
-            throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
+            throw Error("path '%1%' inside NAR file is not a symlink", path);
         return i.target;
     }
 };

src/libstore/nar-info-disk-cache.cc

@@ -189,7 +189,7 @@ public:
                 return {oInvalid, 0};
 
             auto namePart = queryNAR.getStr(1);
-            auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart));
+            auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart));
             narInfo->url = queryNAR.getStr(2);
             narInfo->compression = queryNAR.getStr(3);
             if (!queryNAR.isNull(4))
@@ -198,9 +198,9 @@ public:
             narInfo->narHash = Hash(queryNAR.getStr(6));
             narInfo->narSize = queryNAR.getInt(7);
             for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
-                narInfo->references.insert(StorePath::fromBaseName(r));
+                narInfo->references.insert(StorePath(r));
             if (!queryNAR.isNull(9))
-                narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9));
+                narInfo->deriver = StorePath(queryNAR.getStr(9));
             for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
                 narInfo->sigs.insert(sig);
             narInfo->ca = queryNAR.getStr(11);
@@ -230,9 +230,9 @@ public:
                     (std::string(info->path.name()))
                     (narInfo ? narInfo->url : "", narInfo != 0)
                     (narInfo ? narInfo->compression : "", narInfo != 0)
-                    (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
+                    (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
                     (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
-                    (info->narHash.to_string())
+                    (info->narHash.to_string(Base32, true))
                     (info->narSize)
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)

src/libstore/nar-info.cc

@@ -4,10 +4,10 @@
 namespace nix {
 
 NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
-    : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack
+    : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack
 {
     auto corrupt = [&]() {
-        throw Error(format("NAR info file '%1%' is corrupt") % whence);
+        throw Error("NAR info file '%1%' is corrupt", whence);
     };
 
     auto parseHashField = [&](const string & s) {
@@ -56,11 +56,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
             auto refs = tokenizeString<Strings>(value, " ");
             if (!references.empty()) corrupt();
             for (auto & r : refs)
-                references.insert(StorePath::fromBaseName(r));
+                references.insert(StorePath(r));
         }
         else if (name == "Deriver") {
             if (value != "unknown-deriver")
-                deriver = StorePath::fromBaseName(value);
+                deriver = StorePath(value);
         }
         else if (name == "System")
             system = value;
@@ -87,10 +87,10 @@ std::string NarInfo::to_string(const Store & store) const
     assert(compression != "");
     res += "Compression: " + compression + "\n";
     assert(fileHash.type == htSHA256);
-    res += "FileHash: " + fileHash.to_string(Base32) + "\n";
+    res += "FileHash: " + fileHash.to_string(Base32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
     assert(narHash.type == htSHA256);
-    res += "NarHash: " + narHash.to_string(Base32) + "\n";
+    res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
     res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";

src/libstore/optimise-store.cc

@@ -19,9 +19,9 @@ static void makeWritable(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
-        throw SysError(format("changing writability of '%1%'") % path);
+        throw SysError("changing writability of '%1%'", path);
 }
 
 
@@ -47,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
     InodeHash inodeHash;
 
     AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
+    if (!dir) throw SysError("opening directory '%1%'", linksDir);
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
         // We don't care if we hit non-hash files, anything goes
         inodeHash.insert(dirent->d_ino);
     }
-    if (errno) throw SysError(format("reading directory '%1%'") % linksDir);
+    if (errno) throw SysError("reading directory '%1%'", linksDir);
 
     printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
 
@@ -68,7 +68,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
     Strings names;
 
     AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % path);
+    if (!dir) throw SysError("opening directory '%1%'", path);
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@@ -83,7 +83,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
         if (name == "." || name == "..") continue;
         names.push_back(name);
     }
-    if (errno) throw SysError(format("reading directory '%1%'") % path);
+    if (errno) throw SysError("reading directory '%1%'", path);
 
     return names;
 }
@@ -96,7 +96,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
 #if __APPLE__
     /* HFS/macOS has some undocumented security feature disabling hardlinking for
@@ -130,7 +130,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        NixOS (example: $fontconfig/var/cache being modified).  Skip
        those files.  FIXME: check the modification time. */
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
-        printError(format("skipping suspicious writable file '%1%'") % path);
+        logWarning({
+            .name = "Suspicious file",
+            .hint = hintfmt("skipping suspicious writable file '%1%'", path)
+        });
         return;
     }
 
@@ -150,7 +153,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
     Hash hash = hashPath(htSHA256, path).first;
-    debug(format("'%1%' has hash '%2%'") % path % hash.to_string());
+    debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true));
 
     /* Check if this is a known hash. */
     Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
@@ -186,7 +189,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        current file with a hard link to that file. */
     struct stat stLink;
     if (lstat(linkPath.c_str(), &stLink))
-        throw SysError(format("getting attributes of path '%1%'") % linkPath);
+        throw SysError("getting attributes of path '%1%'", linkPath);
 
     if (st.st_ino == stLink.st_ino) {
         debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
@@ -194,7 +197,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     }
 
     if (st.st_size != stLink.st_size) {
-        printError(format("removing corrupted link '%1%'") % linkPath);
+        logWarning({
+            .name = "Corrupted link",
+            .hint = hintfmt("removing corrupted link '%1%'", linkPath)
+        });
         unlink(linkPath.c_str());
         goto retry;
     }
@@ -229,7 +235,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Atomically replace the old file with the new hard link. */
     if (rename(tempLink.c_str(), path.c_str()) == -1) {
         if (unlink(tempLink.c_str()) == -1)
-            printError(format("unable to unlink '%1%'") % tempLink);
+            logError({
+                .name = "Unlink error",
+                .hint = hintfmt("unable to unlink '%1%'", tempLink)
+            });
         if (errno == EMLINK) {
             /* Some filesystems generate too many links on the rename,
                rather than on the original link.  (Probably it
@@ -238,7 +247,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
             debug("'%s' has reached maximum number of links", linkPath);
             return;
         }
-        throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path);
+        throw SysError("cannot rename '%1%' to '%2%'", tempLink, path);
     }
 
     stats.filesLinked++;

src/libstore/parsed-derivations.cc

@@ -4,8 +4,8 @@
 
 namespace nix {
 
-ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
-    : drvPath(std::move(drvPath)), drv(drv)
+ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv)
+    : drvPath(drvPath), drv(drv)
 {
     /* Parse the __json attribute, if any. */
     auto jsonAttr = drv.env.find("__json");

src/libstore/parsed-derivations.hh

@@ -12,7 +12,7 @@ class ParsedDerivation
 
 public:
 
-    ParsedDerivation(StorePath && drvPath, BasicDerivation & drv);
+    ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv);
 
     ~ParsedDerivation();
 

src/libstore/path.cc

@@ -2,38 +2,38 @@
 
 namespace nix {
 
-extern "C" {
-    rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir);
-    rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir);
-    rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName);
-    rust::String ffi_StorePath_to_string(const StorePath & _this);
-    StorePath ffi_StorePath_clone(const StorePath & _this);
-    rust::StringSlice ffi_StorePath_name(const StorePath & _this);
+MakeError(BadStorePath, Error);
+
+static void checkName(std::string_view path, std::string_view name)
+{
+    if (name.empty())
+        throw BadStorePath("store path '%s' has an empty name", path);
+    if (name.size() > 211)
+        throw BadStorePath("store path '%s' has a name longer than 211 characters", path);
+    for (auto c : name)
+        if (!((c >= '0' && c <= '9')
+                || (c >= 'a' && c <= 'z')
+                || (c >= 'A' && c <= 'Z')
+                || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '='))
+            throw BadStorePath("store path '%s' contains illegal character '%s'", path, c);
 }
 
-StorePath StorePath::make(std::string_view path, std::string_view storeDir)
+StorePath::StorePath(std::string_view _baseName)
+    : baseName(_baseName)
 {
-    return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap();
+    if (baseName.size() < HashLen + 1)
+        throw BadStorePath("'%s' is too short to be a valid store path", baseName);
+    for (auto c : hashPart())
+        if (c == 'e' || c == 'o' || c == 'u' || c == 't'
+            || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')))
+            throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c);
+    checkName(baseName, name());
 }
 
-StorePath StorePath::make(unsigned char hash[20], std::string_view name)
+StorePath::StorePath(const Hash & hash, std::string_view _name)
+    : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name)))
 {
-    return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap();
-}
-
-StorePath StorePath::fromBaseName(std::string_view baseName)
-{
-    return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap();
-}
-
-rust::String StorePath::to_string() const
-{
-    return ffi_StorePath_to_string(*this);
-}
-
-StorePath StorePath::clone() const
-{
-    return ffi_StorePath_clone(*this);
+    checkName(baseName, name());
 }
 
 bool StorePath::isDerivation() const
@@ -41,18 +41,14 @@ bool StorePath::isDerivation() const
     return hasSuffix(name(), drvExtension);
 }
 
-std::string_view StorePath::name() const
-{
-    return ffi_StorePath_name(*this);
-}
-
-StorePath StorePath::dummy(
-    StorePath::make(
-        (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x"));
+StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
 StorePath Store::parseStorePath(std::string_view path) const
 {
-    return StorePath::make(path, storeDir);
+    auto p = canonPath(std::string(path));
+    if (dirOf(p) != storeDir)
+        throw BadStorePath("path '%s' is not in the Nix store", p);
+    return StorePath(baseNameOf(p));
 }
 
 std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
@@ -78,9 +74,7 @@ StorePathSet Store::parseStorePathSet(const PathSet & paths) const
 
 std::string Store::printStorePath(const StorePath & path) const
 {
-    auto s = storeDir + "/";
-    s += (std::string_view) path.to_string();
-    return s;
+    return (storeDir + "/").append(path.to_string());
 }
 
 PathSet Store::printStorePathSet(const StorePathSet & paths) const
@@ -90,29 +84,6 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const
     return res;
 }
 
-StorePathSet cloneStorePathSet(const StorePathSet & paths)
-{
-    StorePathSet res;
-    for (auto & p : paths)
-        res.insert(p.clone());
-    return res;
-}
-
-StorePathSet storePathsToSet(const StorePaths & paths)
-{
-    StorePathSet res;
-    for (auto & p : paths)
-        res.insert(p.clone());
-    return res;
-}
-
-StorePathSet singleton(const StorePath & path)
-{
-    StorePathSet res;
-    res.insert(path.clone());
-    return res;
-}
-
 std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
 {
     size_t n = s.find("!");

src/libstore/path.hh

@@ -1,59 +1,59 @@
 #pragma once
 
-#include "rust-ffi.hh"
+#include "types.hh"
 
 namespace nix {
 
-/* See path.rs. */
-struct StorePath;
-
 class Store;
+struct Hash;
 
-extern "C" {
-    void ffi_StorePath_drop(void *);
-    bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b);
-    bool ffi_StorePath_eq(const StorePath & a, const StorePath & b);
-    unsigned char * ffi_StorePath_hash_data(const StorePath & p);
-}
-
-struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
+class StorePath
 {
+    std::string baseName;
+
+public:
+
+    /* Size of the hash part of store paths, in base-32 characters. */
+    constexpr static size_t HashLen = 32; // i.e. 160 bits
+
     StorePath() = delete;
 
-    static StorePath make(std::string_view path, std::string_view storeDir);
+    StorePath(std::string_view baseName);
 
-    static StorePath make(unsigned char hash[20], std::string_view name);
+    StorePath(const Hash & hash, std::string_view name);
 
-    static StorePath fromBaseName(std::string_view baseName);
-
-    rust::String to_string() const;
+    std::string_view to_string() const
+    {
+        return baseName;
+    }
 
     bool operator < (const StorePath & other) const
     {
-        return ffi_StorePath_less_than(*this, other);
+        return baseName < other.baseName;
     }
 
     bool operator == (const StorePath & other) const
     {
-        return ffi_StorePath_eq(*this, other);
+        return baseName == other.baseName;
     }
 
     bool operator != (const StorePath & other) const
     {
-        return !(*this == other);
+        return baseName != other.baseName;
     }
 
-    StorePath clone() const;
-
     /* Check whether a file name ends with the extension for
        derivations. */
     bool isDerivation() const;
 
-    std::string_view name() const;
-
-    unsigned char * hashData() const
+    std::string_view name() const
     {
-        return ffi_StorePath_hash_data(*this);
+        return std::string_view(baseName).substr(HashLen + 1);
+    }
+
+    std::string_view hashPart() const
+    {
+        return std::string_view(baseName).substr(0, HashLen);
     }
 
     static StorePath dummy;
@@ -62,34 +62,19 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
 typedef std::set<StorePath> StorePathSet;
 typedef std::vector<StorePath> StorePaths;
 
-StorePathSet cloneStorePathSet(const StorePathSet & paths);
-StorePathSet storePathsToSet(const StorePaths & paths);
-
-StorePathSet singleton(const StorePath & path);
-
-/* Size of the hash part of store paths, in base-32 characters. */
-const size_t storePathHashLen = 32; // i.e. 160 bits
-
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
+enum struct FileIngestionMethod : uint8_t {
+    Flat = false,
+    Recursive = true
+};
+
 struct StorePathWithOutputs
 {
     StorePath path;
     std::set<std::string> outputs;
 
-    StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
-        : path(path.clone()), outputs(outputs)
-    { }
-
-    StorePathWithOutputs(StorePath && path, std::set<std::string> && outputs)
-        : path(std::move(path)), outputs(std::move(outputs))
-    { }
-
-    StorePathWithOutputs(const StorePathWithOutputs & other)
-        : path(other.path.clone()), outputs(other.outputs)
-    { }
-
     std::string to_string(const Store & store) const;
 };
 
@@ -102,7 +87,7 @@ namespace std {
 template<> struct hash<nix::StorePath> {
     std::size_t operator()(const nix::StorePath & path) const noexcept
     {
-        return * (std::size_t *) path.hashData();
+        return * (std::size_t *) path.to_string().data();
     }
 };
 

src/libstore/pathlocks.cc

@@ -20,7 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create)
 
     fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
     if (!fd && (create || errno != ENOENT))
-        throw SysError(format("opening lock file '%1%'") % path);
+        throw SysError("opening lock file '%1%'", path);
 
     return fd;
 }
@@ -51,7 +51,7 @@ bool lockFile(int fd, LockType lockType, bool wait)
         while (flock(fd, type) != 0) {
             checkInterrupt();
             if (errno != EINTR)
-                throw SysError(format("acquiring/releasing lock"));
+                throw SysError("acquiring/releasing lock");
             else
                 return false;
         }
@@ -60,7 +60,7 @@ bool lockFile(int fd, LockType lockType, bool wait)
             checkInterrupt();
             if (errno == EWOULDBLOCK) return false;
             if (errno != EINTR)
-                throw SysError(format("acquiring/releasing lock"));
+                throw SysError("acquiring/releasing lock");
         }
     }
 
@@ -124,7 +124,7 @@ bool PathLocks::lockPaths(const PathSet & paths,
                hasn't been unlinked). */
             struct stat st;
             if (fstat(fd.get(), &st) == -1)
-                throw SysError(format("statting lock file '%1%'") % lockPath);
+                throw SysError("statting lock file '%1%'", lockPath);
             if (st.st_size != 0)
                 /* This lock file has been unlinked, so we're holding
                    a lock on a deleted file.  This means that other
@@ -160,7 +160,8 @@ void PathLocks::unlock()
 
         if (close(i.first) == -1)
             printError(
-                format("error (ignored): cannot close lock file on '%1%'") % i.second);
+                "error (ignored): cannot close lock file on '%1%'",
+                i.second);
 
         debug(format("lock released on '%1%'") % i.second);
     }

src/libstore/profiles.cc

@@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen)
             gen.number = n;
             struct stat st;
             if (lstat(gen.path.c_str(), &st) != 0)
-                throw SysError(format("statting '%1%'") % gen.path);
+                throw SysError("statting '%1%'", gen.path);
             gen.creationTime = st.st_mtime;
             gens.push_back(gen);
         }
@@ -117,7 +117,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
 static void removeFile(const Path & path)
 {
     if (remove(path.c_str()) == -1)
-        throw SysError(format("cannot unlink '%1%'") % path);
+        throw SysError("cannot unlink '%1%'", path);
 }
 
 
@@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
     Generations gens = findGenerations(profile, curGen);
 
     if (gensToDelete.find(curGen) != gensToDelete.end())
-        throw Error(format("cannot delete current generation of profile %1%'") % profile);
+        throw Error("cannot delete current generation of profile %1%'", profile);
 
     for (auto & i : gens) {
         if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
@@ -226,7 +226,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
     int days;
 
     if (!string2Int(strDays, days) || days < 1)
-        throw Error(format("invalid number of days specifier '%1%'") % timeSpec);
+        throw Error("invalid number of days specifier '%1%'", timeSpec);
 
     time_t oldTime = curTime - days * 24 * 3600;
 

src/libstore/references.cc

@@ -92,7 +92,7 @@ PathSet scanForReferences(const string & path,
         auto baseName = std::string(baseNameOf(i));
         string::size_type pos = baseName.find('-');
         if (pos == string::npos)
-            throw Error(format("bad reference '%1%'") % i);
+            throw Error("bad reference '%1%'", i);
         string s = string(baseName, 0, pos);
         assert(s.size() == refLength);
         assert(backMap.find(s) == backMap.end());

src/libstore/remote-fs-accessor.cc

@@ -19,7 +19,7 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
 Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
 {
     assert(cacheDir != "");
-    return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
+    return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
 }
 
 void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
@@ -51,7 +51,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
     std::string restPath = std::string(path, storePath.size());
 
     if (!store->isValidPath(store->parseStorePath(storePath)))
-        throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
+        throw InvalidPath("path '%1%' is not a valid store path", storePath);
 
     auto i = nars.find(storePath);
     if (i != nars.end()) return {i->second, restPath};

src/libstore/remote-store.cc

@@ -116,11 +116,11 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
     if (socketPath.size() + 1 >= sizeof(addr.sun_path))
-        throw Error(format("socket path '%1%' is too long") % socketPath);
+        throw Error("socket path '%1%' is too long", socketPath);
     strcpy(addr.sun_path, socketPath.c_str());
 
     if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
-        throw SysError(format("cannot connect to daemon at '%1%'") % socketPath);
+        throw SysError("cannot connect to daemon at '%1%'", socketPath);
 
     conn->from.fd = conn->fd.get();
     conn->to.fd = conn->fd.get();
@@ -268,7 +268,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
         StorePathSet res;
         for (auto & i : paths)
-            if (isValidPath(i)) res.insert(i.clone());
+            if (isValidPath(i)) res.insert(i);
         return res;
     } else {
         conn->to << wopQueryValidPaths;
@@ -296,7 +296,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         for (auto & i : paths) {
             conn->to << wopHasSubstitutes << printStorePath(i);
             conn.processStderr();
-            if (readInt(conn->from)) res.insert(i.clone());
+            if (readInt(conn->from)) res.insert(i);
         }
         return res;
     } else {
@@ -329,7 +329,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
             info.references = readStorePaths<StorePathSet>(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
-            infos.insert_or_assign(i.clone(), std::move(info));
+            infos.insert_or_assign(i, std::move(info));
         }
 
     } else {
@@ -365,14 +365,14 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
             } catch (Error & e) {
                 // Ugly backwards compatibility hack.
                 if (e.msg().find("is not valid") != std::string::npos)
-                    throw InvalidPath(e.what());
+                    throw InvalidPath(e.info());
                 throw;
             }
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
                 bool valid; conn->from >> valid;
                 if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
             }
-            info = std::make_shared<ValidPathInfo>(path.clone());
+            info = std::make_shared<ValidPathInfo>(StorePath(path));
             auto deriver = readString(conn->from);
             if (deriver != "") info->deriver = parseStorePath(deriver);
             info->narHash = Hash(readString(conn->from), htSHA256);
@@ -396,7 +396,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
     conn->to << wopQueryReferrers << printStorePath(path);
     conn.processStderr();
     for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
-        referrers.insert(i.clone());
+        referrers.insert(i);
 }
 
 
@@ -418,15 +418,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 }
 
 
-PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path)
-{
-    auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputNames << printStorePath(path);
-    conn.processStderr();
-    return readStrings<PathSet>(conn->from);
-}
-
-
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
 {
     auto conn(getConnection());
@@ -484,7 +475,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
@@ -492,10 +483,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
 
     Path srcPath(absPath(_srcPath));
 
-    conn->to << wopAddToStore << name
-       << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */
-       << (recursive ? 1 : 0)
-       << printHashType(hashAlgo);
+    conn->to
+        << wopAddToStore
+        << name
+        << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+        << (method == FileIngestionMethod::Recursive ? 1 : 0)
+        << printHashType(hashAlgo);
 
     try {
         conn->to.written = 0;

src/libstore/remote-store.hh

@@ -51,8 +51,6 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    StringSet queryDerivationOutputNames(const StorePath & path) override;
-
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@@ -65,7 +63,7 @@ public:
         std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
 
     StorePath addTextToStore(const string & name, const string & s,

src/libstore/s3-binary-cache-store.cc

@@ -6,7 +6,7 @@
 #include "nar-info-disk-cache.hh"
 #include "globals.hh"
 #include "compression.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
@@ -32,8 +32,10 @@ namespace nix {
 struct S3Error : public Error
 {
     Aws::S3::S3Errors err;
-    S3Error(Aws::S3::S3Errors err, const FormatOrString & fs)
-        : Error(fs), err(err) { };
+
+    template<typename... Args>
+    S3Error(Aws::S3::S3Errors err, const Args & ... args)
+        : Error(args...), err(err) { };
 };
 
 /* Helper: given an Outcome<R, E>, return R in case of success, or
@@ -109,7 +111,9 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
         auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
         if (retry)
             printError("AWS error '%s' (%s), will retry in %d ms",
-                error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries));
+                error.GetExceptionName(),
+                error.GetMessage(),
+                CalculateDelayBeforeNextRetry(error, attemptedRetries));
         return retry;
     }
 };
@@ -132,7 +136,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
     return res;
 }
 
-S3Helper::DownloadResult S3Helper::getObject(
+S3Helper::FileTransferResult S3Helper::getObject(
     const std::string & bucketName, const std::string & key)
 {
     debug("fetching 's3://%s/%s'...", bucketName, key);
@@ -146,7 +150,7 @@ S3Helper::DownloadResult S3Helper::getObject(
         return Aws::New<std::stringstream>("STRINGSTREAM");
     });
 
-    DownloadResult res;
+    FileTransferResult res;
 
     auto now1 = std::chrono::steady_clock::now();
 
@@ -249,7 +253,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                 // If bucket listing is disabled, 404s turn into 403s
                 || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
                 return false;
-            throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
+            throw Error("AWS error fetching '%s': %s", path, error.GetMessage());
         }
 
         return true;

src/libstore/s3.hh

@@ -18,13 +18,13 @@ struct S3Helper
 
     ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
 
-    struct DownloadResult
+    struct FileTransferResult
     {
         std::shared_ptr<std::string> data;
         unsigned int durationMs;
     };
 
-    DownloadResult getObject(
+    FileTransferResult getObject(
         const std::string & bucketName, const std::string & key);
 };
 

src/libstore/sqlite.cc

@@ -29,7 +29,7 @@ SQLite::SQLite(const Path & path, bool create)
 {
     if (sqlite3_open_v2(path.c_str(), &db,
             SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
-        throw Error(format("cannot open SQLite database '%s'") % path);
+        throw Error("cannot open SQLite database '%s'", path);
 
     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
         throwSQLiteError(db, "setting timeout");
@@ -204,7 +204,10 @@ void handleSQLiteBusy(const SQLiteBusy & e)
 
     if (now > lastWarned + 10) {
         lastWarned = now;
-        printError("warning: %s", e.what());
+        logWarning({
+            .name = "Sqlite busy",
+            .hint = hintfmt(e.what())
+        });
     }
 
     /* Sleep for a while since retrying the transaction right away

src/libstore/sqlite.hh

@@ -3,7 +3,7 @@
 #include <functional>
 #include <string>
 
-#include "types.hh"
+#include "error.hh"
 
 struct sqlite3;
 struct sqlite3_stmt;

src/libstore/store-api.cc

@@ -6,6 +6,7 @@
 #include "thread-pool.hh"
 #include "json.hh"
 #include "derivations.hh"
+#include "url.hh"
 
 #include <future>
 
@@ -22,7 +23,7 @@ bool Store::isInStore(const Path & path) const
 Path Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
-        throw Error(format("path '%1%' is not in the Nix store") % path);
+        throw Error("path '%1%' is not in the Nix store", path);
     Path::size_type slash = path.find('/', storeDir.size() + 1);
     if (slash == Path::npos)
         return path;
@@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const
         path = absPath(target, dirOf(path));
     }
     if (!isInStore(path))
-        throw Error(format("path '%1%' is not in the Nix store") % path);
+        throw NotInStore("path '%1%' is not in the Nix store", path);
     return path;
 }
 
@@ -54,15 +55,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
 StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
 {
     auto [path2, outputs] = nix::parsePathWithOutputs(path);
-    return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs));
-}
-
-
-string storePathToHash(const Path & path)
-{
-    auto base = baseNameOf(path);
-    assert(base.size() >= storePathHashLen);
-    return string(base, 0, storePathHashLen);
+    return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
 }
 
 
@@ -141,9 +134,9 @@ StorePath Store::makeStorePath(const string & type,
     const Hash & hash, std::string_view name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name);
+    string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
     auto h = compressHash(hashString(htSHA256, s), 20);
-    return StorePath::make(h.hash, name);
+    return StorePath(h, name);
 }
 
 
@@ -171,19 +164,22 @@ static std::string makeType(
 
 
 StorePath Store::makeFixedOutputPath(
-    bool recursive,
+    FileIngestionMethod recursive,
     const Hash & hash,
     std::string_view name,
     const StorePathSet & references,
     bool hasSelfReference) const
 {
-    if (hash.type == htSHA256 && recursive) {
+    if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
     } else {
         assert(references.empty());
-        return makeStorePath("output:out", hashString(htSHA256,
-                "fixed:out:" + (recursive ? (string) "r:" : "") +
-                hash.to_string(Base16) + ":"), name);
+        return makeStorePath("output:out",
+            hashString(htSHA256,
+                "fixed:out:"
+                + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+                + hash.to_string(Base16, true) + ":"),
+            name);
     }
 }
 
@@ -200,10 +196,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
 
 
 std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
-    const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
+    const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const
 {
-    Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
-    return std::make_pair(makeFixedOutputPath(recursive, h, name), h);
+    Hash h = method == FileIngestionMethod::Recursive
+        ? hashPath(hashAlgo, srcPath, filter).first
+        : hashFile(hashAlgo, srcPath);
+    return std::make_pair(makeFixedOutputPath(method, h, name), h);
 }
 
 
@@ -237,7 +235,7 @@ bool Store::PathInfoCacheValue::isKnownNow()
 
 bool Store::isValidPath(const StorePath & storePath)
 {
-    auto hashPart = storePathToHash(printStorePath(storePath));
+    std::string hashPart(storePath.hashPart());
 
     {
         auto state_(state.lock());
@@ -305,7 +303,7 @@ void Store::queryPathInfo(const StorePath & storePath,
     std::string hashPart;
 
     try {
-        hashPart = storePathToHash(printStorePath(storePath));
+        hashPart = storePath.hashPart();
 
         {
             auto res = state.lock()->pathInfoCache.get(hashPart);
@@ -455,7 +453,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
             auto info = queryPathInfo(storePath);
 
             jsonPath
-                .attr("narHash", info->narHash.to_string(hashBase))
+                .attr("narHash", info->narHash.to_string(hashBase, true))
                 .attr("narSize", info->narSize);
 
             {
@@ -498,7 +496,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
                     if (!narInfo->url.empty())
                         jsonPath.attr("url", narInfo->url);
                     if (narInfo->fileHash)
-                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
+                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
                     if (narInfo->fileSize)
                         jsonPath.attr("downloadSize", narInfo->fileSize);
                     if (showClosureSize)
@@ -547,7 +545,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod
     for (auto & path : paths) {
         if (path.path.isDerivation())
             unsupported("buildPaths");
-        paths2.insert(path.path.clone());
+        paths2.insert(path.path);
     }
 
     if (queryValidPaths(paths2).size() != paths2.size())
@@ -687,21 +685,6 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
 }
 
 
-ValidPathInfo::ValidPathInfo(const ValidPathInfo & other)
-    : path(other.path.clone())
-    , deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{})
-    , narHash(other.narHash)
-    , references(cloneStorePathSet(other.references))
-    , registrationTime(other.registrationTime)
-    , narSize(other.narSize)
-    , id(other.id)
-    , ultimate(other.ultimate)
-    , sigs(other.sigs)
-    , ca(other.ca)
-{
-}
-
-
 std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven)
 {
     std::string path;
@@ -754,7 +737,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
             store.printStorePath(path));
     return
         "1;" + store.printStorePath(path) + ";"
-        + narHash.to_string(Base32) + ";"
+        + narHash.to_string(Base32, true) + ";"
         + std::to_string(narSize) + ";"
         + concatStringsSep(",", store.printStorePathSet(references));
 }
@@ -769,11 +752,15 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
 bool ValidPathInfo::isContentAddressed(const Store & store) const
 {
     auto warn = [&]() {
-        printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
+        logWarning(
+            ErrorInfo{
+                .name = "Path not content-addressed",
+                .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path))
+            });
     };
 
     if (hasPrefix(ca, "text:")) {
-        Hash hash(std::string(ca, 5));
+        Hash hash(ca.substr(5));
         if (store.makeTextPath(path.name(), hash, references) == path)
             return true;
         else
@@ -781,9 +768,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     }
 
     else if (hasPrefix(ca, "fixed:")) {
-        bool recursive = ca.compare(6, 2, "r:") == 0;
-        Hash hash(std::string(ca, recursive ? 8 : 6));
-        auto refs = cloneStorePathSet(references);
+        FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
+        Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6));
+        auto refs = references;
         bool hasSelfReference = false;
         if (refs.count(path)) {
             hasSelfReference = true;
@@ -826,27 +813,14 @@ Strings ValidPathInfo::shortRefs() const
 }
 
 
-std::string makeFixedOutputCA(bool recursive, const Hash & hash)
+std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
 {
-    return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
+    return "fixed:"
+        + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
+        + hash.to_string(Base32, true);
 }
 
 
-void Store::addToStore(const ValidPathInfo & info, Source & narSource,
-    RepairFlag repair, CheckSigsFlag checkSigs,
-    std::shared_ptr<FSAccessor> accessor)
-{
-    addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor);
-}
-
-void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-    RepairFlag repair, CheckSigsFlag checkSigs,
-    std::shared_ptr<FSAccessor> accessor)
-{
-    StringSource source(*nar);
-    addToStore(info, source, repair, checkSigs, accessor);
-}
-
 }
 
 
@@ -866,27 +840,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
     Store::Params params;
     auto q = uri.find('?');
     if (q != std::string::npos) {
-        for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
-            auto e = s.find('=');
-            if (e != std::string::npos) {
-                auto value = s.substr(e + 1);
-                std::string decoded;
-                for (size_t i = 0; i < value.size(); ) {
-                    if (value[i] == '%') {
-                        if (i + 2 >= value.size())
-                            throw Error("invalid URI parameter '%s'", value);
-                        try {
-                            decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
-                            i += 3;
-                        } catch (...) {
-                            throw Error("invalid URI parameter '%s'", value);
-                        }
-                    } else
-                        decoded += value[i++];
-                }
-                params[s.substr(0, e)] = decoded;
-            }
-        }
+        params = decodeQuery(uri.substr(q + 1));
         uri = uri_.substr(0, q);
     }
     return {uri, params};
@@ -961,7 +915,7 @@ std::list<ref<Store>> getDefaultSubstituters()
             try {
                 stores.push_back(openStore(uri));
             } catch (Error & e) {
-                printError("warning: %s", e.what());
+                logWarning(e.info());
             }
         };
 

src/libstore/store-api.hh

@@ -29,6 +29,7 @@ MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
+MakeError(NotInStore, Error);
 
 
 class FSAccessor;
@@ -41,7 +42,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
 enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
 enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
 
-
 /* Magic header of exportPath() output (obsolete). */
 const uint32_t exportMagic = 0x4558494e;
 
@@ -187,8 +187,9 @@ struct ValidPathInfo
 
     Strings shortRefs() const;
 
+    ValidPathInfo(const StorePath & path) : path(path) { }
+
     ValidPathInfo(StorePath && path) : path(std::move(path)) { }
-    explicit ValidPathInfo(const ValidPathInfo & other);
 
     virtual ~ValidPathInfo() { }
 };
@@ -344,7 +345,7 @@ public:
     StorePath makeOutputPath(const string & id,
         const Hash & hash, std::string_view name) const;
 
-    StorePath makeFixedOutputPath(bool recursive,
+    StorePath makeFixedOutputPath(FileIngestionMethod method,
         const Hash & hash, std::string_view name,
         const StorePathSet & references = {},
         bool hasSelfReference = false) const;
@@ -356,7 +357,7 @@ public:
        store path to which srcPath is to be copied.  Returns the store
        path and the cryptographic hash of the contents of srcPath. */
     std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
-        const Path & srcPath, bool recursive = true,
+        const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive,
         HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
 
     /* Preparatory part of addTextToStore().
@@ -428,10 +429,6 @@ public:
     virtual StorePathSet queryDerivationOutputs(const StorePath & path)
     { unsupported("queryDerivationOutputs"); }
 
-    /* Query the output names of the derivation denoted by `path'. */
-    virtual StringSet queryDerivationOutputNames(const StorePath & path)
-    { unsupported("queryDerivationOutputNames"); }
-
     /* Query the full store path given the hash part of a valid store
        path, or empty if the path doesn't exist. */
     virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0;
@@ -448,24 +445,19 @@ public:
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
-        std::shared_ptr<FSAccessor> accessor = 0);
-
-    // FIXME: remove
-    virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
-        std::shared_ptr<FSAccessor> accessor = 0);
+        std::shared_ptr<FSAccessor> accessor = 0) = 0;
 
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
        The function object `filter' can be used to exclude files (see
        libutil/archive.hh). */
     virtual StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(const string & dump, const string & name,
-        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
     {
         throw Error("addToStoreFromDump() is not supported by this store");
     }
@@ -590,6 +582,9 @@ public:
        ensurePath(). */
     Derivation derivationFromPath(const StorePath & drvPath);
 
+    /* Read a derivation (which must already be valid). */
+    Derivation readDerivation(const StorePath & drvPath);
+
     /* Place in `out' the set of all store paths in the file system
        closure of `storePath'; that is, all paths than can be directly
        or indirectly reached from it.  `out' is not cleared.  If
@@ -735,10 +730,6 @@ public:
 };
 
 
-/* Extract the hash part of the given store path. */
-string storePathToHash(const Path & path);
-
-
 /* Copy a path from one store to another. */
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
@@ -848,7 +839,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
 
 /* Compute the content-addressability assertion (ValidPathInfo::ca)
    for paths created by makeFixedOutputPath() / addToStore(). */
-std::string makeFixedOutputCA(bool recursive, const Hash & hash);
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
 
 
 /* Split URI into protocol+hierarchy part and its parameter set. */

src/libstore/worker-protocol.hh

@@ -36,7 +36,7 @@ typedef enum {
     wopClearFailedPaths = 25,
     wopQueryPathInfo = 26,
     wopImportPaths = 27, // obsolete
-    wopQueryDerivationOutputNames = 28,
+    wopQueryDerivationOutputNames = 28, // obsolete
     wopQueryPathFromHashPart = 29,
     wopQuerySubstitutablePathInfos = 30,
     wopQueryValidPaths = 31,

src/libutil/affinity.cc

@@ -12,6 +12,17 @@ namespace nix {
 #if __linux__
 static bool didSaveAffinity = false;
 static cpu_set_t savedAffinity;
+
+std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset)
+{
+    auto count = CPU_COUNT(&cset);
+    for (int i=0; i < count; ++i)
+    {
+        os << (CPU_ISSET(i,&cset) ? "1" : "0");
+    }
+
+    return os;
+}
 #endif
 
 
@@ -25,7 +36,7 @@ void setAffinityTo(int cpu)
     CPU_ZERO(&newAffinity);
     CPU_SET(cpu, &newAffinity);
     if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
-        printError(format("failed to lock thread to CPU %1%") % cpu);
+        printError("failed to lock thread to CPU %1%", cpu);
 #endif
 }
 
@@ -47,7 +58,11 @@ void restoreAffinity()
 #if __linux__
     if (!didSaveAffinity) return;
     if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
-        printError("failed to restore affinity %1%");
+    {
+        std::ostringstream oss;
+        oss << savedAffinity;
+        printError("failed to restore CPU affinity %1%", oss.str());
+    }
 #endif
 }
 

src/libutil/ansicolor.hh

@@ -0,0 +1,15 @@
+#pragma once
+
+namespace nix {
+
+/* Some ANSI escape sequences. */
+#define ANSI_NORMAL "\e[0m"
+#define ANSI_BOLD "\e[1m"
+#define ANSI_FAINT "\e[2m"
+#define ANSI_ITALIC "\e[3m"
+#define ANSI_RED "\e[31;1m"
+#define ANSI_GREEN "\e[32;1m"
+#define ANSI_YELLOW "\e[33;1m"
+#define ANSI_BLUE "\e[34;1m"
+
+}

src/libutil/archive.cc

@@ -46,7 +46,7 @@ static void dumpContents(const Path & path, size_t size,
     sink << "contents" << size;
 
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError(format("opening file '%1%'") % path);
+    if (!fd) throw SysError("opening file '%1%'", path);
 
     std::vector<unsigned char> buf(65536);
     size_t left = size;
@@ -68,7 +68,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     sink << "(";
 
@@ -94,8 +94,9 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
                     name.erase(pos);
                 }
                 if (unhacked.find(name) != unhacked.end())
-                    throw Error(format("file name collision in between '%1%' and '%2%'")
-                        % (path + "/" + unhacked[name]) % (path + "/" + i.name));
+                    throw Error("file name collision in between '%1%' and '%2%'",
+                       (path + "/" + unhacked[name]),
+                       (path + "/" + i.name));
                 unhacked[name] = i.name;
             } else
                 unhacked[i.name] = i.name;
@@ -111,7 +112,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
     else if (S_ISLNK(st.st_mode))
         sink << "type" << "symlink" << "target" << readLink(path);
 
-    else throw Error(format("file '%1%' has an unsupported type") % path);
+    else throw Error("file '%1%' has an unsupported type", path);
 
     sink << ")";
 }
@@ -247,7 +248,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
                 } else if (s == "name") {
                     name = readString(source);
                     if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
-                        throw Error(format("NAR contains invalid file name '%1%'") % name);
+                        throw Error("NAR contains invalid file name '%1%'", name);
                     if (name <= prevName)
                         throw Error("NAR directory is not sorted");
                     prevName = name;
@@ -303,14 +304,14 @@ struct RestoreSink : ParseSink
     {
         Path p = dstPath + path;
         if (mkdir(p.c_str(), 0777) == -1)
-            throw SysError(format("creating directory '%1%'") % p);
+            throw SysError("creating directory '%1%'", p);
     };
 
     void createRegularFile(const Path & path)
     {
         Path p = dstPath + path;
         fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
-        if (!fd) throw SysError(format("creating file '%1%'") % p);
+        if (!fd) throw SysError("creating file '%1%'", p);
     }
 
     void isExecutable()
@@ -332,7 +333,7 @@ struct RestoreSink : ParseSink
                OpenSolaris).  Since preallocation is just an
                optimisation, ignore it. */
             if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
-                throw SysError(format("preallocating file of %1% bytes") % len);
+                throw SysError("preallocating file of %1% bytes", len);
         }
 #endif
     }

src/libutil/args.cc

@@ -3,16 +3,14 @@
 
 namespace nix {
 
-Args::FlagMaker Args::mkFlag()
-{
-    return FlagMaker(*this);
-}
-
-Args::FlagMaker::~FlagMaker()
+void Args::addFlag(Flag && flag_)
 {
+    auto flag = std::make_shared<Flag>(std::move(flag_));
+    if (flag->handler.arity != ArityAny)
+        assert(flag->handler.arity == flag->labels.size());
     assert(flag->longName != "");
-    args.longFlags[flag->longName] = flag;
-    if (flag->shortName) args.shortFlags[flag->shortName] = flag;
+    longFlags[flag->longName] = flag;
+    if (flag->shortName) shortFlags[flag->shortName] = flag;
 }
 
 void Args::parseCmdline(const Strings & _cmdline)
@@ -47,7 +45,7 @@ void Args::parseCmdline(const Strings & _cmdline)
         }
         else if (!dashDash && std::string(arg, 0, 1) == "-") {
             if (!processFlag(pos, cmdline.end()))
-                throw UsageError(format("unrecognised flag '%1%'") % arg);
+                throw UsageError("unrecognised flag '%1%'", arg);
         }
         else {
             pendingArgs.push_back(*pos++);
@@ -61,7 +59,7 @@ void Args::parseCmdline(const Strings & _cmdline)
 
 void Args::printHelp(const string & programName, std::ostream & out)
 {
-    std::cout << "Usage: " << programName << " <FLAGS>...";
+    std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName);
     for (auto & exp : expectedArgs) {
         std::cout << renderLabels({exp.label});
         // FIXME: handle arity > 1
@@ -72,11 +70,11 @@ void Args::printHelp(const string & programName, std::ostream & out)
 
     auto s = description();
     if (s != "")
-        std::cout << "\nSummary: " << s << ".\n";
+        std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n";
 
     if (longFlags.size()) {
         std::cout << "\n";
-        std::cout << "Flags:\n";
+        std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n";
         printFlags(out);
     }
 }
@@ -101,15 +99,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
     auto process = [&](const std::string & name, const Flag & flag) -> bool {
         ++pos;
         std::vector<std::string> args;
-        for (size_t n = 0 ; n < flag.arity; ++n) {
+        for (size_t n = 0 ; n < flag.handler.arity; ++n) {
             if (pos == end) {
-                if (flag.arity == ArityAny) break;
-                throw UsageError(format("flag '%1%' requires %2% argument(s)")
-                    % name % flag.arity);
+                if (flag.handler.arity == ArityAny) break;
+                throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
             }
             args.push_back(*pos++);
         }
-        flag.handler(std::move(args));
+        flag.handler.fun(std::move(args));
         return true;
     };
 
@@ -133,7 +130,7 @@ bool Args::processArgs(const Strings & args, bool finish)
 {
     if (expectedArgs.empty()) {
         if (!args.empty())
-            throw UsageError(format("unexpected argument '%1%'") % args.front());
+            throw UsageError("unexpected argument '%1%'", args.front());
         return true;
     }
 
@@ -157,17 +154,18 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
-Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht)
+Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
 {
-    arity(1);
-    label("type");
-    description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
-    handler([ht](std::string s) {
-        *ht = parseHashType(s);
-        if (*ht == htUnknown)
-            throw UsageError("unknown hash type '%1%'", s);
-    });
-    return *this;
+    return Flag {
+        .longName = std::move(longName),
+        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
+        .labels = {"hash-algo"},
+        .handler = {[ht](std::string s) {
+            *ht = parseHashType(s);
+            if (*ht == htUnknown)
+                throw UsageError("unknown hash type '%1%'", s);
+        }}
+    };
 }
 
 Strings argvToStrings(int argc, char * * argv)
@@ -183,7 +181,7 @@ std::string renderLabels(const Strings & labels)
     std::string res;
     for (auto label : labels) {
         for (auto & c : label) c = std::toupper(c);
-        res += " <" + label + ">";
+        res += " " ANSI_ITALIC + label + ANSI_NORMAL;
     }
     return res;
 }
@@ -192,10 +190,10 @@ void printTable(std::ostream & out, const Table2 & table)
 {
     size_t max = 0;
     for (auto & row : table)
-        max = std::max(max, row.first.size());
+        max = std::max(max, filterANSIEscapes(row.first, true).size());
     for (auto & row : table) {
         out << "  " << row.first
-            << std::string(max - row.first.size() + 2, ' ')
+            << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ')
             << row.second << "\n";
     }
 }
@@ -206,8 +204,7 @@ void Command::printHelp(const string & programName, std::ostream & out)
 
     auto exs = examples();
     if (!exs.empty()) {
-        out << "\n";
-        out << "Examples:\n";
+        out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n";
         for (auto & ex : exs)
             out << "\n"
                 << "  " << ex.description << "\n" // FIXME: wrap
@@ -220,52 +217,63 @@ MultiCommand::MultiCommand(const Commands & commands)
 {
     expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
         assert(!command);
-        auto i = commands.find(ss[0]);
+        auto cmd = ss[0];
+        if (auto alias = get(deprecatedAliases, cmd)) {
+            warn("'%s' is a deprecated alias for '%s'", cmd, *alias);
+            cmd = *alias;
+        }
+        auto i = commands.find(cmd);
         if (i == commands.end())
-            throw UsageError("'%s' is not a recognised command", ss[0]);
-        command = i->second();
-        command->_name = ss[0];
+            throw UsageError("'%s' is not a recognised command", cmd);
+        command = {cmd, i->second()};
     }});
+
+    categories[Command::catDefault] = "Available commands";
 }
 
 void MultiCommand::printHelp(const string & programName, std::ostream & out)
 {
     if (command) {
-        command->printHelp(programName + " " + command->name(), out);
+        command->second->printHelp(programName + " " + command->first, out);
         return;
     }
 
-    out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n";
+    out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName);
 
-    out << "\n";
-    out << "Common flags:\n";
+    out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n";
     printFlags(out);
 
-    out << "\n";
-    out << "Available commands:\n";
+    std::map<Command::Category, std::map<std::string, ref<Command>>> commandsByCategory;
 
-    Table2 table;
-    for (auto & i : commands) {
-        auto command = i.second();
-        command->_name = i.first;
-        auto descr = command->description();
-        if (!descr.empty())
-            table.push_back(std::make_pair(command->name(), descr));
+    for (auto & [name, commandFun] : commands) {
+        auto command = commandFun();
+        commandsByCategory[command->category()].insert_or_assign(name, command);
+    }
+
+    for (auto & [category, commands] : commandsByCategory) {
+        out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]);
+
+        Table2 table;
+        for (auto & [name, command] : commands) {
+            auto descr = command->description();
+            if (!descr.empty())
+                table.push_back(std::make_pair(name, descr));
+        }
+        printTable(out, table);
     }
-    printTable(out, table);
 }
 
 bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
 {
     if (Args::processFlag(pos, end)) return true;
-    if (command && command->processFlag(pos, end)) return true;
+    if (command && command->second->processFlag(pos, end)) return true;
     return false;
 }
 
 bool MultiCommand::processArgs(const Strings & args, bool finish)
 {
     if (command)
-        return command->processArgs(args, finish);
+        return command->second->processArgs(args, finish);
     else
         return Args::processArgs(args, finish);
 }

src/libutil/args.hh

@@ -32,13 +32,59 @@ protected:
     struct Flag
     {
         typedef std::shared_ptr<Flag> ptr;
+
+        struct Handler
+        {
+            std::function<void(std::vector<std::string>)> fun;
+            size_t arity;
+
+            Handler() {}
+
+            Handler(std::function<void(std::vector<std::string>)> && fun)
+                : fun(std::move(fun))
+                , arity(ArityAny)
+            { }
+
+            Handler(std::function<void()> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
+                , arity(0)
+            { }
+
+            Handler(std::function<void(std::string)> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+                    handler(std::move(ss[0]));
+                  })
+                , arity(1)
+            { }
+
+            Handler(std::function<void(std::string, std::string)> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+                    handler(std::move(ss[0]), std::move(ss[1]));
+                  })
+                , arity(2)
+            { }
+
+            template<class T>
+            Handler(T * dest)
+                : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
+                , arity(1)
+            { }
+
+            template<class T>
+            Handler(T * dest, const T & val)
+                : fun([=](std::vector<std::string> ss) { *dest = val; })
+                , arity(0)
+            { }
+        };
+
         std::string longName;
         char shortName = 0;
         std::string description;
-        Strings labels;
-        size_t arity = 0;
-        std::function<void(std::vector<std::string>)> handler;
         std::string category;
+        Strings labels;
+        Handler handler;
+
+        static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
     };
 
     std::map<std::string, Flag::ptr> longFlags;
@@ -65,49 +111,7 @@ protected:
 
 public:
 
-    class FlagMaker
-    {
-        Args & args;
-        Flag::ptr flag;
-        friend class Args;
-        FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { }
-    public:
-        ~FlagMaker();
-        FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }
-        FlagMaker & shortName(char s) { flag->shortName = s; return *this; }
-        FlagMaker & description(const std::string & s) { flag->description = s; return *this; }
-        FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }
-        FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }
-        FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }
-        FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; }
-        FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; }
-        FlagMaker & handler(std::function<void(std::string)> handler) {
-            flag->arity = 1;
-            flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
-            return *this;
-        }
-        FlagMaker & category(const std::string & s) { flag->category = s; return *this; }
-
-        template<class T>
-        FlagMaker & dest(T * dest)
-        {
-            flag->arity = 1;
-            flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
-            return *this;
-        }
-
-        template<class T>
-        FlagMaker & set(T * dest, const T & val)
-        {
-            flag->arity = 0;
-            flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
-            return *this;
-        }
-
-        FlagMaker & mkHashTypeFlag(HashType * ht);
-    };
-
-    FlagMaker mkFlag();
+    void addFlag(Flag && flag);
 
     /* Helper functions for constructing flags / positional
        arguments. */
@@ -116,13 +120,13 @@ public:
         const std::string & label, const std::string & description,
         std::function<void(std::string)> fun)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .labels({label})
-            .description(description)
-            .arity(1)
-            .handler([=](std::vector<std::string> ss) { fun(ss[0]); });
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .labels = {label},
+            .handler = {[=](std::string s) { fun(s); }}
+        });
     }
 
     void mkFlag(char shortName, const std::string & name,
@@ -135,11 +139,12 @@ public:
     void mkFlag(char shortName, const std::string & longName, const std::string & description,
         T * dest, const T & value)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .description(description)
-            .handler([=](std::vector<std::string> ss) { *dest = value; });
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .handler = {[=]() { *dest = value; }}
+        });
     }
 
     template<class I>
@@ -155,18 +160,18 @@ public:
     void mkFlag(char shortName, const std::string & longName,
         const std::string & description, std::function<void(I)> fun)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .labels({"N"})
-            .description(description)
-            .arity(1)
-            .handler([=](std::vector<std::string> ss) {
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .labels = {"N"},
+            .handler = {[=](std::string s) {
                 I n;
-                if (!string2Int(ss[0], n))
+                if (!string2Int(s, n))
                     throw UsageError("flag '--%s' requires a integer argument", longName);
                 fun(n);
-            });
+            }}
+        });
     }
 
     /* Expect a string argument. */
@@ -192,17 +197,10 @@ public:
    run() method. */
 struct Command : virtual Args
 {
-private:
-    std::string _name;
-
     friend class MultiCommand;
 
-public:
-
     virtual ~Command() { }
 
-    std::string name() { return _name; }
-
     virtual void prepare() { };
     virtual void run() = 0;
 
@@ -216,6 +214,12 @@ public:
 
     virtual Examples examples() { return Examples(); }
 
+    typedef int Category;
+
+    static constexpr Category catDefault = 0;
+
+    virtual Category category() { return catDefault; }
+
     void printHelp(const string & programName, std::ostream & out) override;
 };
 
@@ -228,7 +232,12 @@ class MultiCommand : virtual Args
 public:
     Commands commands;
 
-    std::shared_ptr<Command> command;
+    std::map<Command::Category, std::string> categories;
+
+    std::map<std::string, std::string> deprecatedAliases;
+
+    // Selected command, if any.
+    std::optional<std::pair<std::string, ref<Command>>> command;
 
     MultiCommand(const Commands & commands);
 

src/libutil/compression.cc

@@ -481,7 +481,7 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
     else if (method == "br")
         return make_ref<BrotliCompressionSink>(nextSink);
     else
-        throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
+        throw UnknownCompressionMethod("unknown compression method '%s'", method);
 }
 
 ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)

src/libutil/config.cc

@@ -65,60 +65,63 @@ void Config::getSettings(std::map<std::string, SettingInfo> & res, bool override
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
+void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
+    unsigned int pos = 0;
+
+    while (pos < contents.size()) {
+        string line;
+        while (pos < contents.size() && contents[pos] != '\n')
+            line += contents[pos++];
+        pos++;
+
+        string::size_type hash = line.find('#');
+        if (hash != string::npos)
+            line = string(line, 0, hash);
+
+        vector<string> tokens = tokenizeString<vector<string> >(line);
+        if (tokens.empty()) continue;
+
+        if (tokens.size() < 2)
+            throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+
+        auto include = false;
+        auto ignoreMissing = false;
+        if (tokens[0] == "include")
+            include = true;
+        else if (tokens[0] == "!include") {
+            include = true;
+            ignoreMissing = true;
+        }
+
+        if (include) {
+            if (tokens.size() != 2)
+                throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+            auto p = absPath(tokens[1], dirOf(path));
+            if (pathExists(p)) {
+                applyConfigFile(p);
+            } else if (!ignoreMissing) {
+                throw Error("file '%1%' included from '%2%' not found", p, path);
+            }
+            continue;
+        }
+
+        if (tokens[1] != "=")
+            throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+
+        string name = tokens[0];
+
+        vector<string>::iterator i = tokens.begin();
+        advance(i, 2);
+
+        set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
+    };
+}
+
 void AbstractConfig::applyConfigFile(const Path & path)
 {
     try {
         string contents = readFile(path);
-
-        unsigned int pos = 0;
-
-        while (pos < contents.size()) {
-            string line;
-            while (pos < contents.size() && contents[pos] != '\n')
-                line += contents[pos++];
-            pos++;
-
-            string::size_type hash = line.find('#');
-            if (hash != string::npos)
-                line = string(line, 0, hash);
-
-            vector<string> tokens = tokenizeString<vector<string> >(line);
-            if (tokens.empty()) continue;
-
-            if (tokens.size() < 2)
-                throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
-
-            auto include = false;
-            auto ignoreMissing = false;
-            if (tokens[0] == "include")
-                include = true;
-            else if (tokens[0] == "!include") {
-                include = true;
-                ignoreMissing = true;
-            }
-
-            if (include) {
-                if (tokens.size() != 2)
-                    throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
-                auto p = absPath(tokens[1], dirOf(path));
-                if (pathExists(p)) {
-                    applyConfigFile(p);
-                } else if (!ignoreMissing) {
-                    throw Error("file '%1%' included from '%2%' not found", p, path);
-                }
-                continue;
-            }
-
-            if (tokens[1] != "=")
-                throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
-
-            string name = tokens[0];
-
-            vector<string>::iterator i = tokens.begin();
-            advance(i, 2);
-
-            set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
-        };
+        applyConfig(contents, path);
     } catch (SysError &) { }
 }
 
@@ -177,12 +180,13 @@ void BaseSetting<T>::toJSON(JSONPlaceholder & out)
 template<typename T>
 void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description(description)
-        .arity(1)
-        .handler([=](std::vector<std::string> ss) { overriden = true; set(ss[0]); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = description,
+        .category = category,
+        .labels = {"value"},
+        .handler = {[=](std::string s) { overriden = true; set(s); }},
+    });
 }
 
 template<> void BaseSetting<std::string>::set(const std::string & str)
@@ -227,16 +231,18 @@ template<> std::string BaseSetting<bool>::to_string() const
 
 template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description(description)
-        .handler([=](std::vector<std::string> ss) { override(true); })
-        .category(category);
-    args.mkFlag()
-        .longName("no-" + name)
-        .description(description)
-        .handler([=](std::vector<std::string> ss) { override(false); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = description,
+        .category = category,
+        .handler = {[=]() { override(true); }}
+    });
+    args.addFlag({
+        .longName = "no-" + name,
+        .description = description,
+        .category = category,
+        .handler = {[=]() { override(false); }}
+    });
 }
 
 template<> void BaseSetting<Strings>::set(const std::string & str)

src/libutil/config.hh

@@ -1,3 +1,4 @@
+#include <cassert>
 #include <map>
 #include <set>
 
@@ -7,6 +8,38 @@
 
 namespace nix {
 
+/**
+ * The Config class provides Nix runtime configurations.
+ *
+ * What is a Configuration?
+ *   A collection of uniquely named Settings.
+ *
+ * What is a Setting?
+ *   Each property that you can set in a configuration corresponds to a
+ *   `Setting`. A setting records value and description of a property
+ *   with a default and optional aliases.
+ *
+ * A valid configuration consists of settings that are registered to a
+ * `Config` object instance:
+ *
+ *   Config config;
+ *   Setting<std::string> systemSetting{&config, "x86_64-linux", "system", "the current system"};
+ *
+ * The above creates a `Config` object and registers a setting called "system"
+ * via the variable `systemSetting` with it. The setting defaults to the string
+ * "x86_64-linux", it's description is "the current system". All of the
+ * registered settings can then be accessed as shown below:
+ *
+ *   std::map<std::string, Config::SettingInfo> settings;
+ *   config.getSettings(settings);
+ *   config["system"].description == "the current system"
+ *   config["system"].value == "x86_64-linux"
+ *
+ *
+ * The above retrieves all currently known settings from the `Config` object
+ * and adds them to the `settings` map.
+ */
+
 class Args;
 class AbstractSetting;
 class JSONPlaceholder;
@@ -23,6 +56,10 @@ protected:
 
 public:
 
+    /**
+     * Sets the value referenced by `name` to `value`. Returns true if the
+     * setting is known, false otherwise.
+     */
     virtual bool set(const std::string & name, const std::string & value) = 0;
 
     struct SettingInfo
@@ -31,18 +68,52 @@ public:
         std::string description;
     };
 
+    /**
+     * Adds the currently known settings to the given result map `res`.
+     * - res: map to store settings in
+     * - overridenOnly: when set to true only overridden settings will be added to `res`
+     */
     virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
 
+    /**
+     * Parses the configuration in `contents` and applies it
+     * - contents: configuration contents to be parsed and applied
+     * - path: location of the configuration file
+     */
+    void applyConfig(const std::string & contents, const std::string & path = "<unknown>");
+
+    /**
+     * Applies a nix configuration file
+     * - path: the location of the config file to apply
+     */
     void applyConfigFile(const Path & path);
 
+    /**
+     * Resets the `overridden` flag of all Settings
+     */
     virtual void resetOverriden() = 0;
 
+    /**
+     * Outputs all settings to JSON
+     * - out: JSONObject to write the configuration to
+     */
     virtual void toJSON(JSONObject & out) = 0;
 
+    /**
+     * Converts settings to `Args` to be used on the command line interface
+     * - args: args to write to
+     * - category: category of the settings
+     */
     virtual void convertToArgs(Args & args, const std::string & category) = 0;
 
+    /**
+     * Logs a warning for each unregistered setting
+     */
     void warnUnknownSettings();
 
+    /**
+     * Re-applies all previously attempted changes to unknown settings
+     */
     void reapplyUnknownSettings();
 };
 

src/libutil/error.cc

@@ -0,0 +1,221 @@
+#include "error.hh"
+
+#include <iostream>
+#include <optional>
+#include "serialise.hh"
+#include <sstream>
+
+namespace nix {
+
+
+const std::string nativeSystem = SYSTEM;
+
+// addPrefix is used for show-trace.  Strings added with addPrefix
+// will print ahead of the error itself.
+BaseError & BaseError::addPrefix(const FormatOrString & fs)
+{
+    prefix_ = fs.s + prefix_;
+    return *this;
+}
+
+// c++ std::exception descendants must have a 'const char* what()' function.
+// This stringifies the error and caches it for use by what(), or similarly by msg().
+const string& BaseError::calcWhat() const
+{
+    if (what_.has_value())
+        return *what_;
+    else {
+        err.name = sname();
+
+        std::ostringstream oss;
+        oss << err;
+        what_ = oss.str();
+
+        return *what_;
+    }
+}
+
+std::optional<string> ErrorInfo::programName = std::nullopt;
+
+std::ostream& operator<<(std::ostream &os, const hintformat &hf)
+{
+    return os << hf.str();
+}
+
+string showErrPos(const ErrPos &errPos)
+{
+    if (errPos.line > 0) {
+        if (errPos.column > 0) {
+            return fmt("(%1%:%2%)", errPos.line, errPos.column);
+        } else {
+            return fmt("(%1%)", errPos.line);
+        }
+    }
+    else {
+        return "";
+    }
+}
+
+// if nixCode contains lines of code, print them to the ostream, indicating the error column.
+void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode)
+{
+    // previous line of code.
+    if (nixCode.prevLineOfCode.has_value()) {
+        out << std::endl
+            << fmt("%1% %|2$5d|| %3%",
+                prefix,
+                (nixCode.errPos.line - 1),
+                *nixCode.prevLineOfCode);
+    }
+
+    if (nixCode.errLineOfCode.has_value()) {
+        // line of code containing the error.
+        out << std::endl
+            << fmt("%1% %|2$5d|| %3%",
+                prefix,
+                (nixCode.errPos.line),
+                *nixCode.errLineOfCode);
+        // error arrows for the column range.
+        if (nixCode.errPos.column > 0) {
+            int start = nixCode.errPos.column;
+            std::string spaces;
+            for (int i = 0; i < start; ++i) {
+                spaces.append(" ");
+            }
+
+            std::string arrows("^");
+
+            out << std::endl
+                << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
+                    prefix,
+                    spaces,
+                    arrows);
+        }
+    }
+
+    // next line of code.
+    if (nixCode.nextLineOfCode.has_value()) {
+        out << std::endl
+            << fmt("%1% %|2$5d|| %3%",
+                prefix,
+                (nixCode.errPos.line + 1),
+                *nixCode.nextLineOfCode);
+    }
+}
+
+std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
+{
+    auto errwidth = std::max<size_t>(getWindowSize().second, 20);
+    string prefix = "";
+
+    string levelString;
+    switch (einfo.level) {
+        case Verbosity::lvlError: {
+            levelString = ANSI_RED;
+            levelString += "error:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlWarn: {
+            levelString = ANSI_YELLOW;
+            levelString += "warning:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlInfo: {
+            levelString = ANSI_GREEN;
+            levelString += "info:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlTalkative: {
+            levelString = ANSI_GREEN;
+            levelString += "talk:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlChatty: {
+            levelString = ANSI_GREEN;
+            levelString += "chat:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlVomit: {
+            levelString = ANSI_GREEN;
+            levelString += "vomit:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlDebug: {
+            levelString = ANSI_YELLOW;
+            levelString += "debug:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        default: {
+            levelString = fmt("invalid error level: %1%", einfo.level);
+            break;
+        }
+    }
+
+    auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+    auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
+
+    std::string dashes(dashwidth, '-');
+
+    // divider.
+    if (einfo.name != "")
+        out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL,
+            prefix,
+            levelString,
+            einfo.name,
+            dashes,
+            einfo.programName.value_or(""));
+    else
+        out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
+            prefix,
+            levelString,
+            dashes,
+            einfo.programName.value_or(""));
+
+    bool nl = false;  // intersperse newline between sections.
+    if (einfo.nixCode.has_value()) {
+        if (einfo.nixCode->errPos.file != "") {
+            // filename, line, column.
+            out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
+                prefix,
+                einfo.nixCode->errPos.file,
+                showErrPos(einfo.nixCode->errPos));
+        } else {
+            out << std::endl << fmt("%1%from command line argument", prefix);
+        }
+        nl = true;
+    }
+
+    // description
+    if (einfo.description != "") {
+        if (nl)
+            out << std::endl << prefix;
+        out << std::endl << prefix << einfo.description;
+        nl = true;
+    }
+
+    // lines of code.
+    if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) {
+        if (nl)
+            out << std::endl << prefix;
+        printCodeLines(out, prefix, *einfo.nixCode);
+        nl = true;
+    }
+
+    // hint
+    if (einfo.hint.has_value()) {
+        if (nl)
+            out << std::endl << prefix;
+        out << std::endl << prefix << *einfo.hint;
+        nl = true;
+    }
+
+    return out;
+}
+}

src/libutil/error.hh

@@ -0,0 +1,186 @@
+#pragma once
+
+
+#include "ref.hh"
+#include "types.hh"
+
+#include <cstring>
+#include <list>
+#include <memory>
+#include <map>
+#include <optional>
+
+#include "fmt.hh"
+
+/* Before 4.7, gcc's std::exception uses empty throw() specifiers for
+ * its (virtual) destructor and what() in c++11 mode, in violation of spec
+ */
+#ifdef __GNUC__
+#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7)
+#define EXCEPTION_NEEDS_THROW_SPEC
+#endif
+#endif
+
+namespace nix {
+
+/*
+
+This file defines two main structs/classes used in nix error handling.
+
+ErrorInfo provides a standard payload of error information, with conversion to string
+happening in the logger rather than at the call site.
+
+BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
+an ErrorInfo.
+
+ErrorInfo structs are sent to the logger as part of an exception, or directly with the
+logError or logWarning macros.
+
+See the error-demo.cc program for usage examples.
+
+*/
+
+typedef enum {
+    lvlError = 0,
+    lvlWarn,
+    lvlInfo,
+    lvlTalkative,
+    lvlChatty,
+    lvlDebug,
+    lvlVomit
+} Verbosity;
+
+// ErrPos indicates the location of an error in a nix file.
+struct ErrPos {
+    int line = 0;
+    int column = 0;
+    string file;
+
+    operator bool() const
+    {
+        return line != 0;
+    }
+
+    // convert from the Pos struct, found in libexpr.
+    template <class P>
+    ErrPos& operator=(const P &pos)
+    {
+        line = pos.line;
+        column = pos.column;
+        file = pos.file;
+        return *this;
+    }
+
+    template <class P>
+    ErrPos(const P &p)
+    {
+        *this = p;
+    }
+};
+
+struct NixCode {
+    ErrPos errPos;
+    std::optional<string> prevLineOfCode;
+    std::optional<string> errLineOfCode;
+    std::optional<string> nextLineOfCode;
+};
+
+struct ErrorInfo {
+    Verbosity level;
+    string name;
+    string description;
+    std::optional<hintformat> hint;
+    std::optional<NixCode> nixCode;
+
+    static std::optional<string> programName;
+};
+
+std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
+
+/* BaseError should generally not be caught, as it has Interrupted as
+   a subclass. Catch Error instead. */
+class BaseError : public std::exception
+{
+protected:
+    string prefix_; // used for location traces etc.
+    mutable ErrorInfo err;
+
+    mutable std::optional<string> what_;
+    const string& calcWhat() const;
+
+public:
+    unsigned int status = 1; // exit status
+
+    template<typename... Args>
+    BaseError(unsigned int status, const Args & ... args)
+        : err { .level = lvlError,
+                .hint = hintfmt(args...)
+              }
+        , status(status)
+    { }
+
+    template<typename... Args>
+    BaseError(const std::string & fs, const Args & ... args)
+        : err { .level = lvlError,
+                .hint = hintfmt(fs, args...)
+              }
+    { }
+
+    BaseError(hintformat hint)
+        : err { .level = lvlError,
+                .hint = hint
+              }
+    { }
+
+    BaseError(ErrorInfo && e)
+        : err(std::move(e))
+    { }
+
+    BaseError(const ErrorInfo & e)
+        : err(e)
+    { }
+
+    virtual const char* sname() const { return "BaseError"; }
+
+#ifdef EXCEPTION_NEEDS_THROW_SPEC
+    ~BaseError() throw () { };
+    const char * what() const throw () { return calcWhat().c_str(); }
+#else
+    const char * what() const noexcept override { return calcWhat().c_str(); }
+#endif
+
+    const string & msg() const { return calcWhat(); }
+    const string & prefix() const { return prefix_; }
+    BaseError & addPrefix(const FormatOrString & fs);
+
+    const ErrorInfo & info() { calcWhat(); return err; }
+};
+
+#define MakeError(newClass, superClass) \
+    class newClass : public superClass                  \
+    {                                                   \
+    public:                                             \
+        using superClass::superClass;                   \
+        virtual const char* sname() const override { return #newClass; } \
+    }
+
+MakeError(Error, BaseError);
+
+class SysError : public Error
+{
+public:
+    int errNo;
+
+    template<typename... Args>
+    SysError(const Args & ... args)
+      :Error("")
+    {
+        errNo = errno;
+        auto hf = hintfmt(args...);
+        err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
+    }
+
+    virtual const char* sname() const override { return "SysError"; }
+};
+
+}

src/libutil/fmt.hh

@@ -0,0 +1,139 @@
+#pragma once
+
+#include <boost/format.hpp>
+#include <string>
+#include "ansicolor.hh"
+
+
+namespace nix {
+
+
+/* Inherit some names from other namespaces for convenience. */
+using std::string;
+using boost::format;
+
+
+/* A variadic template that does nothing. Useful to call a function
+   for all variadic arguments but ignoring the result. */
+struct nop { template<typename... T> nop(T...) {} };
+
+
+struct FormatOrString
+{
+    string s;
+    FormatOrString(const string & s) : s(s) { };
+    template<class F>
+    FormatOrString(const F & f) : s(f.str()) { };
+    FormatOrString(const char * s) : s(s) { };
+};
+
+
+/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is
+   equivalent to ‘boost::format(format) % a_0 % ... %
+   ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion
+   takes place). */
+
+template<class F>
+inline void formatHelper(F & f)
+{
+}
+
+template<class F, typename T, typename... Args>
+inline void formatHelper(F & f, const T & x, const Args & ... args)
+{
+    formatHelper(f % x, args...);
+}
+
+inline std::string fmt(const std::string & s)
+{
+    return s;
+}
+
+inline std::string fmt(const char * s)
+{
+    return s;
+}
+
+inline std::string fmt(const FormatOrString & fs)
+{
+    return fs.s;
+}
+
+template<typename... Args>
+inline std::string fmt(const std::string & fs, const Args & ... args)
+{
+    boost::format f(fs);
+    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    formatHelper(f, args...);
+    return f.str();
+}
+
+// -----------------------------------------------------------------------------
+// format function for hints in errors.  same as fmt, except templated values
+// are always in yellow.
+
+template <class T>
+struct yellowtxt
+{
+    yellowtxt(const T &s) : value(s) {}
+    const T &value;
+};
+
+template <class T>
+std::ostream& operator<<(std::ostream &out, const yellowtxt<T> &y)
+{
+    return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
+}
+
+template <class T>
+struct normaltxt
+{
+    normaltxt(const T &s) : value(s) {}
+    const T &value;
+};
+
+template <class T>
+std::ostream& operator<<(std::ostream &out, const normaltxt<T> &y)
+{
+    return out << ANSI_NORMAL << y.value;
+}
+
+class hintformat
+{
+public:
+    hintformat(const string &format) :fmt(format)
+    {
+        fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    }
+
+    hintformat(const hintformat &hf)
+    : fmt(hf.fmt)
+    {}
+
+    template<class T>
+    hintformat& operator%(const T &value)
+    {
+        fmt % yellowtxt(value);
+        return *this;
+    }
+
+    std::string str() const
+    {
+        return fmt.str();
+    }
+
+private:
+    format fmt;
+};
+
+std::ostream& operator<<(std::ostream &os, const hintformat &hf);
+
+template<typename... Args>
+inline hintformat hintfmt(const std::string & fs, const Args & ... args)
+{
+    hintformat f(fs);
+    formatHelper(f, args...);
+    return f;
+}
+
+}