wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-06 21:41:48 +02:00
Code Activity

Merge branch 'master' into nixenvjsondrvpath

Browse source
This commit is contained in:
John Ericson 2023-11-18 13:47:14 -05:00 • committed by GitHub
commit 87ac33f29a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
293 changed files with 7443 additions and 3977 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/libcmd/built-path.hh
View file

@ -1,3 +1,6 @@
#pragma once
///@file
#include "derived-path.hh"
#include "realisation.hh"

4
src/libcmd/command.cc
View file

@ -175,7 +175,7 @@ void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
throw UsageError("'--all' does not expect arguments");
// XXX: Only uses opaque paths, ignores all the realisations
for (auto & p : store->queryAllValidPaths())
paths.push_back(BuiltPath::Opaque{p});
paths.emplace_back(BuiltPath::Opaque{p});
} else {
paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
if (recursive) {
@ -188,7 +188,7 @@ void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
}
store->computeFSClosure(pathsRoots, pathsClosure);
for (auto & path : pathsClosure)
paths.push_back(BuiltPath::Opaque{path});
paths.emplace_back(BuiltPath::Opaque{path});
}
}

6
src/libcmd/command.hh
View file

@ -326,6 +326,12 @@ struct MixEnvironment : virtual Args {
void setEnviron();
};
void completeFlakeInputPath(
AddCompletions & completions,
ref<EvalState> evalState,
const std::vector<FlakeRef> & flakeRefs,
std::string_view prefix);
void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
void completeFlakeRefWithFragment(

5
src/libcmd/common-eval-args.cc
View file

@ -2,7 +2,6 @@
#include "common-eval-args.hh"
#include "shared.hh"
#include "filetransfer.hh"
#include "util.hh"
#include "eval.hh"
#include "fetchers.hh"
#include "registry.hh"
@ -165,7 +164,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
return res.finish();
}
SourcePath lookupFileArg(EvalState & state, std::string_view s)
SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir)
{
if (EvalSettings::isPseudoUrl(s)) {
auto storePath = fetchers::downloadTarball(
@ -186,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s)
}
else
return state.rootPath(CanonPath::fromCwd(s));
return state.rootPath(CanonPath(s, baseDir));
}
}

3
src/libcmd/common-eval-args.hh
View file

@ -2,6 +2,7 @@
///@file
#include "args.hh"
#include "canon-path.hh"
#include "common-args.hh"
#include "search-path.hh"
@ -28,6 +29,6 @@ private:
std::map<std::string, std::string> autoArgs;
};
SourcePath lookupFileArg(EvalState & state, std::string_view s);
SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd());
}

2
src/libcmd/editor-for.cc
View file

@ -1,5 +1,5 @@
#include "util.hh"
#include "editor-for.hh"
#include "environment-variables.hh"
namespace nix {

1
src/libcmd/installable-attr-path.hh
View file

@ -4,7 +4,6 @@
#include "globals.hh"
#include "installable-value.hh"
#include "outputs-spec.hh"
#include "util.hh"
#include "command.hh"
#include "attr-path.hh"
#include "common-eval-args.hh"

45
src/libcmd/installables.cc
View file

@ -4,6 +4,7 @@
#include "installable-attr-path.hh"
#include "installable-flake.hh"
#include "outputs-spec.hh"
#include "users.hh"
#include "util.hh"
#include "command.hh"
#include "attr-path.hh"
@ -28,7 +29,7 @@
namespace nix {
static void completeFlakeInputPath(
void completeFlakeInputPath(
AddCompletions & completions,
ref<EvalState> evalState,
const std::vector<FlakeRef> & flakeRefs,
@ -46,13 +47,6 @@ MixFlakeOptions::MixFlakeOptions()
{
auto category = "Common flake-related options";
addFlag({
.longName = "recreate-lock-file",
.description = "Recreate the flake's lock file from scratch.",
.category = category,
.handler = {&lockFlags.recreateLockFile, true}
});
addFlag({
.longName = "no-update-lock-file",
.description = "Do not allow any updates to the flake's lock file.",
@ -85,19 +79,6 @@ MixFlakeOptions::MixFlakeOptions()
.handler = {&lockFlags.commitLockFile, true}
});
addFlag({
.longName = "update-input",
.description = "Update a specific flake input (ignoring its previous entry in the lock file).",
.category = category,
.labels = {"input-path"},
.handler = {[&](std::string s) {
lockFlags.inputUpdates.insert(flake::parseInputPath(s));
}},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
}}
});
addFlag({
.longName = "override-input",
.description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
@ -107,7 +88,7 @@ MixFlakeOptions::MixFlakeOptions()
lockFlags.writeLockFile = false;
lockFlags.inputOverrides.insert_or_assign(
flake::parseInputPath(inputPath),
parseFlakeRef(flakeRef, absPath("."), true));
parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true));
}},
.completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
if (n == 0) {
@ -149,7 +130,7 @@ MixFlakeOptions::MixFlakeOptions()
auto evalState = getEvalState();
auto flake = flake::lockFlake(
*evalState,
parseFlakeRef(flakeRef, absPath(".")),
parseFlakeRef(flakeRef, absPath(getCommandBaseDir())),
{ .writeLockFile = false });
for (auto & [inputName, input] : flake.lockFile.root->inputs) {
auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
@ -313,6 +294,8 @@ void completeFlakeRefWithFragment(
prefixRoot = ".";
}
auto flakeRefS = std::string(prefix.substr(0, hash));
// TODO: ideally this would use the command base directory instead of assuming ".".
auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
auto evalCache = openEvalCache(*evalState,
@ -461,10 +444,12 @@ Installables SourceExprCommand::parseInstallables(
auto e = state->parseStdin();
state->eval(e, *vFile);
}
else if (file)
state->evalFile(lookupFileArg(*state, *file), *vFile);
else if (file) {
state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile);
}
else {
auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
CanonPath dir(CanonPath::fromCwd(getCommandBaseDir()));
auto e = state->parseExprFromString(*expr, state->rootPath(dir));
state->eval(e, *vFile);
}
@ -499,7 +484,7 @@ Installables SourceExprCommand::parseInstallables(
}
try {
auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath("."));
auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir()));
result.push_back(make_ref<InstallableFlake>(
this,
getEvalState(),
@ -683,7 +668,7 @@ BuiltPaths Installable::toBuiltPaths(
BuiltPaths res;
for (auto & drvPath : Installable::toDerivations(store, installables, true))
res.push_back(BuiltPath::Opaque{drvPath});
res.emplace_back(BuiltPath::Opaque{drvPath});
return res;
}
}
@ -773,7 +758,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
for (auto i : rawInstallables)
res.push_back(parseFlakeRefWithFragment(
expandTilde(i),
absPath(".")).first);
absPath(getCommandBaseDir())).first);
return res;
}
@ -795,7 +780,7 @@ std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
return {
parseFlakeRefWithFragment(
expandTilde(_installable),
absPath(".")).first
absPath(getCommandBaseDir())).first
};
}

1
src/libcmd/installables.hh
View file

@ -1,7 +1,6 @@
#pragma once
///@file
#include "util.hh"
#include "path.hh"
#include "outputs-spec.hh"
#include "derived-path.hh"

1
src/libcmd/markdown.cc
View file

@ -1,6 +1,7 @@
#include "markdown.hh"
#include "util.hh"
#include "finally.hh"
#include "terminal.hh"
#include <sys/queue.h>
#include <lowdown.h>

3
src/libcmd/repl.cc
View file

@ -22,6 +22,7 @@ extern "C" {
#include "repl.hh"
#include "ansicolor.hh"
#include "signals.hh"
#include "shared.hh"
#include "eval.hh"
#include "eval-cache.hh"
@ -36,6 +37,8 @@ extern "C" {
#include "globals.hh"
#include "flake/flake.hh"
#include "flake/lockfile.hh"
#include "users.hh"
#include "terminal.hh"
#include "editor-for.hh"
#include "finally.hh"
#include "markdown.hh"

1
src/libexpr/attr-path.cc
View file

@ -1,6 +1,5 @@
#include "attr-path.hh"
#include "eval-inline.hh"
#include "util.hh"
namespace nix {

1
src/libexpr/eval-cache.cc
View file

@ -1,3 +1,4 @@
#include "users.hh"
#include "eval-cache.hh"
#include "sqlite.hh"
#include "eval.hh"

1
src/libexpr/eval-settings.cc
View file

@ -1,3 +1,4 @@
#include "users.hh"
#include "globals.hh"
#include "profiles.hh"
#include "eval.hh"

2
src/libexpr/eval-settings.hh
View file

@ -1,4 +1,6 @@
#pragma once
///@file
#include "config.hh"
namespace nix {

36
src/libexpr/eval.cc
View file

@ -1,6 +1,7 @@
#include "eval.hh"
#include "eval-settings.hh"
#include "hash.hh"
#include "primops.hh"
#include "types.hh"
#include "util.hh"
#include "store-api.hh"
@ -14,6 +15,7 @@
#include "print.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
#include "signals.hh"
#include <algorithm>
#include <chrono>
@ -29,6 +31,7 @@
#include <sys/resource.h>
#include <nlohmann/json.hpp>
#include <boost/container/small_vector.hpp>
#if HAVE_BOEHMGC
@ -721,6 +724,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
}
void PrimOp::check()
{
if (arity > maxPrimOpArity) {
throw Error("primop arity must not exceed %1%", maxPrimOpArity);
}
}
void Value::mkPrimOp(PrimOp * p)
{
p->check();
clearValue();
internalType = tPrimOp;
primOp = p;
}
Value * EvalState::addPrimOp(PrimOp && primOp)
{
/* Hack to make constants lazy: turn them into a application of
@ -1690,7 +1710,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
/* We have all the arguments, so call the primop with
the previous and new arguments. */
Value * vArgs[arity];
Value * vArgs[maxPrimOpArity];
auto n = argsDone;
for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
vArgs[--n] = arg->primOpApp.right;
@ -1747,11 +1767,17 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
Value vFun;
fun->eval(state, env, vFun);
Value * vArgs[args.size()];
// Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5}
// 2: over 4000
// 3: about 300
// 4: about 60
// 5: under 10
// This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
boost::container::small_vector<Value *, 4> vArgs(args.size());
for (size_t i = 0; i < args.size(); ++i)
vArgs[i] = args[i]->maybeThunk(state, env);
state.callFunction(vFun, args.size(), vArgs, v, pos);
state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
}
@ -1990,8 +2016,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
return result;
};
Value values[es->size()];
Value * vTmpP = values;
boost::container::small_vector<Value, conservativeStackReservation> values(es->size());
Value * vTmpP = values.data();
for (auto & [i_pos, i] : *es) {
Value & vTmp = *vTmpP++;

14
src/libexpr/eval.hh
View file

@ -18,6 +18,12 @@
namespace nix {
/**
* We put a limit on primop arity because it lets us use a fixed size array on
* the stack. 8 is already an impractical number of arguments. Use an attrset
* argument for such overly complicated functions.
*/
constexpr size_t maxPrimOpArity = 8;
class Store;
class EvalState;
@ -71,6 +77,12 @@ struct PrimOp
* Optional experimental for this to be gated on.
*/
std::optional<ExperimentalFeature> experimentalFeature;
/**
* Validity check to be performed by functions that introduce primops,
* such as RegisterPrimOp() and Value::mkPrimOp().
*/
void check();
};
/**
@ -827,7 +839,7 @@ std::string showType(const Value & v);
/**
* If `path` refers to a directory, then append "/default.nix".
*/
SourcePath resolveExprPath(const SourcePath & path);
SourcePath resolveExprPath(SourcePath path);
struct InvalidPathError : EvalError
{

3
src/libexpr/flake/config.cc
View file

@ -1,6 +1,7 @@
#include "flake.hh"
#include "users.hh"
#include "globals.hh"
#include "fetch-settings.hh"
#include "flake.hh"
#include <nlohmann/json.hpp>

86
src/libexpr/flake/flake.cc
View file

@ -1,3 +1,4 @@
#include "terminal.hh"
#include "flake.hh"
#include "eval.hh"
#include "eval-settings.hh"
@ -8,6 +9,7 @@
#include "fetchers.hh"
#include "finally.hh"
#include "fetch-settings.hh"
#include "value-to-json.hh"
namespace nix {
@ -140,8 +142,13 @@ static FlakeInput parseFlakeInput(EvalState & state,
attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer);
break;
default:
throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
state.symbols[attr.name], showType(*attr.value));
if (attr.name == state.symbols.create("publicKeys")) {
experimentalFeatureSettings.require(Xp::VerifiedFetches);
NixStringContext emptyContext = {};
attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump());
} else
throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
state.symbols[attr.name], showType(*attr.value));
}
#pragma GCC diagnostic pop
}
@ -447,8 +454,8 @@ LockedFlake lockFlake(
assert(input.ref);
/* Do we have an entry in the existing lock file? And we
don't have a --update-input flag for this input? */
/* Do we have an entry in the existing lock file?
And the input is not in updateInputs? */
std::shared_ptr<LockedNode> oldLock;
updatesUsed.insert(inputPath);
@ -472,9 +479,8 @@ LockedFlake lockFlake(
node->inputs.insert_or_assign(id, childNode);
/* If we have an --update-input flag for an input
of this input, then we must fetch the flake to
update it. */
/* If we have this input in updateInputs, then we
must fetch the flake to update it. */
auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
auto mustRefetch =
@ -616,19 +622,14 @@ LockedFlake lockFlake(
for (auto & i : lockFlags.inputUpdates)
if (!updatesUsed.count(i))
warn("the flag '--update-input %s' does not match any input", printInputPath(i));
warn("'%s' does not match any input of this flake", printInputPath(i));
/* Check 'follows' inputs. */
newLockFile.check();
debug("new lock file: %s", newLockFile);
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
auto sourcePath = topRef.input.getSourcePath();
auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt;
if (lockFlags.outputLockFilePath) {
outputLockFilePath = lockFlags.outputLockFilePath;
}
/* Check whether we need to / can write the new lock file. */
if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) {
@ -636,7 +637,7 @@ LockedFlake lockFlake(
auto diff = LockFile::diff(oldLockFile, newLockFile);
if (lockFlags.writeLockFile) {
if (outputLockFilePath) {
if (sourcePath || lockFlags.outputLockFilePath) {
if (auto unlockedInput = newLockFile.isUnlocked()) {
if (fetchSettings.warnDirty)
warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput);
@ -644,41 +645,48 @@ LockedFlake lockFlake(
if (!lockFlags.updateLockFile)
throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
bool lockFileExists = pathExists(*outputLockFilePath);
auto newLockFileS = fmt("%s\n", newLockFile);
if (lockFlags.outputLockFilePath) {
if (lockFlags.commitLockFile)
throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible");
writeFile(*lockFlags.outputLockFilePath, newLockFileS);
} else {
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
auto outputLockFilePath = *sourcePath + "/" + relPath;
bool lockFileExists = pathExists(outputLockFilePath);
if (lockFileExists) {
auto s = chomp(diff);
if (s.empty())
warn("updating lock file '%s'", *outputLockFilePath);
else
warn("updating lock file '%s':\n%s", *outputLockFilePath, s);
} else
warn("creating lock file '%s'", *outputLockFilePath);
if (lockFileExists) {
if (s.empty())
warn("updating lock file '%s'", outputLockFilePath);
else
warn("updating lock file '%s':\n%s", outputLockFilePath, s);
} else
warn("creating lock file '%s': \n%s", outputLockFilePath, s);
newLockFile.write(*outputLockFilePath);
std::optional<std::string> commitMessage = std::nullopt;
std::optional<std::string> commitMessage = std::nullopt;
if (lockFlags.commitLockFile) {
if (lockFlags.outputLockFilePath) {
throw Error("--commit-lock-file and --output-lock-file are currently incompatible");
}
std::string cm;
if (lockFlags.commitLockFile) {
std::string cm;
cm = fetchSettings.commitLockFileSummary.get();
cm = fetchSettings.commitLockFileSummary.get();
if (cm == "") {
cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
if (cm == "") {
cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
}
cm += "\n\nFlake lock file updates:\n\n";
cm += filterANSIEscapes(diff, true);
commitMessage = cm;
}
cm += "\n\nFlake lock file updates:\n\n";
cm += filterANSIEscapes(diff, true);
commitMessage = cm;
topRef.input.putFile(
CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
newLockFileS, commitMessage);
}
topRef.input.markChangedFile(
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
commitMessage);
/* Rewriting the lockfile changed the top-level
repo, so we should re-read it. FIXME: we could
also just clear the 'rev' field... */

6
src/libexpr/flake/lockfile.cc
View file

@ -214,12 +214,6 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
return stream;
}
void LockFile::write(const Path & path) const
{
createDirs(dirOf(path));
writeFile(path, fmt("%s\n", *this));
}
std::optional<FlakeRef> LockFile::isUnlocked() const
{
std::set<ref<const Node>> nodes;

2
src/libexpr/flake/lockfile.hh
View file

@ -65,8 +65,6 @@ struct LockFile
static LockFile read(const Path & path);
void write(const Path & path) const;
/**
* Check whether this lock file has any unlocked inputs.
*/

1
src/libexpr/get-drvs.cc
View file

@ -1,5 +1,4 @@
#include "get-drvs.hh"
#include "util.hh"
#include "eval-inline.hh"
#include "derivations.hh"
#include "store-api.hh"

4
src/libexpr/local.mk
View file

@ -43,7 +43,9 @@ $(foreach i, $(wildcard src/libexpr/value/*.hh), \
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh

19
src/libexpr/parser.y
View file

@ -19,6 +19,7 @@
#include <variant>
#include "util.hh"
#include "users.hh"
#include "nixexpr.hh"
#include "eval.hh"
@ -685,17 +686,25 @@ Expr * EvalState::parse(
}
SourcePath resolveExprPath(const SourcePath & path)
SourcePath resolveExprPath(SourcePath path)
{
unsigned int followCount = 0, maxFollow = 1024;
/* If `path' is a symlink, follow it. This is so that relative
path references work. */
auto path2 = path.resolveSymlinks();
while (true) {
// Basic cycle/depth limit to avoid infinite loops.
if (++followCount >= maxFollow)
throw Error("too many symbolic links encountered while traversing the path '%s'", path);
if (path.lstat().type != InputAccessor::tSymlink) break;
path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))};
}
/* If `path' refers to a directory, append `/default.nix'. */
if (path2.lstat().type == InputAccessor::tDirectory)
return path2 + "default.nix";
if (path.lstat().type == InputAccessor::tDirectory)
return path + "default.nix";
return path2;
return path;
}

28
src/libexpr/primops.cc
View file

@ -10,6 +10,7 @@
#include "path-references.hh"
#include "store-api.hh"
#include "util.hh"
#include "processes.hh"
#include "value-to-json.hh"
#include "value-to-xml.hh"
#include "primops.hh"
@ -28,7 +29,6 @@
#include <cmath>
namespace nix {
@ -824,7 +824,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
auto message = state.coerceToString(pos, *args[0], context,
"while evaluating the error message passed to builtins.addErrorContext",
false, false).toOwned();
e.addTrace(nullptr, message, true);
e.addTrace(nullptr, hintfmt(message), true);
throw;
}
}
@ -1548,10 +1548,8 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
try {
auto checked = state.checkSourcePath(path);
auto exists = checked.pathExists();
if (exists && mustBeDir) {
exists = checked.lstat().type == InputAccessor::tDirectory;
}
auto st = checked.maybeLstat();
auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
v.mkBool(exists);
} catch (SysError & e) {
/* Don't give away info from errors while canonicalising
@ -2551,6 +2549,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args
/* Get the attribute names to be removed.
We keep them as Attrs instead of Symbols so std::set_difference
can be used to remove them from attrs[0]. */
// 64: large enough to fit the attributes of a derivation
boost::container::small_vector<Attr, 64> names;
names.reserve(args[1]->listSize());
for (auto elem : args[1]->listItems()) {
@ -2730,8 +2729,8 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
Value * res[args[1]->listSize()];
unsigned int found = 0;
boost::container::small_vector<Value *, nonRecursiveStackReservation> res(args[1]->listSize());
size_t found = 0;
for (auto v2 : args[1]->listItems()) {
state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs");
@ -3065,9 +3064,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
// FIXME: putting this on the stack is risky.
Value * vs[args[1]->listSize()];
unsigned int k = 0;
boost::container::small_vector<Value *, nonRecursiveStackReservation> vs(args[1]->listSize());
size_t k = 0;
bool same = true;
for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
@ -3192,10 +3190,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar
state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all"));
state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all"));
std::string_view errorCtx = any
? "while evaluating the return value of the function passed to builtins.any"
: "while evaluating the return value of the function passed to builtins.all";
Value vTmp;
for (auto elem : args[1]->listItems()) {
state.callFunction(*args[0], *elem, vTmp, pos);
bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all"));
bool res = state.forceBool(vTmp, pos, errorCtx);
if (res == any) {
v.mkBool(any);
return;
@ -3451,7 +3453,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
auto nrLists = args[1]->listSize();
Value lists[nrLists];
boost::container::small_vector<Value, conservativeStackReservation> lists(nrLists);
size_t len = 0;
for (unsigned int n = 0; n < nrLists; ++n) {

16
src/libexpr/primops.hh
View file

@ -8,6 +8,22 @@
namespace nix {
/**
* For functions where we do not expect deep recursion, we can use a sizable
* part of the stack a free allocation space.
*
* Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
*/
constexpr size_t nonRecursiveStackReservation = 128;
/**
* Functions that maybe applied to self-similar inputs, such as concatMap on a
* tree, should reserve a smaller part of the stack for allocation.
*
* Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
*/
constexpr size_t conservativeStackReservation = 16;
struct RegisterPrimOp
{
typedef std::vector<PrimOp> PrimOps;

57
src/libexpr/primops/fetchTree.cc
View file

@ -7,6 +7,7 @@
#include "registry.hh"
#include "tarball.hh"
#include "url.hh"
#include "value-to-json.hh"
#include <ctime>
#include <iomanip>
@ -125,6 +126,10 @@ static void fetchTree(
attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean});
else if (attr.value->type() == nInt)
attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer));
else if (state.symbols[attr.name] == "publicKeys") {
experimentalFeatureSettings.require(Xp::VerifiedFetches);
attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump());
}
else
state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
state.symbols[attr.name], showType(*attr.value)));
@ -223,6 +228,7 @@ static RegisterPrimOp primop_fetchTree({
```
)",
.fun = prim_fetchTree,
.experimentalFeature = Xp::FetchTree,
});
static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v,
@ -427,6 +433,42 @@ static RegisterPrimOp primop_fetchGit({
With this argument being true, it's possible to load a `rev` from *any* `ref`
(by default only `rev`s from the specified `ref` are supported).
- `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`)
Whether to check `rev` for a signature matching `publicKey` or `publicKeys`.
If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes.
Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
- `publicKey`
The public key against which `rev` is verified if `verifyCommit` is enabled.
Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
- `keytype` (default: `"ssh-ed25519"`)
The key type of `publicKey`.
Possible values:
- `"ssh-dsa"`
- `"ssh-ecdsa"`
- `"ssh-ecdsa-sk"`
- `"ssh-ed25519"`
- `"ssh-ed25519-sk"`
- `"ssh-rsa"`
Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
- `publicKeys`
The public keys against which `rev` is verified if `verifyCommit` is enabled.
Must be given as a list of attribute sets with the following form:
```nix
{
key = "<public key>";
type = "<key type>"; # optional, default: "ssh-ed25519"
}
```
Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
Here are some examples of how to use `fetchGit`.
- To fetch a private repository over SSH:
@ -501,6 +543,21 @@ static RegisterPrimOp primop_fetchGit({
}
```
- To verify the commit signature:
```nix
builtins.fetchGit {
url = "ssh://git@github.com/nixos/nix.git";
verifyCommit = true;
publicKeys = [
{
type = "ssh-ed25519";
key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi";
}
];
}
```
Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting.
This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).

1
src/libexpr/search-path.cc
View file

@ -1,5 +1,4 @@
#include "search-path.hh"
#include "util.hh"
namespace nix {

6
src/libexpr/tests/local.mk
View file

@ -6,7 +6,11 @@ libexpr-tests_NAME := libnixexpr-tests
libexpr-tests_DIR := $(d)
libexpr-tests_INSTALL_DIR :=
ifeq ($(INSTALL_UNIT_TESTS), yes)
libexpr-tests_INSTALL_DIR := $(checkbindir)
else
libexpr-tests_INSTALL_DIR :=
endif
libexpr-tests_SOURCES := \
$(wildcard $(d)/*.cc) \

3
src/libexpr/tests/value/print.cc
View file

@ -114,7 +114,8 @@ TEST_F(ValuePrintingTests, vLambda)
TEST_F(ValuePrintingTests, vPrimOp)
{
Value vPrimOp;
vPrimOp.mkPrimOp(nullptr);
PrimOp primOp{};
vPrimOp.mkPrimOp(&primOp);
test(vPrimOp, "<PRIMOP>");
}

2
src/libexpr/value-to-json.cc
View file

@ -1,7 +1,7 @@
#include "value-to-json.hh"
#include "eval-inline.hh"
#include "util.hh"
#include "store-api.hh"
#include "signals.hh"
#include <cstdlib>
#include <iomanip>

2
src/libexpr/value-to-xml.cc
View file

@ -1,7 +1,7 @@
#include "value-to-xml.hh"
#include "xml-writer.hh"
#include "eval-inline.hh"
#include "util.hh"
#include "signals.hh"
#include <cstdlib>

8
src/libexpr/value.hh
View file

@ -354,13 +354,7 @@ public:
// Value will be overridden anyways
}
inline void mkPrimOp(PrimOp * p)
{
clearValue();
internalType = tPrimOp;
primOp = p;
}
void mkPrimOp(PrimOp * p);
inline void mkPrimOpApp(Value * l, Value * r)
{

1
src/libexpr/value/context.cc
View file

@ -1,3 +1,4 @@
#include "util.hh"
#include "value/context.hh"
#include <optional>

1
src/libexpr/value/context.hh
View file

@ -1,7 +1,6 @@
#pragma once
///@file
#include "util.hh"
#include "comparator.hh"
#include "derived-path.hh"
#include "variant-wrapper.hh"

1
src/libfetchers/cache.cc
View file

@ -1,4 +1,5 @@
#include "cache.hh"
#include "users.hh"
#include "sqlite.hh"
#include "sync.hh"
#include "store-api.hh"

1
src/libfetchers/cache.hh
View file

@ -2,6 +2,7 @@
///@file
#include "fetchers.hh"
#include "path.hh"
namespace nix::fetchers {

1
src/libfetchers/fetch-settings.hh
View file

@ -3,7 +3,6 @@
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include <map>
#include <limits>

101
src/libfetchers/fetchers.cc
View file

@ -5,12 +5,31 @@
namespace nix::fetchers {
std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
std::unique_ptr<InputSchemeMap> inputSchemes = nullptr;
void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
{
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
inputSchemes->push_back(std::move(inputScheme));
if (!inputSchemes)
inputSchemes = std::make_unique<InputSchemeMap>();
auto schemeName = inputScheme->schemeName();
if (inputSchemes->count(schemeName) > 0)
throw Error("Input scheme with name %s already registered", schemeName);
inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
}
nlohmann::json dumpRegisterInputSchemeInfo() {
using nlohmann::json;
auto res = json::object();
for (auto & [name, scheme] : *inputSchemes) {
auto & r = res[name] = json::object();
r["allowedAttrs"] = scheme->allowedAttrs();
}
return res;
}
Input Input::fromURL(const std::string & url, bool requireTree)
@ -33,7 +52,7 @@ static void fixupInput(Input & input)
Input Input::fromURL(const ParsedURL & url, bool requireTree)
{
for (auto & inputScheme : *inputSchemes) {
for (auto & [_, inputScheme] : *inputSchemes) {
auto res = inputScheme->inputFromURL(url, requireTree);
if (res) {
experimentalFeatureSettings.require(inputScheme->experimentalFeature());
@ -48,20 +67,44 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree)
Input Input::fromAttrs(Attrs && attrs)
{
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromAttrs(attrs);
if (res) {
experimentalFeatureSettings.require(inputScheme->experimentalFeature());
res->scheme = inputScheme;
fixupInput(*res);
return std::move(*res);
}
}
auto schemeName = ({
auto schemeNameOpt = maybeGetStrAttr(attrs, "type");
if (!schemeNameOpt)
throw Error("'type' attribute to specify input scheme is required but not provided");
*std::move(schemeNameOpt);
});
Input input;
input.attrs = attrs;
fixupInput(input);
return input;
auto raw = [&]() {
// Return an input without a scheme; most operations will fail,
// but not all of them. Doing this is to support those other
// operations which are supposed to be robust on
// unknown/uninterpretable inputs.
Input input;
input.attrs = attrs;
fixupInput(input);
return input;
};
std::shared_ptr<InputScheme> inputScheme = ({
auto i = inputSchemes->find(schemeName);
i == inputSchemes->end() ? nullptr : i->second;
});
if (!inputScheme) return raw();
experimentalFeatureSettings.require(inputScheme->experimentalFeature());
auto allowedAttrs = inputScheme->allowedAttrs();
for (auto & [name, _] : attrs)
if (name != "type" && allowedAttrs.count(name) == 0)
throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName);
auto res = inputScheme->inputFromAttrs(attrs);
if (!res) return raw();
res->scheme = inputScheme;
fixupInput(*res);
return std::move(*res);
}
ParsedURL Input::toURL() const
@ -196,12 +239,13 @@ std::optional<Path> Input::getSourcePath() const
return scheme->getSourcePath(*this);
}
void Input::markChangedFile(
std::string_view file,
void Input::putFile(
const CanonPath & path,
std::string_view contents,
std::optional<std::string> commitMsg) const
{
assert(scheme);
return scheme->markChangedFile(*this, file, commitMsg);
return scheme->putFile(*this, path, contents, commitMsg);
}
std::string Input::getName() const
@ -292,14 +336,18 @@ Input InputScheme::applyOverrides(
return input;
}
std::optional<Path> InputScheme::getSourcePath(const Input & input)
std::optional<Path> InputScheme::getSourcePath(const Input & input) const
{
return {};
}
void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
void InputScheme::putFile(
const Input & input,
const CanonPath & path,
std::string_view contents,
std::optional<std::string> commitMsg) const
{
assert(false);
throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
}
void InputScheme::clone(const Input & input, const Path & destDir) const
@ -307,9 +355,14 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
throw Error("do not know how to clone input '%s'", input.to_string());
}
std::optional<ExperimentalFeature> InputScheme::experimentalFeature()
std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
{
return {};
}
std::string publicKeys_to_string(const std::vector<PublicKey>& publicKeys)
{
return ((nlohmann::json) publicKeys).dump();
}
}

53
src/libfetchers/fetchers.hh
View file

@ -3,13 +3,14 @@
#include "types.hh"
#include "hash.hh"
#include "path.hh"
#include "canon-path.hh"
#include "attrs.hh"
#include "url.hh"
#include <memory>
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; }
namespace nix { class Store; class StorePath; }
namespace nix::fetchers {
@ -90,8 +91,13 @@ public:
std::optional<Path> getSourcePath() const;
void markChangedFile(
std::string_view file,
/**
* Write a file to this input, for input types that support
* writing. Optionally commit the change (for e.g. Git inputs).
*/
void putFile(
const CanonPath & path,
std::string_view contents,
std::optional<std::string> commitMsg) const;
std::string getName() const;
@ -126,6 +132,24 @@ struct InputScheme
virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0;
/**
* What is the name of the scheme?
*
* The `type` attribute is used to select which input scheme is
* used, and then the other fields are forwarded to that input
* scheme.
*/
virtual std::string_view schemeName() const = 0;
/**
* Allowed attributes in an attribute set that is converted to an
* input.
*
* `type` is not included from this set, because the `type` field is
parsed first to choose which scheme; `type` is always required.
*/
virtual StringSet allowedAttrs() const = 0;
virtual ParsedURL toURL(const Input & input) const;
virtual Input applyOverrides(
@ -135,16 +159,20 @@ struct InputScheme
virtual void clone(const Input & input, const Path & destDir) const;
virtual std::optional<Path> getSourcePath(const Input & input);
virtual std::optional<Path> getSourcePath(const Input & input) const;
virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
virtual void putFile(
const Input & input,
const CanonPath & path,
std::string_view contents,
std::optional<std::string> commitMsg) const;
virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
/**
* Is this `InputScheme` part of an experimental feature?
*/
virtual std::optional<ExperimentalFeature> experimentalFeature();
virtual std::optional<ExperimentalFeature> experimentalFeature() const;
virtual bool isDirect(const Input & input) const
{ return true; }
@ -152,4 +180,15 @@ struct InputScheme
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
nlohmann::json dumpRegisterInputSchemeInfo();
struct PublicKey
{
std::string type = "ssh-ed25519";
std::string key;
};
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key)
std::string publicKeys_to_string(const std::vector<PublicKey>&);
}

4
src/libfetchers/fs-input-accessor.cc
View file

@ -36,11 +36,11 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
}
Stat lstat(const CanonPath & path) override
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::lstat(absPath);
return PosixSourceAccessor::maybeLstat(absPath);
}
DirEntries readDirectory(const CanonPath & path) override

155
src/libfetchers/git.cc
View file

@ -1,11 +1,12 @@
#include "fetchers.hh"
#include "users.hh"
#include "cache.hh"
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include "pathlocks.hh"
#include "util.hh"
#include "processes.hh"
#include "git.hh"
#include "fetch-settings.hh"
@ -143,6 +144,69 @@ struct WorkdirInfo
bool hasHead = false;
};
std::vector<PublicKey> getPublicKeys(const Attrs & attrs) {
std::vector<PublicKey> publicKeys;
if (attrs.contains("publicKeys")) {
nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys"));
ensureType(publicKeysJson, nlohmann::json::value_t::array);
publicKeys = publicKeysJson.get<std::vector<PublicKey>>();
}
else {
publicKeys = {};
}
if (attrs.contains("publicKey"))
publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")});
return publicKeys;
}
void doCommitVerification(const Path repoDir, const Path gitDir, const std::string rev, const std::vector<PublicKey>& publicKeys) {
// Create ad-hoc allowedSignersFile and populate it with publicKeys
auto allowedSignersFile = createTempFile().second;
std::string allowedSigners;
for (const PublicKey& k : publicKeys) {
if (k.type != "ssh-dsa"
&& k.type != "ssh-ecdsa"
&& k.type != "ssh-ecdsa-sk"
&& k.type != "ssh-ed25519"
&& k.type != "ssh-ed25519-sk"
&& k.type != "ssh-rsa")
warn("Unknown keytype: %s\n"
"Please use one of\n"
"- ssh-dsa\n"
" ssh-ecdsa\n"
" ssh-ecdsa-sk\n"
" ssh-ed25519\n"
" ssh-ed25519-sk\n"
" ssh-rsa", k.type);
allowedSigners += "* " + k.type + " " + k.key + "\n";
}
writeFile(allowedSignersFile, allowedSigners);
// Run verification command
auto [status, output] = runProgram(RunOptions {
.program = "git",
.args = {"-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, "-C", repoDir,
"--git-dir", gitDir, "verify-commit", rev},
.mergeStderrToStdout = true,
});
/* Evaluate result through status code and checking if public key fingerprints appear on stderr
* This is neccessary because the git command might also succeed due to the commit being signed by gpg keys
* that are present in the users key agent. */
std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
for (const PublicKey& k : publicKeys){
// Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
re += "(" + escaped_fingerprint + ")";
}
re += "]";
if (status == 0 && std::regex_search(output, std::regex(re)))
printTalkative("Signature verification on commit %s succeeded", rev);
else
throw Error("Commit signature verification on commit %s failed: \n%s", rev, output);
}
// Returns whether a git workdir is clean and has commits.
WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
{
@ -272,9 +336,9 @@ struct GitInputScheme : InputScheme
attrs.emplace("type", "git");
for (auto & [name, value] : url.query) {
if (name == "rev" || name == "ref")
if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys")
attrs.emplace(name, value);
else if (name == "shallow" || name == "submodules" || name == "allRefs")
else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit")
attrs.emplace(name, Explicit<bool> { value == "1" });
else
url2.query.emplace(name, value);
@ -285,17 +349,47 @@ struct GitInputScheme : InputScheme
return inputFromAttrs(attrs);
}
std::string_view schemeName() const override
{
return "git";
}
StringSet allowedAttrs() const override
{
return {
"url",
"ref",
"rev",
"shallow",
"submodules",
"lastModified",
"revCount",
"narHash",
"allRefs",
"name",
"dirtyRev",
"dirtyShortRev",
"verifyCommit",
"keytype",
"publicKey",
"publicKeys",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
throw Error("unsupported Git input attribute '%s'", name);
for (auto & [name, _] : attrs)
if (name == "verifyCommit"
|| name == "keytype"
|| name == "publicKey"
|| name == "publicKeys")
experimentalFeatureSettings.require(Xp::VerifiedFetches);
maybeGetBoolAttr(attrs, "shallow");
maybeGetBoolAttr(attrs, "submodules");
maybeGetBoolAttr(attrs, "allRefs");
maybeGetBoolAttr(attrs, "verifyCommit");
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (std::regex_search(*ref, badGitRefRegex))
@ -318,6 +412,15 @@ struct GitInputScheme : InputScheme
if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
url.query.insert_or_assign("shallow", "1");
if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
url.query.insert_or_assign("verifyCommit", "1");
auto publicKeys = getPublicKeys(input.attrs);
if (publicKeys.size() == 1) {
url.query.insert_or_assign("keytype", publicKeys.at(0).type);
url.query.insert_or_assign("publicKey", publicKeys.at(0).key);
}
else if (publicKeys.size() > 1)
url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys));
return url;
}
@ -354,7 +457,7 @@ struct GitInputScheme : InputScheme
runProgram("git", true, args, {}, true);
}
std::optional<Path> getSourcePath(const Input & input) override
std::optional<Path> getSourcePath(const Input & input) const override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
@ -362,18 +465,26 @@ struct GitInputScheme : InputScheme
return {};
}
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
void putFile(
const Input & input,
const CanonPath & path,
std::string_view contents,
std::optional<std::string> commitMsg) const override
{
auto sourcePath = getSourcePath(input);
assert(sourcePath);
auto root = getSourcePath(input);
if (!root)
throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
writeFile((CanonPath(*root) + path).abs(), contents);
auto gitDir = ".git";
runProgram("git", true,
{ "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) });
{ "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
if (commitMsg)
runProgram("git", true,
{ "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg });
{ "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
}
std::pair<bool, std::string> getActualUrl(const Input & input) const
@ -399,6 +510,8 @@ struct GitInputScheme : InputScheme
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
std::vector<PublicKey> publicKeys = getPublicKeys(input.attrs);
bool verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty());
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
@ -419,6 +532,8 @@ struct GitInputScheme : InputScheme
{"type", cacheType},
{"name", name},
{"rev", input.getRev()->gitRev()},
{"verifyCommit", verifyCommit},
{"publicKeys", publicKeys_to_string(publicKeys)},
});
};
@ -441,12 +556,15 @@ struct GitInputScheme : InputScheme
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
/* If this is a local directory and no ref or revision is given,
/* If this is a local directory, no ref or revision is given and no signature verification is needed,
allow fetching directly from a dirty workdir. */
if (!input.getRef() && !input.getRev() && isLocal) {
auto workdirInfo = getWorkdirInfo(input, actualUrl);
if (!workdirInfo.clean) {
return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
if (verifyCommit)
throw Error("Can't fetch from a dirty workdir with commit signature verification enabled.");
else
return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
}
}
@ -454,6 +572,8 @@ struct GitInputScheme : InputScheme
{"type", cacheType},
{"name", name},
{"url", actualUrl},
{"verifyCommit", verifyCommit},
{"publicKeys", publicKeys_to_string(publicKeys)},
});
Path repoDir;
@ -611,6 +731,9 @@ struct GitInputScheme : InputScheme
);
}
if (verifyCommit)
doCommitVerification(repoDir, gitDir, input.getRev()->gitRev(), publicKeys);
if (submodules) {
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);

35
src/libfetchers/github.cc
View file

@ -27,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
struct GitArchiveInputScheme : InputScheme
{
virtual std::string type() const = 0;
virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
{
if (url.scheme != type()) return {};
if (url.scheme != schemeName()) return {};
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
@ -91,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
Input input;
input.attrs.insert_or_assign("type", type());
input.attrs.insert_or_assign("type", std::string { schemeName() });
input.attrs.insert_or_assign("owner", path[0]);
input.attrs.insert_or_assign("repo", path[1]);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
@ -101,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
{
return {
"owner",
"repo",
"ref",
"rev",
"narHash",
"lastModified",
"host",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{
if (maybeGetStrAttr(attrs, "type") != type()) return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
throw Error("unsupported input attribute '%s'", name);
getStrAttr(attrs, "owner");
getStrAttr(attrs, "repo");
@ -128,7 +133,7 @@ struct GitArchiveInputScheme : InputScheme
if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(HashFormat::Base16, false);
return ParsedURL {
.scheme = type(),
.scheme = std::string { schemeName() },
.path = path,
};
}
@ -220,7 +225,7 @@ struct GitArchiveInputScheme : InputScheme
return {result.storePath, input};
}
std::optional<ExperimentalFeature> experimentalFeature() override
std::optional<ExperimentalFeature> experimentalFeature() const override
{
return Xp::Flakes;
}
@ -228,7 +233,7 @@ struct GitArchiveInputScheme : InputScheme
struct GitHubInputScheme : GitArchiveInputScheme
{
std::string type() const override { return "github"; }
std::string_view schemeName() const override { return "github"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
@ -309,7 +314,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
struct GitLabInputScheme : GitArchiveInputScheme
{
std::string type() const override { return "gitlab"; }
std::string_view schemeName() const override { return "gitlab"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
@ -377,7 +382,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
struct SourceHutInputScheme : GitArchiveInputScheme
{
std::string type() const override { return "sourcehut"; }
std::string_view schemeName() const override { return "sourcehut"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{

24
src/libfetchers/indirect.cc
View file

@ -1,5 +1,6 @@
#include "fetchers.hh"
#include "url-parts.hh"
#include "path.hh"
namespace nix::fetchers {
@ -49,14 +50,23 @@ struct IndirectInputScheme : InputScheme
return input;
}
std::string_view schemeName() const override
{
return "indirect";
}
StringSet allowedAttrs() const override
{
return {
"id",
"ref",
"rev",
"narHash",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{
if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
throw Error("unsupported indirect input attribute '%s'", name);
auto id = getStrAttr(attrs, "id");
if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id);
@ -92,7 +102,7 @@ struct IndirectInputScheme : InputScheme
throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
}
std::optional<ExperimentalFeature> experimentalFeature() override
std::optional<ExperimentalFeature> experimentalFeature() const override
{
return Xp::Flakes;
}

4
src/libfetchers/input-accessor.hh
View file

@ -1,8 +1,10 @@
#pragma once
///@file
#include "source-accessor.hh"
#include "ref.hh"
#include "types.hh"
#include "file-system.hh"
#include "repair-flag.hh"
#include "content-address.hh"
@ -14,7 +16,7 @@ struct SourcePath;
class StorePath;
class Store;
struct InputAccessor : SourceAccessor, std::enable_shared_from_this<InputAccessor>
struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
{
/**
* Return the maximum last-modified time of the files in this

44
src/libfetchers/memory-input-accessor.cc
View file

@ -1,48 +1,16 @@
#include "memory-input-accessor.hh"
#include "memory-source-accessor.hh"
namespace nix {
struct MemoryInputAccessorImpl : MemoryInputAccessor
struct MemoryInputAccessorImpl : MemoryInputAccessor, MemorySourceAccessor
{
std::map<CanonPath, std::string> files;
std::string readFile(const CanonPath & path) override
{
auto i = files.find(path);
if (i == files.end())
throw Error("file '%s' does not exist", path);
return i->second;
}
bool pathExists(const CanonPath & path) override
{
auto i = files.find(path);
return i != files.end();
}
Stat lstat(const CanonPath & path) override
{
auto i = files.find(path);
if (i != files.end())
return Stat { .type = tRegular, .isExecutable = false };
throw Error("file '%s' does not exist", path);
}
DirEntries readDirectory(const CanonPath & path) override
{
return {};
}
std::string readLink(const CanonPath & path) override
{
throw UnimplementedError("MemoryInputAccessor::readLink");
}
SourcePath addFile(CanonPath path, std::string && contents) override
{
files.emplace(path, std::move(contents));
return {ref(shared_from_this()), std::move(path)};
return {
ref(shared_from_this()),
MemorySourceAccessor::addFile(path, std::move(contents))
};
}
};

46
src/libfetchers/mercurial.cc
View file

@ -1,4 +1,6 @@
#include "fetchers.hh"
#include "processes.hh"
#include "users.hh"
#include "cache.hh"
#include "globals.hh"
#include "tarfile.hh"
@ -69,14 +71,25 @@ struct MercurialInputScheme : InputScheme
return inputFromAttrs(attrs);
}
std::string_view schemeName() const override
{
return "hg";
}
StringSet allowedAttrs() const override
{
return {
"url",
"ref",
"rev",
"revCount",
"narHash",
"name",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
throw Error("unsupported Mercurial input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
@ -109,7 +122,7 @@ struct MercurialInputScheme : InputScheme
return res;
}
std::optional<Path> getSourcePath(const Input & input) override
std::optional<Path> getSourcePath(const Input & input) const override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
@ -117,18 +130,27 @@ struct MercurialInputScheme : InputScheme
return {};
}
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
void putFile(
const Input & input,
const CanonPath & path,
std::string_view contents,
std::optional<std::string> commitMsg) const override
{
auto sourcePath = getSourcePath(input);
assert(sourcePath);
auto [isLocal, repoPath] = getActualUrl(input);
if (!isLocal)
throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string());
auto absPath = CanonPath(repoPath) + path;
writeFile(absPath.abs(), contents);
// FIXME: shut up if file is already tracked.
runHg(
{ "add", *sourcePath + "/" + std::string(file) });
{ "add", absPath.abs() });
if (commitMsg)
runHg(
{ "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
{ "commit", absPath.abs(), "-m", *commitMsg });
}
std::pair<bool, std::string> getActualUrl(const Input & input) const

55
src/libfetchers/path.cc
View file

@ -32,23 +32,30 @@ struct PathInputScheme : InputScheme
return input;
}
std::string_view schemeName() const override
{
return "path";
}
StringSet allowedAttrs() const override
{
return {
"path",
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree work
the same as the repository from which is exported (e.g.
path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
*/
"rev",
"revCount",
"lastModified",
"narHash",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{
if (maybeGetStrAttr(attrs, "type") != "path") return {};
getStrAttr(attrs, "path");
for (auto & [name, value] : attrs)
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree
work the same as the repository from which is exported
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
// checked in Input::fromAttrs
;
else
throw Error("unsupported path input attribute '%s'", name);
Input input;
input.attrs = attrs;
return input;
@ -66,14 +73,28 @@ struct PathInputScheme : InputScheme
};
}
std::optional<Path> getSourcePath(const Input & input) override
std::optional<Path> getSourcePath(const Input & input) const override
{
return getStrAttr(input.attrs, "path");
}
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
void putFile(
const Input & input,
const CanonPath & path,
std::string_view contents,
std::optional<std::string> commitMsg) const override
{
// nothing to do
writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents);
}
CanonPath getAbsPath(const Input & input) const
{
auto path = getStrAttr(input.attrs, "path");
if (path[0] == '/')
return CanonPath(path);
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
@ -121,7 +142,7 @@ struct PathInputScheme : InputScheme
return {std::move(*storePath), input};
}
std::optional<ExperimentalFeature> experimentalFeature() override
std::optional<ExperimentalFeature> experimentalFeature() const override
{
return Xp::Flakes;
}

2
src/libfetchers/registry.cc
View file

@ -1,6 +1,6 @@
#include "registry.hh"
#include "tarball.hh"
#include "util.hh"
#include "users.hh"
#include "globals.hh"
#include "store-api.hh"
#include "local-fs-store.hh"

34
src/libfetchers/tarball.cc
View file

@ -184,7 +184,6 @@ DownloadTarballResult downloadTarball(
// An input scheme corresponding to a curl-downloadable resource.
struct CurlInputScheme : InputScheme
{
virtual const std::string inputType() const = 0;
const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
const bool hasTarballExtension(std::string_view path) const
@ -222,22 +221,27 @@ struct CurlInputScheme : InputScheme
url.query.erase("rev");
url.query.erase("revCount");
input.attrs.insert_or_assign("type", inputType());
input.attrs.insert_or_assign("type", std::string { schemeName() });
input.attrs.insert_or_assign("url", url.to_string());
return input;
}
StringSet allowedAttrs() const override
{
return {
"type",
"url",
"narHash",
"name",
"unpack",
"rev",
"revCount",
"lastModified",
};
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) const override
{
auto type = maybeGetStrAttr(attrs, "type");
if (type != inputType()) return {};
// FIXME: some of these only apply to TarballInputScheme.
std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"};
for (auto & [name, value] : attrs)
if (!allowedNames.count(name))
throw Error("unsupported %s input attribute '%s'", *type, name);
Input input;
input.attrs = attrs;
@ -258,14 +262,14 @@ struct CurlInputScheme : InputScheme
struct FileInputScheme : CurlInputScheme
{
const std::string inputType() const override { return "file"; }
std::string_view schemeName() const override { return "file"; }
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);
return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
&& (parsedUrlScheme.application
? parsedUrlScheme.application.value() == inputType()
? parsedUrlScheme.application.value() == schemeName()
: (!requireTree && !hasTarballExtension(url.path)));
}
@ -278,7 +282,7 @@ struct FileInputScheme : CurlInputScheme
struct TarballInputScheme : CurlInputScheme
{
const std::string inputType() const override { return "tarball"; }
std::string_view schemeName() const override { return "tarball"; }
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
@ -286,7 +290,7 @@ struct TarballInputScheme : CurlInputScheme
return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
&& (parsedUrlScheme.application
? parsedUrlScheme.application.value() == inputType()
? parsedUrlScheme.application.value() == schemeName()
: (requireTree || hasTarballExtension(url.path)));
}

2
src/libmain/common-args.cc
View file

@ -1,7 +1,9 @@
#include "common-args.hh"
#include "args/root.hh"
#include "globals.hh"
#include "logging.hh"
#include "loggers.hh"
#include "util.hh"
namespace nix {

2
src/libmain/loggers.cc
View file

@ -1,6 +1,6 @@
#include "loggers.hh"
#include "environment-variables.hh"
#include "progress-bar.hh"
#include "util.hh"
namespace nix {

2
src/libmain/progress-bar.cc
View file

@ -1,5 +1,5 @@
#include "progress-bar.hh"
#include "util.hh"
#include "terminal.hh"
#include "sync.hh"
#include "store-api.hh"
#include "names.hh"

3
src/libmain/shared.cc
View file

@ -1,10 +1,11 @@
#include "globals.hh"
#include "current-process.hh"
#include "shared.hh"
#include "store-api.hh"
#include "gc-store.hh"
#include "util.hh"
#include "loggers.hh"
#include "progress-bar.hh"
#include "signals.hh"
#include <algorithm>
#include <cctype>

2
src/libmain/shared.hh
View file

@ -1,7 +1,7 @@
#pragma once
///@file
#include "util.hh"
#include "processes.hh"
#include "args.hh"
#include "args/root.hh"
#include "common-args.hh"

29
src/libstore/binary-cache-store.cc
View file

@ -2,7 +2,7 @@
#include "binary-cache-store.hh"
#include "compression.hh"
#include "derivations.hh"
#include "fs-accessor.hh"
#include "source-accessor.hh"
#include "globals.hh"
#include "nar-info.hh"
#include "sync.hh"
@ -11,6 +11,7 @@
#include "nar-accessor.hh"
#include "thread-pool.hh"
#include "callback.hh"
#include "signals.hh"
#include <chrono>
#include <future>
@ -143,7 +144,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink { htSHA256 };
std::shared_ptr<FSAccessor> narAccessor;
std::shared_ptr<SourceAccessor> narAccessor;
HashSink narHashSink { htSHA256 };
{
FdSink fileSink(fdTemp.get());
@ -195,7 +196,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
if (writeNARListing) {
nlohmann::json j = {
{"version", 1},
{"root", listNar(ref<FSAccessor>(narAccessor), "", true)},
{"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
};
upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
@ -206,9 +207,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
specify the NAR file and member containing the debug info. */
if (writeDebugInfo) {
std::string buildIdDir = "/lib/debug/.build-id";
CanonPath buildIdDir("lib/debug/.build-id");
if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) {
ThreadPool threadPool(25);
@ -231,17 +232,17 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
std::regex regex1("^[0-9a-f]{2}$");
std::regex regex2("^[0-9a-f]{38}\\.debug$");
for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
auto dir = buildIdDir + "/" + s1;
for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) {
auto dir = buildIdDir + s1;
if (narAccessor->stat(dir).type != FSAccessor::tDirectory
if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory
|| !std::regex_match(s1, regex1))
continue;
for (auto & s2 : narAccessor->readDirectory(dir)) {
auto debugPath = dir + "/" + s2;
for (auto & [s2, _type] : narAccessor->readDirectory(dir)) {
auto debugPath = dir + s2;
if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular
|| !std::regex_match(s2, regex2))
continue;
@ -250,7 +251,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
std::string key = "debuginfo/" + buildId;
std::string target = "../" + narInfo->url;
threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target));
}
}
@ -503,9 +504,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
upsertFile(filePath, info.toJSON().dump(), "application/json");
}
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
{
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, localNarCache);
}
void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)

16
src/libstore/binary-cache-store.hh
View file

@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression",
const Setting<std::string> compression{this, "xz", "compression",
"NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing",
const Setting<bool> writeNARListing{this, false, "write-nar-listing",
"Whether to write a JSON file that lists the files in each NAR."};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info",
const Setting<bool> writeDebugInfo{this, false, "index-debug-info",
R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand
)"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key",
const Setting<Path> secretKeyFile{this, "", "secret-key",
"Path to the secret key used to sign the binary cache."};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache",
const Setting<Path> localNarCache{this, "", "local-nar-cache",
"Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
const Setting<bool> parallelCompression{this, false, "parallel-compression",
"Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
const Setting<int> compressionLevel{this, -1, "compression-level",
R"(
The *preset level* to be used when compressing NARs.
The meaning and accepted values depend on the compression method selected.
@ -148,7 +148,7 @@ public:
void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override;
ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;

37
src/libstore/build/child.cc Normal file
View file

@ -0,0 +1,37 @@
#include "child.hh"
#include "current-process.hh"
#include "logging.hh"
#include <fcntl.h>
#include <unistd.h>
namespace nix {
void commonChildInit()
{
logger = makeSimpleLogger();
const static std::string pathNullDevice = "/dev/null";
restoreProcessContext(false);
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning
that e.g. ssh cannot open /dev/tty) and it doesn't receive
terminal signals. */
if (setsid() == -1)
throw SysError("creating a new session");
/* Dup stderr to stdout. */
if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
throw SysError("cannot dup stderr into stdout");
/* Reroute stdin to /dev/null. */
int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
if (fdDevNull == -1)
throw SysError("cannot open '%1%'", pathNullDevice);
if (dup2(fdDevNull, STDIN_FILENO) == -1)
throw SysError("cannot dup null device into stdin");
close(fdDevNull);
}
}

11
src/libstore/build/child.hh Normal file
View file

@ -0,0 +1,11 @@
#pragma once
///@file
namespace nix {
/**
* Common initialisation performed in child processes.
*/
void commonChildInit();
}

24
src/libstore/build/derivation-goal.cc
View file

@ -1317,9 +1317,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
// ensure that logs from a builder using `ssh-ng://` as protocol
// are also available to `nix log`.
if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) {
auto f = (*json)["fields"];
(*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n");
if (s && !isWrittenToLog && logSink) {
const auto type = (*json)["type"];
const auto fields = (*json)["fields"];
if (type == resBuildLogLine) {
(*logSink)((fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n");
} else if (type == resSetPhase && ! fields.is_null()) {
const auto phase = fields[0];
if (! phase.is_null()) {
// nixpkgs' stdenv produces lines in the log to signal
// phase changes.
// We want to get the same lines in case of remote builds.
// The format is:
// @nix { "action": "setPhase", "phase": "$curPhase" }
const auto logLine = nlohmann::json::object({
{"action", "setPhase"},
{"phase", phase}
});
(*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n");
}
}
}
}
currentHookLine.clear();
@ -1474,6 +1491,7 @@ void DerivationGoal::done(
SingleDrvOutputs builtOutputs,
std::optional<Error> ex)
{
outputLocks.unlock();
buildResult.status = status;
if (ex)
buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));

2
src/libstore/build/hook-instance.cc
View file

@ -1,5 +1,7 @@
#include "globals.hh"
#include "hook-instance.hh"
#include "file-system.hh"
#include "child.hh"
namespace nix {

1
src/libstore/build/hook-instance.hh
View file

@ -3,6 +3,7 @@
#include "logging.hh"
#include "serialise.hh"
#include "processes.hh"
namespace nix {

12
src/libstore/build/local-derivation-goal.cc
View file

@ -15,7 +15,10 @@
#include "json-utils.hh"
#include "cgroup.hh"
#include "personality.hh"
#include "current-process.hh"
#include "namespaces.hh"
#include "child.hh"
#include "unix-domain-socket.hh"
#include <regex>
#include <queue>
@ -649,8 +652,8 @@ void LocalDerivationGoal::startBuilder()
#if __linux__
/* Create a temporary directory in which we set up the chroot
environment using bind-mounts. We put it in the Nix store
to ensure that we can create hard-links to non-directory
inputs in the fake Nix store in the chroot (see below). */
so that the build outputs can be moved efficiently from the
chroot to their final location. */
chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
deletePath(chrootRootDir);
@ -1563,10 +1566,11 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
Path source = worker.store.Store::toRealPath(path);
Path target = chrootRootDir + worker.store.printStorePath(path);
if (pathExists(target))
if (pathExists(target)) {
// There is a similar debug message in doBind, so only run it in this block to not have double messages.
debug("bind-mounting %s -> %s", target, source);
throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
}
/* Bind-mount the path into the sandbox. This requires
entering its mount namespace, which is not possible
@ -1619,6 +1623,8 @@ void setupSeccomp()
seccomp_release(ctx);
});
constexpr std::string_view nativeSystem = SYSTEM;
if (nativeSystem == "x86_64-linux" &&
seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
throw SysError("unable to add 32-bit seccomp architecture");

1
src/libstore/build/local-derivation-goal.hh
View file

@ -3,6 +3,7 @@
#include "derivation-goal.hh"
#include "local-store.hh"
#include "processes.hh"
namespace nix {

1
src/libstore/build/worker.cc
View file

@ -4,6 +4,7 @@
#include "drv-output-substitution-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
#include "signals.hh"
#include <poll.h>

1
src/libstore/common-protocol.cc
View file

@ -1,5 +1,4 @@
#include "serialise.hh"
#include "util.hh"
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "build-result.hh"

1
src/libstore/crypto.cc
View file

@ -1,4 +1,5 @@
#include "crypto.hh"
#include "file-system.hh"
#include "util.hh"
#include "globals.hh"

6
src/libstore/daemon.cc
View file

@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
eagerly consume the entire stream it's given, past the
length of the Nar. */
TeeSource savedNARSource(from, saved);
ParseSink sink; /* null sink; just parse the NAR */
NullParseSink sink; /* just parse the NAR */
parseDump(sink, savedNARSource);
} else {
/* Incrementally parse the NAR file, stripping the
metadata, and streaming the sole file we expect into
`saved`. */
RetrieveRegularNARSink savedRegular { saved };
RegularFileSink savedRegular { saved };
parseDump(savedRegular, from);
if (!savedRegular.regular) throw Error("regular file expected");
}
@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
source = std::make_unique<TunnelSource>(from, to);
else {
TeeSource tee { from, saved };
ParseSink ether;
NullParseSink ether;
parseDump(ether, tee);
source = std::make_unique<StringSource>(saved.s);
}

15
src/libstore/derivations.cc
View file

@ -6,7 +6,6 @@
#include "split.hh"
#include "common-protocol.hh"
#include "common-protocol-impl.hh"
#include "fs-accessor.hh"
#include <boost/container/small_vector.hpp>
#include <nlohmann/json.hpp>
@ -152,11 +151,10 @@ StorePath writeDerivation(Store & store,
/* Read string `s' from stream `str'. */
static void expect(std::istream & str, std::string_view s)
{
char s2[s.size()];
str.read(s2, s.size());
std::string_view s2View { s2, s.size() };
if (s2View != s)
throw FormatError("expected string '%s', got '%s'", s, s2View);
for (auto & c : s) {
if (str.get() != c)
throw FormatError("expected string '%1%'", s);
}
}
@ -353,7 +351,7 @@ Derivation parseDerivation(
expect(str, "erive(");
version = DerivationATermVersion::Traditional;
break;
case 'r':
case 'r': {
expect(str, "rvWithVersion(");
auto versionS = parseString(str);
if (versionS == "xp-dyn-drv") {
@ -366,6 +364,9 @@ Derivation parseDerivation(
expect(str, ",");
break;
}
default:
throw Error("derivation does not start with 'Derive' or 'DrvWithVersion'");
}
/* Parse the list of outputs. */
expect(str, "[");

1
src/libstore/derived-path-map.cc
View file

@ -1,4 +1,5 @@
#include "derived-path-map.hh"
#include "util.hh"
namespace nix {

1
src/libstore/derived-path-map.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "types.hh"
#include "derived-path.hh"

2
src/libstore/derived-path.hh
View file

@ -1,10 +1,10 @@
#pragma once
///@file
#include "util.hh"
#include "path.hh"
#include "outputs-spec.hh"
#include "comparator.hh"
#include "config.hh"
#include <variant>

2
src/libstore/dummy-store.cc
View file

@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
{ callback(nullptr); }
virtual ref<FSAccessor> getFSAccessor() override
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
{ unsupported("getFSAccessor"); }
};

2
src/libstore/export-import.cc
View file

@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
/* Extract the NAR from the source. */
StringSink saved;
TeeSource tee { source, saved };
ParseSink ether;
NullParseSink ether;
parseDump(ether, tee);
uint32_t magic = readInt(source);

3
src/libstore/filetransfer.cc
View file

@ -1,11 +1,12 @@
#include "filetransfer.hh"
#include "util.hh"
#include "namespaces.hh"
#include "globals.hh"
#include "store-api.hh"
#include "s3.hh"
#include "compression.hh"
#include "finally.hh"
#include "callback.hh"
#include "signals.hh"
#if ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>

52
src/libstore/fs-accessor.hh
View file

@ -1,52 +0,0 @@
#pragma once
///@file
#include "types.hh"
namespace nix {
/**
* An abstract class for accessing a filesystem-like structure, such
* as a (possibly remote) Nix store or the contents of a NAR file.
*/
class FSAccessor
{
public:
enum Type { tMissing, tRegular, tSymlink, tDirectory };
struct Stat
{
Type type = tMissing;
/**
* regular files only
*/
uint64_t fileSize = 0;
/**
* regular files only
*/
bool isExecutable = false; // regular files only
/**
* regular files only
*/
uint64_t narOffset = 0; // regular files only
};
virtual ~FSAccessor() { }
virtual Stat stat(const Path & path) = 0;
virtual StringSet readDirectory(const Path & path) = 0;
/**
* Read a file inside the store.
*
* If `requireValidPath` is set to `true` (the default), the path must be
* inside a valid store path, otherwise it just needs to be physically
* present (but not necessarily properly registered)
*/
virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0;
virtual std::string readLink(const Path & path) = 0;
};
}

16
src/libstore/gc.cc
View file

@ -2,6 +2,13 @@
#include "globals.hh"
#include "local-store.hh"
#include "finally.hh"
#include "unix-domain-socket.hh"
#include "signals.hh"
#if !defined(__linux__)
// For shelling out to lsof
# include "processes.hh"
#endif
#include <functional>
#include <queue>
@ -323,9 +330,7 @@ typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots
static void readProcLink(const std::string & file, UncheckedRoots & roots)
{
/* 64 is the starting buffer size gnu readlink uses... */
auto bufsiz = ssize_t{64};
try_again:
constexpr auto bufsiz = PATH_MAX;
char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) {
@ -334,10 +339,7 @@ try_again:
throw SysError("reading symlink");
}
if (res == bufsiz) {
if (SSIZE_MAX / 2 < bufsiz)
throw Error("stupidly long symlink");
bufsiz *= 2;
goto try_again;
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
}
if (res > 0 && buf[0] == '/')
roots[std::string(static_cast<char *>(buf), res)]

13
src/libstore/globals.cc
View file

@ -1,7 +1,8 @@
#include "globals.hh"
#include "util.hh"
#include "current-process.hh"
#include "archive.hh"
#include "args.hh"
#include "users.hh"
#include "abstract-setting-to-json.hh"
#include "compute-levels.hh"
@ -17,9 +18,13 @@
#include <sodium/core.h>
#ifdef __GLIBC__
#include <gnu/lib-names.h>
#include <nss.h>
#include <dlfcn.h>
# include <gnu/lib-names.h>
# include <nss.h>
# include <dlfcn.h>
#endif
#if __APPLE__
# include "processes.hh"
#endif
#include "config-impl.hh"

12
src/libstore/globals.hh
View file

@ -3,7 +3,7 @@
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include "environment-variables.hh"
#include "experimental-features.hh"
#include <map>
@ -1084,6 +1084,16 @@ public:
true, // document default
Xp::ConfigurableImpureEnv
};
Setting<std::string> upgradeNixStorePathUrl{
this,
"https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix",
"upgrade-nix-store-path-url",
R"(
Used by `nix upgrade-nix`, the URL of the file that contains the
store paths of the latest Nix release.
)"
};
};

8
src/libstore/legacy-ssh-store.cc
View file

@ -17,10 +17,10 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
{
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program",
const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent SSH connections."};
const std::string name() override { return "SSH Store"; }
@ -38,7 +38,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
// Hack for getting remote build log output.
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation
const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection
{
@ -363,7 +363,7 @@ public:
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
virtual ref<FSAccessor> getFSAccessor() override
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
{ unsupported("getFSAccessor"); }
/**

64
src/libstore/local-fs-store.cc
View file

@ -1,5 +1,5 @@
#include "archive.hh"
#include "fs-accessor.hh"
#include "posix-source-accessor.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "globals.hh"
@ -13,69 +13,53 @@ LocalFSStore::LocalFSStore(const Params & params)
{
}
struct LocalStoreAccessor : public FSAccessor
struct LocalStoreAccessor : PosixSourceAccessor
{
ref<LocalFSStore> store;
bool requireValidPath;
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { }
LocalStoreAccessor(ref<LocalFSStore> store, bool requireValidPath)
: store(store)
, requireValidPath(requireValidPath)
{ }
Path toRealPath(const Path & path, bool requireValidPath = true)
CanonPath toRealPath(const CanonPath & path)
{
auto storePath = store->toStorePath(path).first;
auto [storePath, rest] = store->toStorePath(path.abs());
if (requireValidPath && !store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest);
}
FSAccessor::Stat stat(const Path & path) override
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
auto realPath = toRealPath(path);
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
throw SysError("getting status of '%1%'", path);
}
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
throw Error("file '%1%' has unsupported type", path);
return {
S_ISREG(st.st_mode) ? Type::tRegular :
S_ISLNK(st.st_mode) ? Type::tSymlink :
Type::tDirectory,
S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
return PosixSourceAccessor::maybeLstat(toRealPath(path));
}
StringSet readDirectory(const Path & path) override
DirEntries readDirectory(const CanonPath & path) override
{
auto realPath = toRealPath(path);
auto entries = nix::readDirectory(realPath);
StringSet res;
for (auto & entry : entries)
res.insert(entry.name);
return res;
return PosixSourceAccessor::readDirectory(toRealPath(path));
}
std::string readFile(const Path & path, bool requireValidPath = true) override
void readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback) override
{
return nix::readFile(toRealPath(path, requireValidPath));
return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback);
}
std::string readLink(const Path & path) override
std::string readLink(const CanonPath & path) override
{
return nix::readLink(toRealPath(path));
return PosixSourceAccessor::readLink(toRealPath(path));
}
};
ref<FSAccessor> LocalFSStore::getFSAccessor()
ref<SourceAccessor> LocalFSStore::getFSAccessor(bool requireValidPath)
{
return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())),
requireValidPath);
}
void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)

14
src/libstore/local-fs-store.hh
View file

@ -11,25 +11,21 @@ struct LocalFSStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
// it to omit the call to the Setting constructor. Clang works fine
// either way.
const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt,
const OptionalPathSetting rootDir{this, std::nullopt,
"root",
"Directory prefixed to all other paths."};
const PathSetting stateDir{(StoreConfig*) this,
const PathSetting stateDir{this,
rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir,
"state",
"Directory where Nix will store state."};
const PathSetting logDir{(StoreConfig*) this,
const PathSetting logDir{this,
rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir,
"log",
"directory where Nix will store log files."};
const PathSetting realStoreDir{(StoreConfig*) this,
const PathSetting realStoreDir{this,
rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real",
"Physical path of the Nix store."};
};
@ -47,7 +43,7 @@ public:
LocalFSStore(const Params & params);
void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override;
ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
/**
* Creates symlink from the `gcRoot` to the `storePath` and

3
src/libstore/local-store.cc
View file

@ -10,6 +10,7 @@
#include "topo-sort.hh"
#include "finally.hh"
#include "compression.hh"
#include "signals.hh"
#include <iostream>
#include <algorithm>
@ -1200,7 +1201,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
bool narRead = false;
Finally cleanup = [&]() {
if (!narRead) {
ParseSink sink;
NullParseSink sink;
parseDump(sink, source);
}
};

5
src/libstore/local-store.hh
View file

@ -7,7 +7,6 @@
#include "store-api.hh"
#include "indirect-root-store.hh"
#include "sync.hh"
#include "util.hh"
#include <chrono>
#include <future>
@ -40,12 +39,12 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
{
using LocalFSStoreConfig::LocalFSStoreConfig;
Setting<bool> requireSigs{(StoreConfig*) this,
Setting<bool> requireSigs{this,
settings.requireSigs,
"require-sigs",
"Whether store paths copied into this store should have a trusted signature."};
Setting<bool> readOnly{(StoreConfig*) this,
Setting<bool> readOnly{this,
false,
"read-only",
R"(

1
src/libstore/lock.cc
View file

@ -1,4 +1,5 @@
#include "lock.hh"
#include "file-system.hh"
#include "globals.hh"
#include "pathlocks.hh"

1
src/libstore/machines.cc
View file

@ -1,5 +1,4 @@
#include "machines.hh"
#include "util.hh"
#include "globals.hh"
#include "store-api.hh"

143
src/libstore/nar-accessor.cc
View file

@ -11,13 +11,7 @@ namespace nix {
struct NarMember
{
FSAccessor::Type type = FSAccessor::Type::tMissing;
bool isExecutable = false;
/* If this is a regular file, position of the contents of this
file in the NAR. */
uint64_t start = 0, size = 0;
SourceAccessor::Stat stat;
std::string target;
@ -25,7 +19,7 @@ struct NarMember
std::map<std::string, NarMember> children;
};
struct NarAccessor : public FSAccessor
struct NarAccessor : public SourceAccessor
{
std::optional<const std::string> nar;
@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor
acc.root = std::move(member);
parents.push(&acc.root);
} else {
if (parents.top()->type != FSAccessor::Type::tDirectory)
if (parents.top()->stat.type != Type::tDirectory)
throw Error("NAR file missing parent directory of path '%s'", path);
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
parents.push(&result.first->second);
@ -66,12 +60,22 @@ struct NarAccessor : public FSAccessor
void createDirectory(const Path & path) override
{
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
createMember(path, NarMember{ .stat = {
.type = Type::tDirectory,
.fileSize = 0,
.isExecutable = false,
.narOffset = 0
} });
}
void createRegularFile(const Path & path) override
{
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
createMember(path, NarMember{ .stat = {
.type = Type::tRegular,
.fileSize = 0,
.isExecutable = false,
.narOffset = 0
} });
}
void closeRegularFile() override
@ -79,14 +83,14 @@ struct NarAccessor : public FSAccessor
void isExecutable() override
{
parents.top()->isExecutable = true;
parents.top()->stat.isExecutable = true;
}
void preallocateContents(uint64_t size) override
{
assert(size <= std::numeric_limits<uint64_t>::max());
parents.top()->size = (uint64_t) size;
parents.top()->start = pos;
auto & st = parents.top()->stat;
st.fileSize = size;
st.narOffset = pos;
}
void receiveContents(std::string_view data) override
@ -95,7 +99,9 @@ struct NarAccessor : public FSAccessor
void createSymlink(const Path & path, const std::string & target) override
{
createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
NarMember{
.stat = {.type = Type::tSymlink},
.target = target});
}
size_t read(char * data, size_t len) override
@ -130,18 +136,19 @@ struct NarAccessor : public FSAccessor
std::string type = v["type"];
if (type == "directory") {
member.type = FSAccessor::Type::tDirectory;
for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
std::string name = i.key();
recurse(member.children[name], i.value());
member.stat = {.type = Type::tDirectory};
for (const auto &[name, function] : v["entries"].items()) {
recurse(member.children[name], function);
}
} else if (type == "regular") {
member.type = FSAccessor::Type::tRegular;
member.size = v["size"];
member.isExecutable = v.value("executable", false);
member.start = v["narOffset"];
member.stat = {
.type = Type::tRegular,
.fileSize = v["size"],
.isExecutable = v.value("executable", false),
.narOffset = v["narOffset"]
};
} else if (type == "symlink") {
member.type = FSAccessor::Type::tSymlink;
member.stat = {.type = Type::tSymlink};
member.target = v.value("target", "");
} else return;
};
@ -150,134 +157,122 @@ struct NarAccessor : public FSAccessor
recurse(root, v);
}
NarMember * find(const Path & path)
NarMember * find(const CanonPath & path)
{
Path canon = path == "" ? "" : canonPath(path);
NarMember * current = &root;
auto end = path.end();
for (auto it = path.begin(); it != end; ) {
// because it != end, the remaining component is non-empty so we need
// a directory
if (current->type != FSAccessor::Type::tDirectory) return nullptr;
// skip slash (canonPath above ensures that this is always a slash)
assert(*it == '/');
it += 1;
// lookup current component
auto next = std::find(it, end, '/');
auto child = current->children.find(std::string(it, next));
for (const auto & i : path) {
if (current->stat.type != Type::tDirectory) return nullptr;
auto child = current->children.find(std::string(i));
if (child == current->children.end()) return nullptr;
current = &child->second;
it = next;
}
return current;
}
NarMember & get(const Path & path) {
NarMember & get(const CanonPath & path) {
auto result = find(path);
if (result == nullptr)
if (!result)
throw Error("NAR file does not contain path '%1%'", path);
return *result;
}
Stat stat(const Path & path) override
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
auto i = find(path);
if (i == nullptr)
return {FSAccessor::Type::tMissing, 0, false};
return {i->type, i->size, i->isExecutable, i->start};
if (!i)
return std::nullopt;
return i->stat;
}
StringSet readDirectory(const Path & path) override
DirEntries readDirectory(const CanonPath & path) override
{
auto i = get(path);
if (i.type != FSAccessor::Type::tDirectory)
if (i.stat.type != Type::tDirectory)
throw Error("path '%1%' inside NAR file is not a directory", path);
StringSet res;
for (auto & child : i.children)
res.insert(child.first);
DirEntries res;
for (const auto & child : i.children)
res.insert_or_assign(child.first, std::nullopt);
return res;
}
std::string readFile(const Path & path, bool requireValidPath = true) override
std::string readFile(const CanonPath & path) override
{
auto i = get(path);
if (i.type != FSAccessor::Type::tRegular)
if (i.stat.type != Type::tRegular)
throw Error("path '%1%' inside NAR file is not a regular file", path);
if (getNarBytes) return getNarBytes(i.start, i.size);
if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize);
assert(nar);
return std::string(*nar, i.start, i.size);
return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize);
}
std::string readLink(const Path & path) override
std::string readLink(const CanonPath & path) override
{
auto i = get(path);
if (i.type != FSAccessor::Type::tSymlink)
if (i.stat.type != Type::tSymlink)
throw Error("path '%1%' inside NAR file is not a symlink", path);
return i.target;
}
};
ref<FSAccessor> makeNarAccessor(std::string && nar)
ref<SourceAccessor> makeNarAccessor(std::string && nar)
{
return make_ref<NarAccessor>(std::move(nar));
}
ref<FSAccessor> makeNarAccessor(Source & source)
ref<SourceAccessor> makeNarAccessor(Source & source)
{
return make_ref<NarAccessor>(source);
}
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
ref<SourceAccessor> makeLazyNarAccessor(const std::string & listing,
GetNarBytes getNarBytes)
{
return make_ref<NarAccessor>(listing, getNarBytes);
}
using nlohmann::json;
json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
{
auto st = accessor->stat(path);
auto st = accessor->lstat(path);
json obj = json::object();
switch (st.type) {
case FSAccessor::Type::tRegular:
case SourceAccessor::Type::tRegular:
obj["type"] = "regular";
obj["size"] = st.fileSize;
if (st.fileSize)
obj["size"] = *st.fileSize;
if (st.isExecutable)
obj["executable"] = true;
if (st.narOffset)
obj["narOffset"] = st.narOffset;
if (st.narOffset && *st.narOffset)
obj["narOffset"] = *st.narOffset;
break;
case FSAccessor::Type::tDirectory:
case SourceAccessor::Type::tDirectory:
obj["type"] = "directory";
{
obj["entries"] = json::object();
json &res2 = obj["entries"];
for (auto & name : accessor->readDirectory(path)) {
for (const auto & [name, type] : accessor->readDirectory(path)) {
if (recurse) {
res2[name] = listNar(accessor, path + "/" + name, true);
res2[name] = listNar(accessor, path + name, true);
} else
res2[name] = json::object();
}
}
break;
case FSAccessor::Type::tSymlink:
case SourceAccessor::Type::tSymlink:
obj["type"] = "symlink";
obj["target"] = accessor->readLink(path);
break;
case FSAccessor::Type::tMissing:
default:
throw Error("path '%s' does not exist in NAR", path);
case SourceAccessor::Type::tMisc:
assert(false); // cannot happen for NARs
}
return obj;
}

13
src/libstore/nar-accessor.hh
View file

@ -1,10 +1,11 @@
#pragma once
///@file
#include "source-accessor.hh"
#include <functional>
#include <nlohmann/json_fwd.hpp>
#include "fs-accessor.hh"
namespace nix {
@ -14,9 +15,9 @@ struct Source;
* Return an object that provides access to the contents of a NAR
* file.
*/
ref<FSAccessor> makeNarAccessor(std::string && nar);
ref<SourceAccessor> makeNarAccessor(std::string && nar);
ref<FSAccessor> makeNarAccessor(Source & source);
ref<SourceAccessor> makeNarAccessor(Source & source);
/**
* Create a NAR accessor from a NAR listing (in the format produced by
@ -24,9 +25,9 @@ ref<FSAccessor> makeNarAccessor(Source & source);
* readFile() method of the accessor to get the contents of files
* inside the NAR.
*/
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
ref<FSAccessor> makeLazyNarAccessor(
ref<SourceAccessor> makeLazyNarAccessor(
const std::string & listing,
GetNarBytes getNarBytes);
@ -34,6 +35,6 @@ ref<FSAccessor> makeLazyNarAccessor(
* Write a JSON representation of the contents of a NAR (except file
* contents).
*/
nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse);
nlohmann::json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse);
}

1
src/libstore/nar-info-disk-cache.cc
View file

@ -1,4 +1,5 @@
#include "nar-info-disk-cache.hh"
#include "users.hh"
#include "sync.hh"
#include "sqlite.hh"
#include "globals.hh"

68
src/libstore/nar-info.cc
View file

@ -4,6 +4,15 @@
namespace nix {
GENERATE_CMP_EXT(
,
NarInfo,
me->url,
me->compression,
me->fileHash,
me->fileSize,
static_cast<const ValidPathInfo &>(*me));
NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
: ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack
{
@ -29,12 +38,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
while (pos < s.size()) {
size_t colon = s.find(':', pos);
if (colon == std::string::npos) throw corrupt("expecting ':'");
if (colon == s.npos) throw corrupt("expecting ':'");
std::string name(s, pos, colon - pos);
size_t eol = s.find('\n', colon + 2);
if (eol == std::string::npos) throw corrupt("expecting '\\n'");
if (eol == s.npos) throw corrupt("expecting '\\n'");
std::string value(s, colon + 2, eol - colon - 2);
@ -125,4 +134,59 @@ std::string NarInfo::to_string(const Store & store) const
return res;
}
nlohmann::json NarInfo::toJSON(
const Store & store,
bool includeImpureInfo,
HashFormat hashFormat) const
{
using nlohmann::json;
auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat);
if (includeImpureInfo) {
if (!url.empty())
jsonObject["url"] = url;
if (!compression.empty())
jsonObject["compression"] = compression;
if (fileHash)
jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true);
if (fileSize)
jsonObject["downloadSize"] = fileSize;
}
return jsonObject;
}
NarInfo NarInfo::fromJSON(
const Store & store,
const StorePath & path,
const nlohmann::json & json)
{
using nlohmann::detail::value_t;
NarInfo res {
ValidPathInfo {
path,
UnkeyedValidPathInfo::fromJSON(store, json),
}
};
if (json.contains("url"))
res.url = ensureType(valueAt(json, "url"), value_t::string);
if (json.contains("compression"))
res.compression = ensureType(valueAt(json, "compression"), value_t::string);
if (json.contains("downloadHash"))
res.fileHash = Hash::parseAny(
static_cast<const std::string &>(
ensureType(valueAt(json, "downloadHash"), value_t::string)),
std::nullopt);
if (json.contains("downloadSize"))
res.fileSize = ensureType(valueAt(json, "downloadSize"), value_t::number_integer);
return res;
}
}

15
src/libstore/nar-info.hh
View file

@ -17,14 +17,25 @@ struct NarInfo : ValidPathInfo
uint64_t fileSize = 0;
NarInfo() = delete;
NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash)
NarInfo(const Store & store, std::string name, ContentAddressWithReferences ca, Hash narHash)
: ValidPathInfo(store, std::move(name), std::move(ca), narHash)
{ }
NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
NarInfo(StorePath path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
NarInfo(const Store & store, const std::string & s, const std::string & whence);
DECLARE_CMP(NarInfo);
std::string to_string(const Store & store) const;
nlohmann::json toJSON(
const Store & store,
bool includeImpureInfo,
HashFormat hashFormat) const override;
static NarInfo fromJSON(
const Store & store,
const StorePath & path,
const nlohmann::json & json);
};
}

2
src/libstore/optimise-store.cc
View file

@ -1,6 +1,6 @@
#include "util.hh"
#include "local-store.hh"
#include "globals.hh"
#include "signals.hh"
#include <cstdlib>
#include <cstring>

39
src/libstore/parsed-derivations.cc
View file

@ -132,6 +132,41 @@ bool ParsedDerivation::useUidRange() const
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
/**
* Write a JSON representation of store object metadata, such as the
* hash and the references.
*/
static nlohmann::json pathInfoToJSON(
Store & store,
const StorePathSet & storePaths)
{
nlohmann::json::array_t jsonList = nlohmann::json::array();
for (auto & storePath : storePaths) {
auto info = store.queryPathInfo(storePath);
auto & jsonPath = jsonList.emplace_back(
info->toJSON(store, false, HashFormat::Base32));
// Add the path to the object whose metadata we are including.
jsonPath["path"] = store.printStorePath(storePath);
jsonPath["valid"] = true;
jsonPath["closureSize"] = ({
uint64_t totalNarSize = 0;
StorePathSet closure;
store.computeFSClosure(info->path, closure, false, false);
for (auto & p : closure) {
auto info = store.queryPathInfo(p);
totalNarSize += info->narSize;
}
totalNarSize;
});
}
return jsonList;
}
std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths)
{
auto structuredAttrs = getStructuredAttrs();
@ -152,8 +187,8 @@ std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & s
StorePathSet storePaths;
for (auto & p : *i)
storePaths.insert(store.parseStorePath(p.get<std::string>()));
json[i.key()] = store.pathInfoToJSON(
store.exportReferences(storePaths, inputPaths), false, true);
json[i.key()] = pathInfoToJSON(store,
store.exportReferences(storePaths, inputPaths));
}
}

93
src/libstore/path-info.cc
View file

@ -1,5 +1,8 @@
#include <nlohmann/json.hpp>
#include "path-info.hh"
#include "store-api.hh"
#include "json-utils.hh"
namespace nix {
@ -144,4 +147,94 @@ ValidPathInfo::ValidPathInfo(
}, std::move(ca).raw);
}
nlohmann::json UnkeyedValidPathInfo::toJSON(
const Store & store,
bool includeImpureInfo,
HashFormat hashFormat) const
{
using nlohmann::json;
auto jsonObject = json::object();
jsonObject["narHash"] = narHash.to_string(hashFormat, true);
jsonObject["narSize"] = narSize;
{
auto& jsonRefs = (jsonObject["references"] = json::array());
for (auto & ref : references)
jsonRefs.emplace_back(store.printStorePath(ref));
}
if (ca)
jsonObject["ca"] = renderContentAddress(ca);
if (includeImpureInfo) {
if (deriver)
jsonObject["deriver"] = store.printStorePath(*deriver);
if (registrationTime)
jsonObject["registrationTime"] = registrationTime;
if (ultimate)
jsonObject["ultimate"] = ultimate;
if (!sigs.empty()) {
for (auto & sig : sigs)
jsonObject["signatures"].push_back(sig);
}
}
return jsonObject;
}
UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(
const Store & store,
const nlohmann::json & json)
{
using nlohmann::detail::value_t;
UnkeyedValidPathInfo res {
Hash(Hash::dummy),
};
ensureType(json, value_t::object);
res.narHash = Hash::parseAny(
static_cast<const std::string &>(
ensureType(valueAt(json, "narHash"), value_t::string)),
std::nullopt);
res.narSize = ensureType(valueAt(json, "narSize"), value_t::number_integer);
try {
auto & references = ensureType(valueAt(json, "references"), value_t::array);
for (auto & input : references)
res.references.insert(store.parseStorePath(static_cast<const std::string &>
(input)));
} catch (Error & e) {
e.addTrace({}, "while reading key 'references'");
throw;
}
if (json.contains("ca"))
res.ca = ContentAddress::parse(
static_cast<const std::string &>(
ensureType(valueAt(json, "ca"), value_t::string)));
if (json.contains("deriver"))
res.deriver = store.parseStorePath(
static_cast<const std::string &>(
ensureType(valueAt(json, "deriver"), value_t::string)));
if (json.contains("registrationTime"))
res.registrationTime = ensureType(valueAt(json, "registrationTime"), value_t::number_integer);
if (json.contains("ultimate"))
res.ultimate = ensureType(valueAt(json, "ultimate"), value_t::boolean);
if (json.contains("signatures"))
res.sigs = valueAt(json, "signatures");
return res;
}
}

12
src/libstore/path-info.hh
View file

@ -78,6 +78,18 @@ struct UnkeyedValidPathInfo
DECLARE_CMP(UnkeyedValidPathInfo);
virtual ~UnkeyedValidPathInfo() { }
/**
* @param includeImpureInfo If true, variable elements such as the
* registration time are included.
*/
virtual nlohmann::json toJSON(
const Store & store,
bool includeImpureInfo,
HashFormat hashFormat) const;
static UnkeyedValidPathInfo fromJSON(
const Store & store,
const nlohmann::json & json);
};
struct ValidPathInfo : UnkeyedValidPathInfo {

1
src/libstore/path-references.cc
View file

@ -1,6 +1,5 @@
#include "path-references.hh"
#include "hash.hh"
#include "util.hh"
#include "archive.hh"
#include <map>

1
src/libstore/path-references.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "references.hh"
#include "path.hh"

1
src/libstore/pathlocks.cc
View file

@ -1,6 +1,7 @@
#include "pathlocks.hh"
#include "util.hh"
#include "sync.hh"
#include "signals.hh"
#include <cerrno>
#include <cstdlib>

2
src/libstore/pathlocks.hh
View file

@ -1,7 +1,7 @@
#pragma once
///@file
#include "util.hh"
#include "file-descriptor.hh"
namespace nix {

2
src/libstore/profiles.cc
View file

@ -1,7 +1,7 @@
#include "profiles.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "util.hh"
#include "users.hh"
#include <sys/types.h>
#include <sys/stat.h>

Some files were not shown because too many files have changed in this diff Show more