wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-29 10:31:15 +02:00
Code Activity

Merge branch 'master' into overlayfs-store

Browse source
This commit is contained in:
John Ericson 2024-02-29 09:54:21 -05:00
commit cb4f85f11c
253 changed files with 5623 additions and 2830 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/build-remote/build-remote.cc
View file

@ -202,7 +202,7 @@ static int main_build_remote(int argc, char * * argv)
else
drvstr = "<unknown>";
auto error = hintformat(errorText);
auto error = HintFmt(errorText);
error
% drvstr
% neededSystem

12
src/libcmd/common-eval-args.cc
View file

@ -9,6 +9,7 @@
#include "store-api.hh"
#include "command.hh"
#include "tarball.hh"
#include "fetch-to-store.hh"
namespace nix {
@ -156,7 +157,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
for (auto & i : autoArgs) {
auto v = state.allocValue();
if (i.second[0] == 'E')
state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(CanonPath::fromCwd())));
state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(".")));
else
v->mkString(((std::string_view) i.second).substr(1));
res.insert(state.symbols.create(i.first), v);
@ -164,11 +165,12 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
return res.finish();
}
SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir)
SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
{
if (EvalSettings::isPseudoUrl(s)) {
auto storePath = fetchers::downloadTarball(
state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath;
auto accessor = fetchers::downloadTarball(
EvalSettings::resolvePseudoUrl(s)).accessor;
auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
}
@ -185,7 +187,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDi
}
else
return state.rootPath(CanonPath(s, baseDir));
return state.rootPath(baseDir ? absPath(s, *baseDir) : absPath(s));
}
}

2
src/libcmd/common-eval-args.hh
View file

@ -29,6 +29,6 @@ private:
std::map<std::string, std::string> autoArgs;
};
SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd());
SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr);
}

2
src/libcmd/editor-for.cc
View file

@ -17,7 +17,7 @@ Strings editorFor(const SourcePath & file, uint32_t line)
editor.find("vim") != std::string::npos ||
editor.find("kak") != std::string::npos))
args.push_back(fmt("+%d", line));
args.push_back(path->abs());
args.push_back(path->string());
return args;
}

2
src/libcmd/installable-value.cc
View file

@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
{
if (v.type() == nPath) {
auto storePath = fetchToStore(*state->store, v.path());
auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
return {{
.path = DerivedPath::Opaque {
.path = std::move(storePath),

5
src/libcmd/installables.cc
View file

@ -487,10 +487,11 @@ Installables SourceExprCommand::parseInstallables(
state->eval(e, *vFile);
}
else if (file) {
state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile);
auto dir = absPath(getCommandBaseDir());
state->evalFile(lookupFileArg(*state, *file, &dir), *vFile);
}
else {
CanonPath dir(CanonPath::fromCwd(getCommandBaseDir()));
Path dir = absPath(getCommandBaseDir());
auto e = state->parseExprFromString(*expr, state->rootPath(dir));
state->eval(e, *vFile);
}

121
src/libcmd/misc-store-flags.cc Normal file
View file

@ -0,0 +1,121 @@
#include "misc-store-flags.hh"
namespace nix::flag
{
static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
{
for (auto & format : hashFormats) {
if (hasPrefix(format, prefix)) {
completions.add(format);
}
}
}
Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf)
{
assert(*hf == nix::HashFormat::SRI);
return Args::Flag {
.longName = std::move(longName),
.description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.",
.labels = {"hash-format"},
.handler = {[hf](std::string s) {
*hf = parseHashFormat(s);
}},
.completer = hashFormatCompleter,
};
}
Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf)
{
return Args::Flag {
.longName = std::move(longName),
.description = "Hash format (`base16`, `nix32`, `base64`, `sri`).",
.labels = {"hash-format"},
.handler = {[ohf](std::string s) {
*ohf = std::optional<HashFormat>{parseHashFormat(s)};
}},
.completer = hashFormatCompleter,
};
}
static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
{
for (auto & algo : hashAlgorithms)
if (hasPrefix(algo, prefix))
completions.add(algo);
}
Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha)
{
return Args::Flag {
.longName = std::move(longName),
.description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
.labels = {"hash-algo"},
.handler = {[ha](std::string s) {
*ha = parseHashAlgo(s);
}},
.completer = hashAlgoCompleter,
};
}
Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha)
{
return Args::Flag {
.longName = std::move(longName),
.description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
.labels = {"hash-algo"},
.handler = {[oha](std::string s) {
*oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
}},
.completer = hashAlgoCompleter,
};
}
Args::Flag fileIngestionMethod(FileIngestionMethod * method)
{
return Args::Flag {
.longName = "mode",
// FIXME indentation carefully made for context, this is messed up.
.description = R"(
How to compute the hash of the input.
One of:
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
)",
.labels = {"file-ingestion-method"},
.handler = {[method](std::string s) {
*method = parseFileIngestionMethod(s);
}},
};
}
Args::Flag contentAddressMethod(ContentAddressMethod * method)
{
return Args::Flag {
.longName = "mode",
// FIXME indentation carefully made for context, this is messed up.
.description = R"(
How to compute the content-address of the store object.
One of:
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
- `text`: Like `flat`, but used for
[derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
[`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
For advanced use-cases only;
for regular usage prefer `nar` and `flat.
)",
.labels = {"content-address-method"},
.handler = {[method](std::string s) {
*method = ContentAddressMethod::parse(s);
}},
};
}
}

21
src/libcmd/misc-store-flags.hh Normal file
View file

@ -0,0 +1,21 @@
#include "args.hh"
#include "content-address.hh"
namespace nix::flag {
Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha);
static inline Args::Flag hashAlgo(HashAlgorithm * ha)
{
return hashAlgo("hash-algo", ha);
}
Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha);
Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf);
Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf);
static inline Args::Flag hashAlgoOpt(std::optional<HashAlgorithm> * oha)
{
return hashAlgoOpt("hash-algo", oha);
}
Args::Flag fileIngestionMethod(FileIngestionMethod * method);
Args::Flag contentAddressMethod(ContentAddressMethod * method);
}

86
src/libcmd/repl.cc
View file

@ -52,6 +52,27 @@ extern "C" {
namespace nix {
/**
* Returned by `NixRepl::processLine`.
*/
enum class ProcessLineResult {
/**
* The user exited with `:quit`. The REPL should exit. The surrounding
* program or evaluation (e.g., if the REPL was acting as the debugger)
* should also exit.
*/
Quit,
/**
* The user exited with `:continue`. The REPL should exit, but the program
* should continue running.
*/
Continue,
/**
* The user did not exit. The REPL should request another line of input.
*/
PromptAgain,
};
struct NixRepl
: AbstractNixRepl
#if HAVE_BOEHMGC
@ -75,13 +96,13 @@ struct NixRepl
std::function<AnnotatedValues()> getValues);
virtual ~NixRepl();
void mainLoop() override;
ReplExitStatus mainLoop() override;
void initEnv() override;
StringSet completePrefix(const std::string & prefix);
bool getLine(std::string & input, const std::string & prompt);
StorePath getDerivationPath(Value & v);
bool processLine(std::string line);
ProcessLineResult processLine(std::string line);
void loadFile(const Path & path);
void loadFlake(const std::string & flakeRef);
@ -101,7 +122,8 @@ struct NixRepl
.ansiColors = true,
.force = true,
.derivationPaths = true,
.maxDepth = maxDepth
.maxDepth = maxDepth,
.prettyIndent = 2
});
}
};
@ -232,7 +254,7 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
: positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
if (pos) {
out << pos;
out << *pos;
if (auto loc = pos->getCodeLines()) {
out << "\n";
printCodeLines(out, "", *pos, *loc);
@ -243,10 +265,19 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
return out;
}
void NixRepl::mainLoop()
static bool isFirstRepl = true;
ReplExitStatus NixRepl::mainLoop()
{
std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n");
if (isFirstRepl) {
std::string_view debuggerNotice = "";
if (state->debugRepl) {
debuggerNotice = " debugger";
}
notice("Nix %1%%2%\nType :? for help.", nixVersion, debuggerNotice);
}
isFirstRepl = false;
loadFiles();
@ -277,15 +308,25 @@ void NixRepl::mainLoop()
// When continuing input from previous lines, don't print a prompt, just align to the same
// number of chars as the prompt.
if (!getLine(input, input.empty() ? "nix-repl> " : " ")) {
// ctrl-D should exit the debugger.
// Ctrl-D should exit the debugger.
state->debugStop = false;
state->debugQuit = true;
logger->cout("");
break;
// TODO: Should Ctrl-D exit just the current debugger session or
// the entire program?
return ReplExitStatus::QuitAll;
}
logger->resume();
try {
if (!removeWhitespace(input).empty() && !processLine(input)) return;
switch (processLine(input)) {
case ProcessLineResult::Quit:
return ReplExitStatus::QuitAll;
case ProcessLineResult::Continue:
return ReplExitStatus::Continue;
case ProcessLineResult::PromptAgain:
break;
default:
abort();
}
} catch (ParseError & e) {
if (e.msg().find("unexpected end of file") != std::string::npos) {
// For parse errors on incomplete input, we continue waiting for the next line of
@ -342,7 +383,6 @@ bool NixRepl::getLine(std::string & input, const std::string & prompt)
};
setupSignals();
Finally resetTerminal([&]() { rl_deprep_terminal(); });
char * s = readline(prompt.c_str());
Finally doFree([&]() { free(s); });
restoreSignals();
@ -422,8 +462,6 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
// Quietly ignore parse errors.
} catch (EvalError & e) {
// Quietly ignore evaluation errors.
} catch (UndefinedVarError & e) {
// Quietly ignore undefined variable errors.
} catch (BadURL & e) {
// Quietly ignore BadURL flake-related errors.
}
@ -475,10 +513,11 @@ void NixRepl::loadDebugTraceEnv(DebugTrace & dt)
}
}
bool NixRepl::processLine(std::string line)
ProcessLineResult NixRepl::processLine(std::string line)
{
line = trim(line);
if (line == "") return true;
if (line.empty())
return ProcessLineResult::PromptAgain;
_isInterrupted = false;
@ -573,13 +612,13 @@ bool NixRepl::processLine(std::string line)
else if (state->debugRepl && (command == ":s" || command == ":step")) {
// set flag to stop at next DebugTrace; exit repl.
state->debugStop = true;
return false;
return ProcessLineResult::Continue;
}
else if (state->debugRepl && (command == ":c" || command == ":continue")) {
// set flag to run to next breakpoint or end of program; exit repl.
state->debugStop = false;
return false;
return ProcessLineResult::Continue;
}
else if (command == ":a" || command == ":add") {
@ -722,8 +761,7 @@ bool NixRepl::processLine(std::string line)
else if (command == ":q" || command == ":quit") {
state->debugStop = false;
state->debugQuit = true;
return false;
return ProcessLineResult::Quit;
}
else if (command == ":doc") {
@ -784,7 +822,7 @@ bool NixRepl::processLine(std::string line)
}
}
return true;
return ProcessLineResult::PromptAgain;
}
void NixRepl::loadFile(const Path & path)
@ -890,7 +928,7 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
Expr * NixRepl::parseString(std::string s)
{
return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv);
return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv);
}
@ -915,7 +953,7 @@ std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
}
void AbstractNixRepl::runSimple(
ReplExitStatus AbstractNixRepl::runSimple(
ref<EvalState> evalState,
const ValMap & extraEnv)
{
@ -937,7 +975,7 @@ void AbstractNixRepl::runSimple(
for (auto & [name, value] : extraEnv)
repl->addVarToScope(repl->state->symbols.create(name), *value);
repl->mainLoop();
return repl->mainLoop();
}
}

4
src/libcmd/repl.hh
View file

@ -28,13 +28,13 @@ struct AbstractNixRepl
const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
std::function<AnnotatedValues()> getValues);
static void runSimple(
static ReplExitStatus runSimple(
ref<EvalState> evalState,
const ValMap & extraEnv);
virtual void initEnv() = 0;
virtual void mainLoop() = 0;
virtual ReplExitStatus mainLoop() = 0;
};
}

8
src/libexpr/attr-path.cc
View file

@ -65,10 +65,10 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
if (!attrIndex) {
if (v->type() != nAttrs)
throw TypeError(
state.error<TypeError>(
"the expression selected by the selection path '%1%' should be a set but is %2%",
attrPath,
showType(*v));
showType(*v)).debugThrow();
if (attr.empty())
throw Error("empty attribute name in selection path '%1%'", attrPath);
@ -88,10 +88,10 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
else {
if (!v->isList())
throw TypeError(
state.error<TypeError>(
"the expression selected by the selection path '%1%' should be a list but is %2%",
attrPath,
showType(*v));
showType(*v)).debugThrow();
if (*attrIndex >= v->listSize())
throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath);

30
src/libexpr/eval-cache.cc
View file

@ -491,7 +491,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
if (forceErrors)
debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
else
throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name));
} else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
@ -500,7 +500,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
// evaluate to see whether 'name' exists
} else
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
//error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
}
}
@ -508,7 +508,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
if (v.type() != nAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
//error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
auto attr = v.attrs->get(name);
@ -574,14 +574,14 @@ std::string AttrCursor::getString()
debug("using cached string attribute '%s'", getAttrPathStr());
return s->first;
} else
root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow<TypeError>();
root->state.error<TypeError>("'%s' is not a string", getAttrPathStr()).debugThrow();
}
}
auto & v = forceValue();
if (v.type() != nString && v.type() != nPath)
root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>();
root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr()).debugThrow();
return v.type() == nString ? v.c_str() : v.path().to_string();
}
@ -616,7 +616,7 @@ string_t AttrCursor::getStringWithContext()
return *s;
}
} else
root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow<TypeError>();
root->state.error<TypeError>("'%s' is not a string", getAttrPathStr()).debugThrow();
}
}
@ -630,7 +630,7 @@ string_t AttrCursor::getStringWithContext()
else if (v.type() == nPath)
return {v.path().to_string(), {}};
else
root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>();
root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr()).debugThrow();
}
bool AttrCursor::getBool()
@ -643,14 +643,14 @@ bool AttrCursor::getBool()
debug("using cached Boolean attribute '%s'", getAttrPathStr());
return *b;
} else
root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow<TypeError>();
root->state.error<TypeError>("'%s' is not a Boolean", getAttrPathStr()).debugThrow();
}
}
auto & v = forceValue();
if (v.type() != nBool)
root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow<TypeError>();
root->state.error<TypeError>("'%s' is not a Boolean", getAttrPathStr()).debugThrow();
return v.boolean;
}
@ -665,14 +665,14 @@ NixInt AttrCursor::getInt()
debug("using cached integer attribute '%s'", getAttrPathStr());
return i->x;
} else
throw TypeError("'%s' is not an integer", getAttrPathStr());
root->state.error<TypeError>("'%s' is not an integer", getAttrPathStr()).debugThrow();
}
}
auto & v = forceValue();
if (v.type() != nInt)
throw TypeError("'%s' is not an integer", getAttrPathStr());
root->state.error<TypeError>("'%s' is not an integer", getAttrPathStr()).debugThrow();
return v.integer;
}
@ -687,7 +687,7 @@ std::vector<std::string> AttrCursor::getListOfStrings()
debug("using cached list of strings attribute '%s'", getAttrPathStr());
return *l;
} else
throw TypeError("'%s' is not a list of strings", getAttrPathStr());
root->state.error<TypeError>("'%s' is not a list of strings", getAttrPathStr()).debugThrow();
}
}
@ -697,7 +697,7 @@ std::vector<std::string> AttrCursor::getListOfStrings()
root->state.forceValue(v, noPos);
if (v.type() != nList)
throw TypeError("'%s' is not a list", getAttrPathStr());
root->state.error<TypeError>("'%s' is not a list", getAttrPathStr()).debugThrow();
std::vector<std::string> res;
@ -720,14 +720,14 @@ std::vector<Symbol> AttrCursor::getAttrs()
debug("using cached attrset attribute '%s'", getAttrPathStr());
return *attrs;
} else
root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow<TypeError>();
root->state.error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
}
}
auto & v = forceValue();
if (v.type() != nAttrs)
root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow<TypeError>();
root->state.error<TypeError>("'%s' is not an attribute set", getAttrPathStr()).debugThrow();
std::vector<Symbol> attrs;
for (auto & attr : *getValue().attrs)

105
src/libexpr/eval-error.cc Normal file
View file

@ -0,0 +1,105 @@
#include "eval-error.hh"
#include "eval.hh"
#include "value.hh"
namespace nix {
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withExitStatus(unsigned int exitStatus)
{
error.withExitStatus(exitStatus);
return *this;
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(PosIdx pos)
{
error.err.pos = error.state.positions[pos];
return *this;
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
{
return atPos(value.determinePos(fallback));
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
{
error.err.traces.push_front(
Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
return *this;
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withSuggestions(Suggestions & s)
{
error.err.suggestions = s;
return *this;
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrame(const Env & env, const Expr & expr)
{
// NOTE: This is abusing side-effects.
// TODO: check compatibility with nested debugger calls.
// TODO: What side-effects??
error.state.debugTraces.push_front(DebugTrace{
.pos = error.state.positions[expr.getPos()],
.expr = expr,
.env = env,
.hint = HintFmt("Fake frame for debugging purposes"),
.isError = true});
return *this;
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::addTrace(PosIdx pos, HintFmt hint)
{
error.addTrace(error.state.positions[pos], hint);
return *this;
}
template<class T>
template<typename... Args>
EvalErrorBuilder<T> &
EvalErrorBuilder<T>::addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs)
{
addTrace(error.state.positions[pos], HintFmt(std::string(formatString), formatArgs...));
return *this;
}
template<class T>
void EvalErrorBuilder<T>::debugThrow()
{
if (error.state.debugRepl && !error.state.debugTraces.empty()) {
const DebugTrace & last = error.state.debugTraces.front();
const Env * env = &last.env;
const Expr * expr = &last.expr;
error.state.runDebugRepl(&error, *env, *expr);
}
// `EvalState` is the only class that can construct an `EvalErrorBuilder`,
// and it does so in dynamic storage. This is the final method called on
// any such instance and must delete itself before throwing the underlying
// error.
auto error = std::move(this->error);
delete this;
throw error;
}
template class EvalErrorBuilder<EvalError>;
template class EvalErrorBuilder<AssertionError>;
template class EvalErrorBuilder<ThrownError>;
template class EvalErrorBuilder<Abort>;
template class EvalErrorBuilder<TypeError>;
template class EvalErrorBuilder<UndefinedVarError>;
template class EvalErrorBuilder<MissingArgumentError>;
template class EvalErrorBuilder<InfiniteRecursionError>;
template class EvalErrorBuilder<CachedEvalError>;
template class EvalErrorBuilder<InvalidPathError>;
}

104
src/libexpr/eval-error.hh Normal file
View file

@ -0,0 +1,104 @@
#pragma once
#include <algorithm>
#include "error.hh"
#include "pos-idx.hh"
namespace nix {
struct Env;
struct Expr;
struct Value;
class EvalState;
template<class T>
class EvalErrorBuilder;
class EvalError : public Error
{
template<class T>
friend class EvalErrorBuilder;
public:
EvalState & state;
EvalError(EvalState & state, ErrorInfo && errorInfo)
: Error(errorInfo)
, state(state)
{
}
template<typename... Args>
explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
: Error(formatString, formatArgs...)
, state(state)
{
}
};
MakeError(ParseError, Error);
MakeError(AssertionError, EvalError);
MakeError(ThrownError, AssertionError);
MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, EvalError);
MakeError(MissingArgumentError, EvalError);
MakeError(CachedEvalError, EvalError);
MakeError(InfiniteRecursionError, EvalError);
struct InvalidPathError : public EvalError
{
public:
Path path;
InvalidPathError(EvalState & state, const Path & path)
: EvalError(state, "path '%s' is not valid", path)
{
}
};
/**
* `EvalErrorBuilder`s may only be constructed by `EvalState`. The `debugThrow`
* method must be the final method in any such `EvalErrorBuilder` usage, and it
* handles deleting the object.
*/
template<class T>
class EvalErrorBuilder final
{
friend class EvalState;
template<typename... Args>
explicit EvalErrorBuilder(EvalState & state, const Args &... args)
: error(T(state, args...))
{
}
public:
T error;
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withExitStatus(unsigned int exitStatus);
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & atPos(PosIdx pos);
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & atPos(Value & value, PosIdx fallback = noPos);
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withTrace(PosIdx pos, const std::string_view text);
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withFrameTrace(PosIdx pos, const std::string_view text);
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withSuggestions(Suggestions & s);
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withFrame(const Env & e, const Expr & ex);
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint);
template<typename... Args>
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &
addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs);
/**
* Delete the `EvalErrorBuilder` and throw the underlying exception.
*/
[[gnu::noinline, gnu::noreturn]] void debugThrow();
};
}

19
src/libexpr/eval-inline.hh
View file

@ -3,6 +3,7 @@
#include "print.hh"
#include "eval.hh"
#include "eval-error.hh"
namespace nix {
@ -115,10 +116,11 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e
PosIdx pos = getPos();
forceValue(v, pos);
if (v.type() != nAttrs) {
error("expected a set but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx).debugThrow<TypeError>();
error<TypeError>(
"expected a set but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).withTrace(pos, errorCtx).debugThrow();
}
}
@ -128,10 +130,11 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e
{
forceValue(v, pos);
if (!v.isList()) {
error("expected a list but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx).debugThrow<TypeError>();
error<TypeError>(
"expected a list but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).withTrace(pos, errorCtx).debugThrow();
}
}

10
src/libexpr/eval-settings.hh
View file

@ -127,6 +127,16 @@ struct EvalSettings : Config
Setting<unsigned int> maxCallDepth{this, 10000, "max-call-depth",
"The maximum function call depth to allow before erroring."};
Setting<bool> builtinsTraceDebugger{this, false, "debugger-on-trace",
R"(
If set to true and the `--debugger` flag is given,
[`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) will
enter the debugger like
[`builtins.break`](@docroot@/language/builtins.md#builtins-break).
This is useful for debugging warnings in third-party Nix code.
)"};
};
extern EvalSettings evalSettings;

365
src/libexpr/eval.cc
View file

@ -3,6 +3,7 @@
#include "hash.hh"
#include "primops.hh"
#include "print-options.hh"
#include "shared.hh"
#include "types.hh"
#include "util.hh"
#include "store-api.hh"
@ -339,46 +340,6 @@ void initGC()
gcInitialised = true;
}
ErrorBuilder & ErrorBuilder::atPos(PosIdx pos)
{
info.errPos = state.positions[pos];
return *this;
}
ErrorBuilder & ErrorBuilder::withTrace(PosIdx pos, const std::string_view text)
{
info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = false });
return *this;
}
ErrorBuilder & ErrorBuilder::withFrameTrace(PosIdx pos, const std::string_view text)
{
info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = true });
return *this;
}
ErrorBuilder & ErrorBuilder::withSuggestions(Suggestions & s)
{
info.suggestions = s;
return *this;
}
ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr)
{
// NOTE: This is abusing side-effects.
// TODO: check compatibility with nested debugger calls.
state.debugTraces.push_front(DebugTrace {
.pos = nullptr,
.expr = expr,
.env = env,
.hint = hintformat("Fake frame for debugging purposes"),
.isError = true
});
return *this;
}
EvalState::EvalState(
const SearchPath & _searchPath,
ref<Store> store,
@ -434,14 +395,14 @@ EvalState::EvalState(
, emptyBindings(0)
, rootFS(
evalSettings.restrictEval || evalSettings.pureEval
? ref<InputAccessor>(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {},
? ref<InputAccessor>(AllowListInputAccessor::create(makeFSInputAccessor(), {},
[](const CanonPath & path) -> RestrictedPathError {
auto modeInformation = evalSettings.pureEval
? "in pure evaluation mode (use '--impure' to override)"
: "in restricted mode";
throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
}))
: makeFSInputAccessor(CanonPath::root))
: makeFSInputAccessor())
, corepkgsFS(makeMemoryInputAccessor())
, internalFS(makeMemoryInputAccessor())
, derivationInternal{corepkgsFS->addFile(
@ -456,7 +417,6 @@ EvalState::EvalState(
, buildStore(buildStore ? buildStore : store)
, debugRepl(nullptr)
, debugStop(false)
, debugQuit(false)
, trylevel(0)
, regexCache(makeRegexCache())
#if HAVE_BOEHMGC
@ -507,13 +467,13 @@ EvalState::~EvalState()
void EvalState::allowPath(const Path & path)
{
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
rootFS2->allowPath(CanonPath(path));
rootFS2->allowPrefix(CanonPath(path));
}
void EvalState::allowPath(const StorePath & storePath)
{
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
rootFS2->allowPath(CanonPath(store->toRealPath(storePath)));
rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath)));
}
void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
@ -744,7 +704,8 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env &
if (se.up && env.up) {
std::cout << "static: ";
printStaticEnvBindings(st, se);
printWithBindings(st, env);
if (se.isWith)
printWithBindings(st, env);
std::cout << std::endl;
printEnvBindings(st, *se.up, *env.up, ++lvl);
} else {
@ -756,7 +717,8 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env &
std::cout << st[i.first] << " ";
std::cout << ANSI_NORMAL;
std::cout << std::endl;
printWithBindings(st, env); // probably nothing there for the top level.
if (se.isWith)
printWithBindings(st, env); // probably nothing there for the top level.
std::cout << std::endl;
}
@ -778,7 +740,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En
if (env.up && se.up) {
mapStaticEnvBindings(st, *se.up, *env.up, vm);
if (!env.values[0]->isThunk()) {
if (se.isWith && !env.values[0]->isThunk()) {
// add 'with' bindings.
Bindings::iterator j = env.values[0]->attrs->begin();
while (j != env.values[0]->attrs->end()) {
@ -811,7 +773,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
? std::make_unique<DebugTraceStacker>(
*this,
DebugTrace {
.pos = error->info().errPos ? error->info().errPos : positions[expr.getPos()],
.pos = error->info().pos ? error->info().pos : positions[expr.getPos()],
.expr = expr,
.env = env,
.hint = error->info().msg,
@ -821,45 +783,55 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
if (error)
{
printError("%s\n\n", error->what());
printError("%s\n", error->what());
if (trylevel > 0 && error->info().level != lvlInfo)
printError("This exception occurred in a 'tryEval' call. Use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n");
printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL);
}
auto se = getStaticEnv(expr);
if (se) {
auto vm = mapStaticEnvBindings(symbols, *se.get(), env);
(debugRepl)(ref<EvalState>(shared_from_this()), *vm);
auto exitStatus = (debugRepl)(ref<EvalState>(shared_from_this()), *vm);
switch (exitStatus) {
case ReplExitStatus::QuitAll:
if (error)
throw *error;
throw Exit(0);
case ReplExitStatus::Continue:
break;
default:
abort();
}
}
}
void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) const
template<typename... Args>
void EvalState::addErrorTrace(Error & e, const Args & ... formatArgs) const
{
e.addTrace(nullptr, s, s2);
e.addTrace(nullptr, HintFmt(formatArgs...));
}
void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame) const
template<typename... Args>
void EvalState::addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const
{
e.addTrace(positions[pos], hintfmt(s, s2), frame);
e.addTrace(positions[pos], HintFmt(formatArgs...));
}
template<typename... Args>
static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
EvalState & state,
Expr & expr,
Env & env,
std::shared_ptr<Pos> && pos,
const char * s,
const std::string & s2)
const Args & ... formatArgs)
{
return std::make_unique<DebugTraceStacker>(state,
DebugTrace {
.pos = std::move(pos),
.expr = expr,
.env = env,
.hint = hintfmt(s, s2),
.hint = HintFmt(formatArgs...),
.isError = false
});
}
@ -930,7 +902,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
return j->value;
}
if (!fromWith->parentWith)
error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow<UndefinedVarError>();
error<UndefinedVarError>("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow();
for (size_t l = fromWith->prevWith; l; --l, env = env->up) ;
fromWith = fromWith->parentWith;
}
@ -1136,7 +1108,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
// computation.
if (mustBeTrivial &&
!(dynamic_cast<ExprAttrs *>(e)))
error("file '%s' must be an attribute set", path).debugThrow<EvalError>();
error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
eval(e, v);
} catch (Error & e) {
addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string());
@ -1167,10 +1139,11 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri
Value v;
e->eval(*this, env, v);
if (v.type() != nBool)
error("expected a Boolean but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.withFrame(env, *e).debugThrow<TypeError>();
error<TypeError>(
"expected a Boolean but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).withFrame(env, *e).debugThrow();
return v.boolean;
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
@ -1184,10 +1157,11 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po
try {
e->eval(*this, env, v);
if (v.type() != nAttrs)
error("expected a set but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.withFrame(env, *e).debugThrow<TypeError>();
error<TypeError>(
"expected a set but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).withFrame(env, *e).debugThrow();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
@ -1224,6 +1198,18 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v)
}
Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
{
Env & inheritEnv = state.allocEnv(inheritFromExprs->size());
inheritEnv.up = &up;
Displacement displ = 0;
for (auto from : *inheritFromExprs)
inheritEnv.values[displ++] = from->maybeThunk(state, up);
return &inheritEnv;
}
void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
{
v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish());
@ -1235,6 +1221,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
Env & env2(state.allocEnv(attrs.size()));
env2.up = &env;
dynamicEnv = &env2;
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
AttrDefs::iterator overrides = attrs.find(state.sOverrides);
bool hasOverrides = overrides != attrs.end();
@ -1245,11 +1232,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
Displacement displ = 0;
for (auto & i : attrs) {
Value * vAttr;
if (hasOverrides && !i.second.inherited) {
if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
vAttr = state.allocValue();
mkThunk(*vAttr, env2, i.second.e);
mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e);
} else
vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
env2.values[displ++] = vAttr;
v.attrs->push_back(Attr(i.first, vAttr, i.second.pos));
}
@ -1281,9 +1268,15 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
}
}
else
for (auto & i : attrs)
v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), i.second.pos));
else {
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
for (auto & i : attrs) {
v.attrs->push_back(Attr(
i.first,
i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)),
i.second.pos));
}
}
/* Dynamic attrs apply *after* rec and __overrides. */
for (auto & i : dynamicAttrs) {
@ -1296,7 +1289,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
auto nameSym = state.symbols.create(nameVal.string_view());
Bindings::iterator j = v.attrs->find(nameSym);
if (j != v.attrs->end())
state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow<EvalError>();
state.error<EvalError>("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow();
i.valueExpr->setName(nameSym);
/* Keep sorted order so find can catch duplicates */
@ -1315,12 +1308,30 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
Env & env2(state.allocEnv(attrs->attrs.size()));
env2.up = &env;
Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
/* The recursive attributes are evaluated in the new environment,
while the inherited attributes are evaluated in the original
environment. */
Displacement displ = 0;
for (auto & i : attrs->attrs)
env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
for (auto & i : attrs->attrs) {
env2.values[displ++] = i.second.e->maybeThunk(
state,
*i.second.chooseByKind(&env2, &env, inheritEnv));
}
auto dts = state.debugRepl
? makeDebugTraceStacker(
state,
*this,
env2,
getPos()
? std::make_shared<Pos>(state.positions[getPos()])
: nullptr,
"while evaluating a '%1%' expression",
"let"
)
: nullptr;
body->eval(state, env2, v);
}
@ -1384,7 +1395,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
state,
*this,
env,
state.positions[pos2],
state.positions[getPos()],
"while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath))
: nullptr;
@ -1408,8 +1419,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
for (auto & attr : *vAttrs->attrs)
allAttrNames.insert(state.symbols[attr.name]);
auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
state.error("attribute '%1%' missing", state.symbols[name])
.atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow<EvalError>();
state.error<EvalError>("attribute '%1%' missing", state.symbols[name])
.atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow();
}
}
vAttrs = j->value;
@ -1482,7 +1493,7 @@ public:
void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
{
if (callDepth > evalSettings.maxCallDepth)
error("stack overflow; max-call-depth exceeded").atPos(pos).template debugThrow<EvalError>();
error<EvalError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
CallDepth _level(callDepth);
auto trace = evalSettings.traceFunctionCalls
@ -1540,13 +1551,13 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
auto j = args[0]->attrs->get(i.name);
if (!j) {
if (!i.def) {
error("function '%1%' called without required argument '%2%'",
error<TypeError>("function '%1%' called without required argument '%2%'",
(lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
symbols[i.name])
.atPos(lambda.pos)
.withTrace(pos, "from call site")
.withFrame(*fun.lambda.env, lambda)
.debugThrow<TypeError>();
.debugThrow();
}
env2.values[displ++] = i.def->maybeThunk(*this, env2);
} else {
@ -1566,14 +1577,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
for (auto & formal : lambda.formals->formals)
formalNames.insert(symbols[formal.name]);
auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]);
error("function '%1%' called with unexpected argument '%2%'",
error<TypeError>("function '%1%' called with unexpected argument '%2%'",
(lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
symbols[i.name])
.atPos(lambda.pos)
.withTrace(pos, "from call site")
.withSuggestions(suggestions)
.withFrame(*fun.lambda.env, lambda)
.debugThrow<TypeError>();
.debugThrow();
}
abort(); // can't happen
}
@ -1602,9 +1613,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
"while calling %s",
lambda.name
? concatStrings("'", symbols[lambda.name], "'")
: "anonymous lambda",
true);
if (pos) addErrorTrace(e, pos, "from call site%s", "", true);
: "anonymous lambda");
if (pos) addErrorTrace(e, pos, "from call site");
}
throw;
}
@ -1705,11 +1715,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
}
else
error("attempt to call something which is not a function but %1%: %2%",
error<TypeError>(
"attempt to call something which is not a function but %1%: %2%",
showType(vCur),
ValuePrinter(*this, vCur, errorPrintOptions))
.atPos(pos)
.debugThrow<TypeError>();
.debugThrow();
}
vRes = vCur;
@ -1718,6 +1729,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
void ExprCall::eval(EvalState & state, Env & env, Value & v)
{
auto dts = state.debugRepl
? makeDebugTraceStacker(
state,
*this,
env,
getPos()
? std::make_shared<Pos>(state.positions[getPos()])
: nullptr,
"while calling a function"
)
: nullptr;
Value vFun;
fun->eval(state, env, vFun);
@ -1779,12 +1802,12 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
if (j != args.end()) {
attrs.insert(*j);
} else if (!i.def) {
error(R"(cannot evaluate a function that has an argument without a value ('%1%')
error<MissingArgumentError>(R"(cannot evaluate a function that has an argument without a value ('%1%')
Nix attempted to evaluate a function as a top level expression; in
this case it must have its arguments supplied either by default
values, or passed explicitly with '--arg' or '--argstr'. See
https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name])
.atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow<MissingArgumentError>();
.atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow();
}
}
}
@ -1815,7 +1838,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
std::ostringstream out;
cond->show(state.symbols, out);
state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow<AssertionError>();
state.error<AssertionError>("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow();
}
body->eval(state, env, v);
}
@ -1993,14 +2016,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
nf = n;
nf += vTmp.fpoint;
} else
state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow<EvalError>();
state.error<EvalError>("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
} else if (firstType == nFloat) {
if (vTmp.type() == nInt) {
nf += vTmp.integer;
} else if (vTmp.type() == nFloat) {
nf += vTmp.fpoint;
} else
state.error("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow<EvalError>();
state.error<EvalError>("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
} else {
if (s.empty()) s.reserve(es->size());
/* skip canonization of first path, which would only be not
@ -2022,7 +2045,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
v.mkFloat(nf);
else if (firstType == nPath) {
if (!context.empty())
state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>();
state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
} else
v.mkStringMove(c_str(), context);
@ -2037,8 +2060,9 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v)
void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
{
state.error("infinite recursion encountered")
.debugThrow<InfiniteRecursionError>();
state.error<InfiniteRecursionError>("infinite recursion encountered")
.atPos(v.determinePos(noPos))
.debugThrow();
}
// always force this to be separate, otherwise forceValue may inline it and take
@ -2052,7 +2076,7 @@ void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
try {
std::rethrow_exception(e);
} catch (InfiniteRecursionError & e) {
e.err.errPos = positions[pos];
e.atPos(positions[pos]);
} catch (...) {
}
}
@ -2100,15 +2124,18 @@ NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCt
try {
forceValue(v, pos);
if (v.type() != nInt)
error("expected an integer but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.debugThrow<TypeError>();
error<TypeError>(
"expected an integer but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
return v.integer;
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
return v.integer;
}
@ -2119,10 +2146,11 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err
if (v.type() == nInt)
return v.integer;
else if (v.type() != nFloat)
error("expected a float but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.debugThrow<TypeError>();
error<TypeError>(
"expected a float but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
return v.fpoint;
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
@ -2136,15 +2164,18 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx
try {
forceValue(v, pos);
if (v.type() != nBool)
error("expected a Boolean but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.debugThrow<TypeError>();
error<TypeError>(
"expected a Boolean but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
return v.boolean;
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
return v.boolean;
}
@ -2159,10 +2190,11 @@ void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view erro
try {
forceValue(v, pos);
if (v.type() != nFunction && !isFunctor(v))
error("expected a function but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.debugThrow<TypeError>();
error<TypeError>(
"expected a function but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
@ -2175,10 +2207,11 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
try {
forceValue(v, pos);
if (v.type() != nString)
error("expected a string but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.debugThrow<TypeError>();
error<TypeError>(
"expected a string but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
return v.string_view();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
@ -2207,7 +2240,7 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
{
auto s = forceString(v, pos, errorCtx);
if (v.context()) {
error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow<EvalError>();
error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
}
return s;
}
@ -2272,11 +2305,13 @@ BackedStringView EvalState::coerceToString(
return std::move(*maybeString);
auto i = v.attrs->find(sOutPath);
if (i == v.attrs->end()) {
error("cannot coerce %1% to a string: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
error<TypeError>(
"cannot coerce %1% to a string: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
)
.withTrace(pos, errorCtx)
.debugThrow<TypeError>();
.debugThrow();
}
return coerceToString(pos, *i->value, context, errorCtx,
coerceMore, copyToStore, canonicalizePath);
@ -2284,7 +2319,7 @@ BackedStringView EvalState::coerceToString(
if (v.type() == nExternal) {
try {
return v.external->coerceToString(positions[pos], context, coerceMore, copyToStore);
return v.external->coerceToString(*this, pos, context, coerceMore, copyToStore);
} catch (Error & e) {
e.addTrace(nullptr, errorCtx);
throw;
@ -2320,25 +2355,33 @@ BackedStringView EvalState::coerceToString(
}
}
error("cannot coerce %1% to a string: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
error<TypeError>("cannot coerce %1% to a string: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
)
.withTrace(pos, errorCtx)
.debugThrow<TypeError>();
.debugThrow();
}
StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path)
{
if (nix::isDerivation(path.path.abs()))
error("file names are not allowed to end in '%1%'", drvExtension).debugThrow<EvalError>();
error<EvalError>("file names are not allowed to end in '%1%'", drvExtension).debugThrow();
auto i = srcToStore.find(path);
auto dstPath = i != srcToStore.end()
? i->second
: [&]() {
auto dstPath = fetchToStore(*store, path.resolveSymlinks(), path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
auto dstPath = fetchToStore(
*store,
path.resolveSymlinks(),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
path.baseName(),
FileIngestionMethod::Recursive,
nullptr,
repair);
allowPath(dstPath);
srcToStore.insert_or_assign(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@ -2380,7 +2423,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
relative to the root filesystem. */
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (path == "" || path[0] != '/')
error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
error<EvalError>("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow();
return rootPath(CanonPath(path));
}
@ -2390,7 +2433,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (auto storePath = store->maybeParseStorePath(path))
return *storePath;
error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
}
@ -2400,18 +2443,18 @@ std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedP
auto s = forceString(v, context, pos, errorCtx);
auto csize = context.size();
if (csize != 1)
error(
error<EvalError>(
"string '%s' has %d entries in its context. It should only have exactly one entry",
s, csize)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
.withTrace(pos, errorCtx).debugThrow();
auto derivedPath = std::visit(overloaded {
[&](NixStringContextElem::Opaque && o) -> SingleDerivedPath {
return std::move(o);
},
[&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
error(
error<EvalError>(
"string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
s).withTrace(pos, errorCtx).debugThrow<EvalError>();
s).withTrace(pos, errorCtx).debugThrow();
},
[&](NixStringContextElem::Built && b) -> SingleDerivedPath {
return std::move(b);
@ -2434,16 +2477,16 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
error message. */
std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
error(
error<EvalError>(
"path string '%s' has context with the different path '%s'",
s, sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
.withTrace(pos, errorCtx).debugThrow();
},
[&](const SingleDerivedPath::Built & b) {
error(
error<EvalError>(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
s, b.output, b.drvPath->to_string(*store), sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
.withTrace(pos, errorCtx).debugThrow();
}
}, derivedPath.raw());
}
@ -2528,7 +2571,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
case nThunk: // Must not be left by forceValue
default:
error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow<EvalError>();
error<EvalError>("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow();
}
}
@ -2689,14 +2732,14 @@ SourcePath resolveExprPath(SourcePath path)
// Basic cycle/depth limit to avoid infinite loops.
if (++followCount >= maxFollow)
throw Error("too many symbolic links encountered while traversing the path '%s'", path);
auto p = path.parent().resolveSymlinks() + path.baseName();
auto p = path.parent().resolveSymlinks() / path.baseName();
if (p.lstat().type != InputAccessor::tSymlink) break;
path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))};
}
/* If `path' refers to a directory, append `/default.nix'. */
if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory)
return path + "default.nix";
return path / "default.nix";
return path;
}
@ -2738,7 +2781,7 @@ Expr * EvalState::parseStdin()
// drainFD should have left some extra space for terminators
buffer.append("\0\0", 2);
auto s = make_ref<std::string>(std::move(buffer));
return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv);
}
@ -2767,13 +2810,12 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
if (hasPrefix(path, "nix/"))
return {corepkgsFS, CanonPath(path.substr(3))};
debugThrow(ThrownError({
.msg = hintfmt(evalSettings.pureEval
error<ThrownError>(
evalSettings.pureEval
? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
: "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
path),
.errPos = positions[pos]
}), 0, 0);
path
).atPos(pos).debugThrow();
}
@ -2787,12 +2829,13 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
if (EvalSettings::isPseudoUrl(value)) {
try {
auto storePath = fetchers::downloadTarball(
store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath;
auto accessor = fetchers::downloadTarball(
EvalSettings::resolvePseudoUrl(value)).accessor;
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
res = { store->toRealPath(storePath) };
} catch (FileTransferError & e) {
} catch (Error & e) {
logWarning({
.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
.msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
});
}
}
@ -2825,7 +2868,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
res = { path };
else {
logWarning({
.msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)
.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)
});
res = std::nullopt;
}
@ -2856,11 +2899,11 @@ Expr * EvalState::parse(
}
std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
{
throw TypeError({
.msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), *this)
});
state.error<TypeError>(
"cannot coerce %1% to a string: %2%", showType(), *this
).atPos(pos).debugThrow();
}

110
src/libexpr/eval.hh
View file

@ -2,6 +2,7 @@
///@file
#include "attr-set.hh"
#include "eval-error.hh"
#include "types.hh"
#include "value.hh"
#include "nixexpr.hh"
@ -10,6 +11,7 @@
#include "experimental-features.hh"
#include "input-accessor.hh"
#include "search-path.hh"
#include "repl-exit-status.hh"
#include <map>
#include <optional>
@ -147,49 +149,10 @@ struct DebugTrace {
std::shared_ptr<Pos> pos;
const Expr & expr;
const Env & env;
hintformat hint;
HintFmt hint;
bool isError;
};
void debugError(Error * e, Env & env, Expr & expr);
class ErrorBuilder
{
private:
EvalState & state;
ErrorInfo info;
ErrorBuilder(EvalState & s, ErrorInfo && i): state(s), info(i) { }
public:
template<typename... Args>
[[nodiscard, gnu::noinline]]
static ErrorBuilder * create(EvalState & s, const Args & ... args)
{
return new ErrorBuilder(s, ErrorInfo { .msg = hintfmt(args...) });
}
[[nodiscard, gnu::noinline]]
ErrorBuilder & atPos(PosIdx pos);
[[nodiscard, gnu::noinline]]
ErrorBuilder & withTrace(PosIdx pos, const std::string_view text);
[[nodiscard, gnu::noinline]]
ErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text);
[[nodiscard, gnu::noinline]]
ErrorBuilder & withSuggestions(Suggestions & s);
[[nodiscard, gnu::noinline]]
ErrorBuilder & withFrame(const Env & e, const Expr & ex);
template<class ErrorType>
[[gnu::noinline, gnu::noreturn]]
void debugThrow();
};
class EvalState : public std::enable_shared_from_this<EvalState>
{
public:
@ -257,9 +220,8 @@ public:
/**
* Debugger
*/
void (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
ReplExitStatus (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
bool debugStop;
bool debugQuit;
int trylevel;
std::list<DebugTrace> debugTraces;
std::map<const Expr*, const std::shared_ptr<const StaticEnv>> exprEnvs;
@ -274,39 +236,11 @@ public:
void runDebugRepl(const Error * error, const Env & env, const Expr & expr);
template<class E>
[[gnu::noinline, gnu::noreturn]]
void debugThrowLastTrace(E && error)
{
debugThrow(error, nullptr, nullptr);
}
template<class E>
[[gnu::noinline, gnu::noreturn]]
void debugThrow(E && error, const Env * env, const Expr * expr)
{
if (debugRepl && ((env && expr) || !debugTraces.empty())) {
if (!env || !expr) {
const DebugTrace & last = debugTraces.front();
env = &last.env;
expr = &last.expr;
}
runDebugRepl(&error, *env, *expr);
}
throw std::move(error);
}
// This is dangerous, but gets in line with the idea that error creation and
// throwing should not allocate on the stack of hot functions.
// as long as errors are immediately thrown, it works.
ErrorBuilder * errorBuilder;
template<typename... Args>
template<class T, typename... Args>
[[nodiscard, gnu::noinline]]
ErrorBuilder & error(const Args & ... args) {
errorBuilder = ErrorBuilder::create(*this, args...);
return *errorBuilder;
EvalErrorBuilder<T> & error(const Args & ... args) {
// `EvalErrorBuilder::debugThrow` performs the corresponding `delete`.
return *new EvalErrorBuilder<T>(*this, args...);
}
private:
@ -372,6 +306,11 @@ public:
*/
SourcePath rootPath(CanonPath path);
/**
* Variant which accepts relative paths too.
*/
SourcePath rootPath(PathView path);
/**
* Allow access to a path.
*/
@ -493,10 +432,12 @@ public:
std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx);
std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);
template<typename... Args>
[[gnu::noinline]]
void addErrorTrace(Error & e, const char * s, const std::string & s2) const;
void addErrorTrace(Error & e, const Args & ... formatArgs) const;
template<typename... Args>
[[gnu::noinline]]
void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const;
void addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const;
public:
/**
@ -819,7 +760,6 @@ struct DebugTraceStacker {
DebugTraceStacker(EvalState & evalState, DebugTrace t);
~DebugTraceStacker()
{
// assert(evalState.debugTraces.front() == trace);
evalState.debugTraces.pop_front();
}
EvalState & evalState;
@ -845,22 +785,6 @@ SourcePath resolveExprPath(SourcePath path);
*/
bool isAllowedURI(std::string_view uri, const Strings & allowedPaths);
struct InvalidPathError : EvalError
{
Path path;
InvalidPathError(const Path & path);
#ifdef EXCEPTION_NEEDS_THROW_SPEC
~InvalidPathError() throw () { };
#endif
};
template<class ErrorType>
void ErrorBuilder::debugThrow()
{
// NOTE: We always use the -LastTrace version as we push the new trace in withFrame()
state.debugThrowLastTrace(ErrorType(info));
}
}
#include "eval-inline.hh"

61
src/libexpr/flake/call-flake.nix
View file

@ -1,20 +1,52 @@
lockFileStr: rootSrc: rootSubdir:
# This is a helper to callFlake() to lazily fetch flake inputs.
# The contents of the lock file, in JSON format.
lockFileStr:
# A mapping of lock file node IDs to { sourceInfo, subdir } attrsets,
# with sourceInfo.outPath providing an InputAccessor to a previously
# fetched tree. This is necessary for possibly unlocked inputs, in
# particular the root input, but also --override-inputs pointing to
# unlocked trees.
overrides:
let
lockFile = builtins.fromJSON lockFileStr;
# Resolve a input spec into a node name. An input spec is
# either a node name, or a 'follows' path from the root
# node.
resolveInput = inputSpec:
if builtins.isList inputSpec
then getInputByPath lockFile.root inputSpec
else inputSpec;
# Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
# root node, returning the final node.
getInputByPath = nodeName: path:
if path == []
then nodeName
else
getInputByPath
# Since this could be a 'follows' input, call resolveInput.
(resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
(builtins.tail path);
allNodes =
builtins.mapAttrs
(key: node:
let
sourceInfo =
if key == lockFile.root
then rootSrc
else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
if overrides ? ${key}
then
overrides.${key}.sourceInfo
else
# FIXME: remove obsolete node.info.
fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
subdir = overrides.${key}.dir or node.locked.dir or "";
outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir);
@ -24,25 +56,6 @@ let
(inputName: inputSpec: allNodes.${resolveInput inputSpec})
(node.inputs or {});
# Resolve a input spec into a node name. An input spec is
# either a node name, or a 'follows' path from the root
# node.
resolveInput = inputSpec:
if builtins.isList inputSpec
then getInputByPath lockFile.root inputSpec
else inputSpec;
# Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
# root node, returning the final node.
getInputByPath = nodeName: path:
if path == []
then nodeName
else
getInputByPath
# Since this could be a 'follows' input, call resolveInput.
(resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
(builtins.tail path);
outputs = flake.outputs (inputs // { self = result; });
result =

104
src/libexpr/flake/flake.cc
View file

@ -147,15 +147,15 @@ static FlakeInput parseFlakeInput(EvalState & state,
NixStringContext emptyContext = {};
attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump());
} else
throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
state.symbols[attr.name], showType(*attr.value));
state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
state.symbols[attr.name], showType(*attr.value)).debugThrow();
}
#pragma GCC diagnostic pop
}
} catch (Error & e) {
e.addTrace(
state.positions[attr.pos],
hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name]));
HintFmt("while evaluating flake attribute '%s'", state.symbols[attr.name]));
throw;
}
}
@ -164,7 +164,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
try {
input.ref = FlakeRef::fromAttrs(attrs);
} catch (Error & e) {
e.addTrace(state.positions[pos], hintfmt("while evaluating flake input"));
e.addTrace(state.positions[pos], HintFmt("while evaluating flake input"));
throw;
}
else {
@ -295,15 +295,15 @@ static Flake getFlake(
std::vector<std::string> ss;
for (auto elem : setting.value->listItems()) {
if (elem->type() != nString)
throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
state.symbols[setting.name], showType(*setting.value));
state.error<TypeError>("list element in flake configuration setting '%s' is %s while a string is expected",
state.symbols[setting.name], showType(*setting.value)).debugThrow();
ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos, ""));
}
flake.config.settings.emplace(state.symbols[setting.name], ss);
}
else
throw TypeError("flake configuration setting '%s' is %s",
state.symbols[setting.name], showType(*setting.value));
state.error<TypeError>("flake configuration setting '%s' is %s",
state.symbols[setting.name], showType(*setting.value)).debugThrow();
}
}
@ -365,6 +365,7 @@ LockedFlake lockFlake(
std::map<InputPath, FlakeInput> overrides;
std::set<InputPath> explicitCliOverrides;
std::set<InputPath> overridesUsed, updatesUsed;
std::map<ref<Node>, StorePath> nodePaths;
for (auto & i : lockFlags.inputOverrides) {
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
@ -535,11 +536,13 @@ LockedFlake lockFlake(
}
}
computeLocks(
mustRefetch
? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs
: fakeInputs,
childNode, inputPath, oldLock, lockRootPath, parentPath, !mustRefetch);
if (mustRefetch) {
auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath);
nodePaths.emplace(childNode, inputFlake.storePath);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false);
} else {
computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true);
}
} else {
/* We need to create a new lock file entry. So fetch
@ -584,6 +587,7 @@ LockedFlake lockFlake(
flake. Also, unless we already have this flake
in the top-level lock file, use this flake's
own lock file. */
nodePaths.emplace(childNode, inputFlake.storePath);
computeLocks(
inputFlake.inputs, childNode, inputPath,
oldLock
@ -596,11 +600,13 @@ LockedFlake lockFlake(
}
else {
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, useRegistries, flakeCache);
auto childNode = make_ref<LockedNode>(lockedRef, ref, false);
nodePaths.emplace(childNode, storePath);
node->inputs.insert_or_assign(id, childNode);
}
}
@ -615,6 +621,8 @@ LockedFlake lockFlake(
// Bring in the current ref for relative path resolution if we have it
auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true);
nodePaths.emplace(newLockFile.root, flake.storePath);
computeLocks(
flake.inputs,
newLockFile.root,
@ -707,14 +715,6 @@ LockedFlake lockFlake(
flake.lockedRef.input.getRev() &&
prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
/* Make sure that we picked up the change,
i.e. the tree should usually be dirty
now. Corner case: we could have reverted from a
dirty to a clean tree! */
if (flake.lockedRef.input == prevLockedRef.input
&& !flake.lockedRef.input.isLocked())
throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
}
} else
throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
@ -724,7 +724,11 @@ LockedFlake lockFlake(
}
}
return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
return LockedFlake {
.flake = std::move(flake),
.lockFile = std::move(newLockFile),
.nodePaths = std::move(nodePaths)
};
} catch (Error & e) {
e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string());
@ -736,30 +740,48 @@ void callFlake(EvalState & state,
const LockedFlake & lockedFlake,
Value & vRes)
{
auto vLocks = state.allocValue();
auto vRootSrc = state.allocValue();
auto vRootSubdir = state.allocValue();
auto vTmp1 = state.allocValue();
auto vTmp2 = state.allocValue();
experimentalFeatureSettings.require(Xp::Flakes);
vLocks->mkString(lockedFlake.lockFile.to_string());
auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string();
emitTreeAttrs(
state,
lockedFlake.flake.storePath,
lockedFlake.flake.lockedRef.input,
*vRootSrc,
false,
lockedFlake.flake.forceDirty);
auto overrides = state.buildBindings(lockedFlake.nodePaths.size());
vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
for (auto & [node, storePath] : lockedFlake.nodePaths) {
auto override = state.buildBindings(2);
auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo"));
auto lockedNode = node.dynamic_pointer_cast<const LockedNode>();
emitTreeAttrs(
state,
storePath,
lockedNode ? lockedNode->lockedRef.input : lockedFlake.flake.lockedRef.input,
vSourceInfo,
false,
!lockedNode && lockedFlake.flake.forceDirty);
auto key = keyMap.find(node);
assert(key != keyMap.end());
override
.alloc(state.symbols.create("dir"))
.mkString(lockedNode ? lockedNode->lockedRef.subdir : lockedFlake.flake.lockedRef.subdir);
overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
}
auto & vOverrides = state.allocValue()->mkAttrs(overrides);
auto vCallFlake = state.allocValue();
state.evalFile(state.callFlakeInternal, *vCallFlake);
auto vTmp1 = state.allocValue();
auto vLocks = state.allocValue();
vLocks->mkString(lockFileStr);
state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
state.callFunction(*vTmp1, vOverrides, vRes, noPos);
}
static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@ -865,11 +887,11 @@ static void prim_flakeRefToString(
attrs.emplace(state.symbols[attr.name],
std::string(attr.value->string_view()));
} else {
state.error(
state.error<EvalError>(
"flake reference attribute sets may only contain integers, Booleans, "
"and strings, but attribute '%s' is %s",
state.symbols[attr.name],
showType(*attr.value)).debugThrow<EvalError>();
showType(*attr.value)).debugThrow();
}
}
auto flakeRef = FlakeRef::fromAttrs(attrs);

7
src/libexpr/flake/flake.hh
View file

@ -103,6 +103,13 @@ struct LockedFlake
Flake flake;
LockFile lockFile;
/**
* Store paths of nodes that have been fetched in
* lockFlake(); in particular, the root node and the overriden
* inputs.
*/
std::map<ref<Node>, StorePath> nodePaths;
Fingerprint getFingerprint() const;
};

19
src/libexpr/flake/lockfile.cc
View file

@ -38,7 +38,7 @@ LockedNode::LockedNode(const nlohmann::json & json)
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
{
if (!lockedRef.input.isLocked())
throw Error("lock file contains mutable lock '%s'",
throw Error("lock file contains unlocked input '%s'",
fetchers::attrsToJSON(lockedRef.input.toAttrs()));
}
@ -107,7 +107,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
std::string inputKey = i.value();
auto k = nodeMap.find(inputKey);
if (k == nodeMap.end()) {
auto nodes = json["nodes"];
auto & nodes = json["nodes"];
auto jsonNode2 = nodes.find(inputKey);
if (jsonNode2 == nodes.end())
throw Error("lock file references missing node '%s'", inputKey);
@ -134,10 +134,10 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
// a bit since we don't need to worry about cycles.
}
nlohmann::json LockFile::toJSON() const
std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
{
nlohmann::json nodes;
std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
KeyMap nodeKeys;
std::unordered_set<std::string> keys;
std::function<std::string(const std::string & key, ref<const Node> node)> dumpNode;
@ -194,12 +194,13 @@ nlohmann::json LockFile::toJSON() const
json["root"] = dumpNode("root", root);
json["nodes"] = std::move(nodes);
return json;
return {json, std::move(nodeKeys)};
}
std::string LockFile::to_string() const
std::pair<std::string, LockFile::KeyMap> LockFile::to_string() const
{
return toJSON().dump(2);
auto [json, nodeKeys] = toJSON();
return {json.dump(2), std::move(nodeKeys)};
}
LockFile LockFile::read(const Path & path)
@ -210,7 +211,7 @@ LockFile LockFile::read(const Path & path)
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
{
stream << lockFile.toJSON().dump(2);
stream << lockFile.toJSON().first.dump(2);
return stream;
}
@ -243,7 +244,7 @@ std::optional<FlakeRef> LockFile::isUnlocked() const
bool LockFile::operator ==(const LockFile & other) const
{
// FIXME: slow
return toJSON() == other.toJSON();
return toJSON().first == other.toJSON().first;
}
bool LockFile::operator !=(const LockFile & other) const

7
src/libexpr/flake/lockfile.hh
View file

@ -59,14 +59,15 @@ struct LockFile
typedef std::map<ref<const Node>, std::string> KeyMap;
nlohmann::json toJSON() const;
std::pair<nlohmann::json, KeyMap> toJSON() const;
std::string to_string() const;
std::pair<std::string, KeyMap> to_string() const;
static LockFile read(const Path & path);
/**
* Check whether this lock file has any unlocked inputs.
* Check whether this lock file has any unlocked inputs. If so,
* return one.
*/
std::optional<FlakeRef> isUnlocked() const;

14
src/libexpr/flake/url-name.cc
View file

@ -5,13 +5,12 @@
namespace nix {
static const std::string attributeNamePattern("[a-zA-Z0-9_-]+");
static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
static const std::regex lastAttributeRegex("^((?:" + attributeNamePattern + "\\.)*)(" + attributeNamePattern +")(\\^.*)?$");
static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
static const std::regex gitProviderRegex("github|gitlab|sourcehut");
static const std::regex gitSchemeRegex("git($|\\+.*)");
static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)");
std::optional<std::string> getNameFromURL(const ParsedURL & url)
{
@ -22,8 +21,11 @@ std::optional<std::string> getNameFromURL(const ParsedURL & url)
return url.query.at("dir");
/* If the fragment isn't a "default" and contains two attribute elements, use the last one */
if (std::regex_match(url.fragment, match, lastAttributeRegex))
return match.str(1);
if (std::regex_match(url.fragment, match, lastAttributeRegex)
&& match.str(1) != "defaultPackage."
&& match.str(2) != "default") {
return match.str(2);
}
/* If this is a github/gitlab/sourcehut flake, use the repo name */
if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
@ -33,10 +35,6 @@ std::optional<std::string> getNameFromURL(const ParsedURL & url)
if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
return match.str(1);
/* If everything failed but there is a non-default fragment, use it in full */
if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
return url.fragment;
/* If there is no fragment, take the last element of the path */
if (std::regex_match(url.path, match, lastPathSegmentRegex))
return match.str(1);

5
src/libexpr/get-drvs.cc
View file

@ -49,7 +49,7 @@ std::string PackageInfo::queryName() const
{
if (name == "" && attrs) {
auto i = attrs->find(state->sName);
if (i == attrs->end()) throw TypeError("derivation name missing");
if (i == attrs->end()) state->error<TypeError>("derivation name missing").debugThrow();
name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation");
}
return name;
@ -396,7 +396,8 @@ static void getDerivations(EvalState & state, Value & vIn,
}
}
else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
else
state.error<TypeError>("expression does not evaluate to a derivation (or a set or list of those)").debugThrow();
}

4
src/libexpr/json-to-value.cc
View file

@ -1,4 +1,6 @@
#include "json-to-value.hh"
#include "value.hh"
#include "eval.hh"
#include <variant>
#include <nlohmann/json.hpp>
@ -159,7 +161,7 @@ public:
}
bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
throw JSONParseError(ex.what());
throw JSONParseError("%s", ex.what());
}
};

7
src/libexpr/json-to-value.hh
View file

@ -1,13 +1,16 @@
#pragma once
///@file
#include "eval.hh"
#include "error.hh"
#include <string>
namespace nix {
MakeError(JSONParseError, EvalError);
class EvalState;
struct Value;
MakeError(JSONParseError, Error);
void parseJSON(EvalState & state, const std::string_view & s, Value & v);

21
src/libexpr/lexer.l
View file

@ -94,6 +94,9 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
}
// yacc generates code that uses unannotated fallthrough.
#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
#define YY_USER_INIT initLoc(yylloc)
#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
@ -146,9 +149,9 @@ or { return OR_KW; }
try {
yylval->n = boost::lexical_cast<int64_t>(yytext);
} catch (const boost::bad_lexical_cast &) {
throw ParseError({
.msg = hintfmt("invalid integer '%1%'", yytext),
.errPos = state->positions[CUR_POS],
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid integer '%1%'", yytext),
.pos = state->positions[CUR_POS],
});
}
return INT_LIT;
@ -156,9 +159,9 @@ or { return OR_KW; }
{FLOAT} { errno = 0;
yylval->nf = strtod(yytext, 0);
if (errno != 0)
throw ParseError({
.msg = hintfmt("invalid float '%1%'", yytext),
.errPos = state->positions[CUR_POS],
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid float '%1%'", yytext),
.pos = state->positions[CUR_POS],
});
return FLOAT_LIT;
}
@ -285,9 +288,9 @@ or { return OR_KW; }
<INPATH_SLASH>{ANY} |
<INPATH_SLASH><<EOF>> {
throw ParseError({
.msg = hintfmt("path has a trailing slash"),
.errPos = state->positions[CUR_POS],
throw ParseError(ErrorInfo{
.msg = HintFmt("path has a trailing slash"),
.pos = state->positions[CUR_POS],
});
}

130
src/libexpr/nixexpr.cc
View file

@ -70,10 +70,8 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
str << ") ? " << showAttrPath(symbols, attrPath) << ")";
}
void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
{
if (recursive) str << "rec ";
str << "{ ";
typedef const decltype(attrs)::value_type * Attr;
std::vector<Attr> sorted;
for (auto & i : attrs) sorted.push_back(&i);
@ -81,10 +79,37 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
std::string_view sa = symbols[a->first], sb = symbols[b->first];
return sa < sb;
});
std::vector<Symbol> inherits;
std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
for (auto & i : sorted) {
if (i->second.inherited)
str << "inherit " << symbols[i->first] << " " << "; ";
else {
switch (i->second.kind) {
case AttrDef::Kind::Plain:
break;
case AttrDef::Kind::Inherited:
inherits.push_back(i->first);
break;
case AttrDef::Kind::InheritedFrom: {
auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
inheritsFrom[&from].push_back(i->first);
break;
}
}
}
if (!inherits.empty()) {
str << "inherit";
for (auto sym : inherits) str << " " << symbols[sym];
str << "; ";
}
for (const auto & [from, syms] : inheritsFrom) {
str << "inherit (";
(*inheritFromExprs)[from->displ]->show(symbols, str);
str << ")";
for (auto sym : syms) str << " " << symbols[sym];
str << "; ";
}
for (auto & i : sorted) {
if (i->second.kind == AttrDef::Kind::Plain) {
str << symbols[i->first] << " = ";
i->second.e->show(symbols, str);
str << "; ";
@ -97,6 +122,13 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
i.valueExpr->show(symbols, str);
str << "; ";
}
}
void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
{
if (recursive) str << "rec ";
str << "{ ";
showBindings(symbols, str);
str << "}";
}
@ -152,15 +184,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
void ExprLet::show(const SymbolTable & symbols, std::ostream & str) const
{
str << "(let ";
for (auto & i : attrs->attrs)
if (i.second.inherited) {
str << "inherit " << symbols[i.first] << "; ";
}
else {
str << symbols[i.first] << " = ";
i.second.e->show(symbols, str);
str << "; ";
}
attrs->showBindings(symbols, str);
str << "in ";
body->show(symbols, str);
str << ")";
@ -296,15 +320,21 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
enclosing `with'. If there is no `with', then we can issue an
"undefined variable" error now. */
if (withLevel == -1)
throw UndefinedVarError({
.msg = hintfmt("undefined variable '%1%'", es.symbols[name]),
.errPos = es.positions[pos]
});
es.error<UndefinedVarError>(
"undefined variable '%1%'",
es.symbols[name]
).atPos(pos).debugThrow();
for (auto * e = env.get(); e && !fromWith; e = e->up)
fromWith = e->isWith;
this->level = withLevel;
}
void ExprInheritFrom::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
}
void ExprSelect::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
if (es.debugRepl)
@ -328,22 +358,47 @@ void ExprOpHasAttr::bindVars(EvalState & es, const std::shared_ptr<const StaticE
i.expr->bindVars(es, env);
}
std::shared_ptr<const StaticEnv> ExprAttrs::bindInheritSources(
EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
if (!inheritFromExprs)
return nullptr;
// the inherit (from) source values are inserted into an env of its own, which
// does not introduce any variable names.
// analysis must see an empty env, or an env that contains only entries with
// otherwise unused names to not interfere with regular names. the parser
// has already filled all exprs that access this env with appropriate level
// and displacement, and nothing else is allowed to access it. ideally we'd
// not even *have* an expr that grabs anything from this env since it's fully
// invisible, but the evaluator does not allow for this yet.
auto inner = std::make_shared<StaticEnv>(nullptr, env.get(), 0);
for (auto from : *inheritFromExprs)
from->bindVars(es, env);
return inner;
}
void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
if (recursive) {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), recursive ? attrs.size() : 0);
auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs.size());
Displacement displ = 0;
for (auto & i : attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
Displacement displ = 0;
for (auto & i : attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
return newEnv;
}();
// No need to sort newEnv since attrs is in sorted order.
auto inheritFromEnv = bindInheritSources(es, newEnv);
for (auto & i : attrs)
i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(es, newEnv);
@ -351,8 +406,10 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
}
}
else {
auto inheritFromEnv = bindInheritSources(es, env);
for (auto & i : attrs)
i.second.e->bindVars(es, env);
i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(es, env);
@ -409,19 +466,23 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
Displacement displ = 0;
for (auto & i : attrs->attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
Displacement displ = 0;
for (auto & i : attrs->attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
return newEnv;
}();
// No need to sort newEnv since attrs->attrs is in sorted order.
auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
for (auto & i : attrs->attrs)
i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, newEnv));
body->bindVars(es, newEnv);
}
@ -447,9 +508,6 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
break;
}
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
attrs->bindVars(es, env);
auto newEnv = std::make_shared<StaticEnv>(this, env.get());
body->bindVars(es, newEnv);

155
src/libexpr/nixexpr.hh
View file

@ -9,110 +9,13 @@
#include "error.hh"
#include "chunked-vector.hh"
#include "position.hh"
#include "eval-error.hh"
#include "pos-idx.hh"
#include "pos-table.hh"
namespace nix {
MakeError(EvalError, Error);
MakeError(ParseError, Error);
MakeError(AssertionError, EvalError);
MakeError(ThrownError, AssertionError);
MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError);
class InfiniteRecursionError : public EvalError
{
friend class EvalState;
public:
using EvalError::EvalError;
};
class PosIdx {
friend class PosTable;
private:
uint32_t id;
explicit PosIdx(uint32_t id): id(id) {}
public:
PosIdx() : id(0) {}
explicit operator bool() const { return id > 0; }
bool operator <(const PosIdx other) const { return id < other.id; }
bool operator ==(const PosIdx other) const { return id == other.id; }
bool operator !=(const PosIdx other) const { return id != other.id; }
};
class PosTable
{
public:
class Origin {
friend PosTable;
private:
// must always be invalid by default, add() replaces this with the actual value.
// subsequent add() calls use this index as a token to quickly check whether the
// current origins.back() can be reused or not.
mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
// Used for searching in PosTable::[].
explicit Origin(uint32_t idx): idx(idx), origin{std::monostate()} {}
public:
const Pos::Origin origin;
Origin(Pos::Origin origin): origin(origin) {}
};
struct Offset {
uint32_t line, column;
};
private:
std::vector<Origin> origins;
ChunkedVector<Offset, 8192> offsets;
public:
PosTable(): offsets(1024)
{
origins.reserve(1024);
}
PosIdx add(const Origin & origin, uint32_t line, uint32_t column)
{
const auto idx = offsets.add({line, column}).second;
if (origins.empty() || origins.back().idx != origin.idx) {
origin.idx = idx;
origins.push_back(origin);
}
return PosIdx(idx + 1);
}
Pos operator[](PosIdx p) const
{
if (p.id == 0 || p.id > offsets.size())
return {};
const auto idx = p.id - 1;
/* we want the last key <= idx, so we'll take prev(first key > idx).
this is guaranteed to never rewind origin.begin because the first
key is always 0. */
const auto pastOrigin = std::upper_bound(
origins.begin(), origins.end(), Origin(idx),
[] (const auto & a, const auto & b) { return a.idx < b.idx; });
const auto origin = *std::prev(pastOrigin);
const auto offset = offsets[idx];
return {offset.line, offset.column, origin.origin};
}
};
inline PosIdx noPos = {};
struct Env;
struct Value;
class EvalState;
@ -232,6 +135,23 @@ struct ExprVar : Expr
COMMON_METHODS
};
/**
* A pseudo-expression for the purpose of evaluating the `from` expression in `inherit (from)` syntax.
* Unlike normal variable references, the displacement is set during parsing, and always refers to
* `ExprAttrs::inheritFromExprs` (by itself or in `ExprLet`), whose values are put into their own `Env`.
*/
struct ExprInheritFrom : ExprVar
{
ExprInheritFrom(PosIdx pos, Displacement displ): ExprVar(pos, {})
{
this->level = 0;
this->displ = displ;
this->fromWith = nullptr;
}
void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
};
struct ExprSelect : Expr
{
PosIdx pos;
@ -257,16 +177,40 @@ struct ExprAttrs : Expr
bool recursive;
PosIdx pos;
struct AttrDef {
bool inherited;
enum class Kind {
/** `attr = expr;` */
Plain,
/** `inherit attr1 attrn;` */
Inherited,
/** `inherit (expr) attr1 attrn;` */
InheritedFrom,
};
Kind kind;
Expr * e;
PosIdx pos;
Displacement displ; // displacement
AttrDef(Expr * e, const PosIdx & pos, bool inherited=false)
: inherited(inherited), e(e), pos(pos) { };
AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain)
: kind(kind), e(e), pos(pos) { };
AttrDef() { };
template<typename T>
const T & chooseByKind(const T & plain, const T & inherited, const T & inheritedFrom) const
{
switch (kind) {
case Kind::Plain:
return plain;
case Kind::Inherited:
return inherited;
default:
case Kind::InheritedFrom:
return inheritedFrom;
}
}
};
typedef std::map<Symbol, AttrDef> AttrDefs;
AttrDefs attrs;
std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
struct DynamicAttrDef {
Expr * nameExpr, * valueExpr;
PosIdx pos;
@ -279,6 +223,11 @@ struct ExprAttrs : Expr
ExprAttrs() : recursive(false) { };
PosIdx getPos() const override { return pos; }
COMMON_METHODS
std::shared_ptr<const StaticEnv> bindInheritSources(
EvalState & es, const std::shared_ptr<const StaticEnv> & env);
Env * buildInheritFromEnv(EvalState & state, Env & up);
void showBindings(const SymbolTable & symbols, std::ostream & str) const;
};
struct ExprList : Expr

29
src/libexpr/parser-state.hh
View file

@ -64,17 +64,17 @@ struct ParserState
inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
{
throw ParseError({
.msg = hintfmt("attribute '%1%' already defined at %2%",
.msg = HintFmt("attribute '%1%' already defined at %2%",
showAttrPath(symbols, attrPath), positions[prevPos]),
.errPos = positions[pos]
.pos = positions[pos]
});
}
inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
{
throw ParseError({
.msg = hintfmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]),
.errPos = positions[pos]
.msg = HintFmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]),
.pos = positions[pos]
});
}
@ -89,7 +89,7 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
if (i->symbol) {
ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
if (j != attrs->attrs.end()) {
if (!j->second.inherited) {
if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
attrs = attrs2;
@ -118,13 +118,24 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
auto ae = dynamic_cast<ExprAttrs *>(e);
auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
if (jAttrs && ae) {
if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
for (auto & ad : ae->attrs) {
auto j2 = jAttrs->attrs.find(ad.first);
if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
dupAttr(ad.first, j2->second.pos, ad.second.pos);
jAttrs->attrs.emplace(ad.first, ad.second);
if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
from.displ += jAttrs->inheritFromExprs->size();
}
}
jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
if (ae->inheritFromExprs) {
jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
}
} else {
dupAttr(attrPath, pos, j->second.pos);
}
@ -154,14 +165,14 @@ inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Sym
}
if (duplicate)
throw ParseError({
.msg = hintfmt("duplicate formal function argument '%1%'", symbols[duplicate->first]),
.errPos = positions[duplicate->second]
.msg = HintFmt("duplicate formal function argument '%1%'", symbols[duplicate->first]),
.pos = positions[duplicate->second]
});
if (arg && formals->has(arg))
throw ParseError({
.msg = hintfmt("duplicate formal function argument '%1%'", symbols[arg]),
.errPos = positions[pos]
.msg = HintFmt("duplicate formal function argument '%1%'", symbols[arg]),
.pos = positions[pos]
});
return formals;

32
src/libexpr/parser.y
View file

@ -65,8 +65,8 @@ using namespace nix;
void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
{
throw ParseError({
.msg = hintfmt(error),
.errPos = state->positions[state->at(*loc)]
.msg = HintFmt(error),
.pos = state->positions[state->at(*loc)]
});
}
@ -154,8 +154,8 @@ expr_function
| LET binds IN_KW expr_function
{ if (!$2->dynamicAttrs.empty())
throw ParseError({
.msg = hintfmt("dynamic attributes not allowed in let"),
.errPos = state->positions[CUR_POS]
.msg = HintFmt("dynamic attributes not allowed in let"),
.pos = state->positions[CUR_POS]
});
$$ = new ExprLet($2, $4);
}
@ -244,8 +244,8 @@ expr_simple
static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
if (noURLLiterals)
throw ParseError({
.msg = hintfmt("URL literals are disabled"),
.errPos = state->positions[CUR_POS]
.msg = HintFmt("URL literals are disabled"),
.pos = state->positions[CUR_POS]
});
$$ = new ExprString(std::string($1));
}
@ -313,17 +313,27 @@ binds
if ($$->attrs.find(i.symbol) != $$->attrs.end())
state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos);
auto pos = state->at(@3);
$$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
$$->attrs.emplace(
i.symbol,
ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, ExprAttrs::AttrDef::Kind::Inherited));
}
delete $3;
}
| binds INHERIT '(' expr ')' attrs ';'
{ $$ = $1;
/* !!! Should ensure sharing of the expression in $4. */
if (!$$->inheritFromExprs)
$$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
$$->inheritFromExprs->push_back($4);
auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
for (auto & i : *$6) {
if ($$->attrs.find(i.symbol) != $$->attrs.end())
state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos);
$$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->at(@6)));
$$->attrs.emplace(
i.symbol,
ExprAttrs::AttrDef(
new ExprSelect(CUR_POS, from, i.symbol),
state->at(@6),
ExprAttrs::AttrDef::Kind::InheritedFrom));
}
delete $6;
}
@ -340,8 +350,8 @@ attrs
delete str;
} else
throw ParseError({
.msg = hintfmt("dynamic attributes not allowed in inherit"),
.errPos = state->positions[state->at(@2)]
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
}
| { $$ = new AttrPath; }

6
src/libexpr/paths.cc
View file

@ -1,5 +1,4 @@
#include "eval.hh"
#include "fs-input-accessor.hh"
namespace nix {
@ -8,4 +7,9 @@ SourcePath EvalState::rootPath(CanonPath path)
return {rootFS, std::move(path)};
}
SourcePath EvalState::rootPath(PathView path)
{
return {rootFS, CanonPath(absPath(path))};
}
}

48
src/libexpr/pos-idx.hh Normal file
View file

@ -0,0 +1,48 @@
#pragma once
#include <cinttypes>
namespace nix {
class PosIdx
{
friend class PosTable;
private:
uint32_t id;
explicit PosIdx(uint32_t id)
: id(id)
{
}
public:
PosIdx()
: id(0)
{
}
explicit operator bool() const
{
return id > 0;
}
bool operator<(const PosIdx other) const
{
return id < other.id;
}
bool operator==(const PosIdx other) const
{
return id == other.id;
}
bool operator!=(const PosIdx other) const
{
return id != other.id;
}
};
inline PosIdx noPos = {};
}

83
src/libexpr/pos-table.hh Normal file
View file

@ -0,0 +1,83 @@
#pragma once
#include <cinttypes>
#include <numeric>
#include <vector>
#include "chunked-vector.hh"
#include "pos-idx.hh"
#include "position.hh"
namespace nix {
class PosTable
{
public:
class Origin
{
friend PosTable;
private:
// must always be invalid by default, add() replaces this with the actual value.
// subsequent add() calls use this index as a token to quickly check whether the
// current origins.back() can be reused or not.
mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
// Used for searching in PosTable::[].
explicit Origin(uint32_t idx)
: idx(idx)
, origin{std::monostate()}
{
}
public:
const Pos::Origin origin;
Origin(Pos::Origin origin)
: origin(origin)
{
}
};
struct Offset
{
uint32_t line, column;
};
private:
std::vector<Origin> origins;
ChunkedVector<Offset, 8192> offsets;
public:
PosTable()
: offsets(1024)
{
origins.reserve(1024);
}
PosIdx add(const Origin & origin, uint32_t line, uint32_t column)
{
const auto idx = offsets.add({line, column}).second;
if (origins.empty() || origins.back().idx != origin.idx) {
origin.idx = idx;
origins.push_back(origin);
}
return PosIdx(idx + 1);
}
Pos operator[](PosIdx p) const
{
if (p.id == 0 || p.id > offsets.size())
return {};
const auto idx = p.id - 1;
/* we want the last key <= idx, so we'll take prev(first key > idx).
this is guaranteed to never rewind origin.begin because the first
key is always 0. */
const auto pastOrigin = std::upper_bound(
origins.begin(), origins.end(), Origin(idx), [](const auto & a, const auto & b) { return a.idx < b.idx; });
const auto origin = *std::prev(pastOrigin);
const auto offset = offsets[idx];
return {offset.line, offset.column, origin.origin};
}
};
}

403
src/libexpr/primops.cc
View file

@ -39,10 +39,6 @@ namespace nix {
* Miscellaneous
*************************************************************/
InvalidPathError::InvalidPathError(const Path & path) :
EvalError("path '%s' is not valid", path), path(path) {}
StringMap EvalState::realiseContext(const NixStringContext & context)
{
std::vector<DerivedPath::Built> drvs;
@ -51,7 +47,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
for (auto & c : context) {
auto ensureValid = [&](const StorePath & p) {
if (!store->isValidPath(p))
debugThrowLastTrace(InvalidPathError(store->printStorePath(p)));
error<InvalidPathError>(store->printStorePath(p)).debugThrow();
};
std::visit(overloaded {
[&](const NixStringContextElem::Built & b) {
@ -78,9 +74,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
if (drvs.empty()) return {};
if (!evalSettings.enableImportFromDerivation)
debugThrowLastTrace(Error(
error<EvalError>(
"cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
drvs.begin()->to_string(*store)));
drvs.begin()->to_string(*store)
).debugThrow();
/* Build/substitute the context. */
std::vector<DerivedPath> buildReqs;
@ -118,7 +115,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
return res;
}
static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bool resolveSymlinks = true)
static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full)
{
NixStringContext context;
@ -130,7 +127,7 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bo
auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
path = {path.accessor, CanonPath(realPath)};
}
return resolveSymlinks ? path.resolveSymlinks() : path;
return resolveSymlinks ? path.resolveSymlinks(*resolveSymlinks) : path;
} catch (Error & e) {
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
throw;
@ -170,7 +167,7 @@ static void mkOutputString(
argument. */
static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
{
auto path = realisePath(state, pos, vPath, false);
auto path = realisePath(state, pos, vPath, std::nullopt);
auto path2 = path.path.abs();
// FIXME
@ -340,16 +337,16 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu
void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL);
if (!handle)
state.debugThrowLastTrace(EvalError("could not open '%1%': %2%", path, dlerror()));
state.error<EvalError>("could not open '%1%': %2%", path, dlerror()).debugThrow();
dlerror();
ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str());
if(!func) {
char *message = dlerror();
if (message)
state.debugThrowLastTrace(EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message));
state.error<EvalError>("could not load symbol '%1%' from '%2%': %3%", sym, path, message).debugThrow();
else
state.debugThrowLastTrace(EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path));
state.error<EvalError>("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path).debugThrow();
}
(func)(state, v);
@ -365,7 +362,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
auto elems = args[0]->listElems();
auto count = args[0]->listSize();
if (count == 0)
state.error("at least one argument to 'exec' required").atPos(pos).debugThrow<EvalError>();
state.error<EvalError>("at least one argument to 'exec' required").atPos(pos).debugThrow();
NixStringContext context;
auto program = state.coerceToString(pos, *elems[0], context,
"while evaluating the first element of the argument passed to builtins.exec",
@ -380,7 +377,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
try {
auto _ = state.realiseContext(context); // FIXME: Handle CA derivations
} catch (InvalidPathError & e) {
state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow<EvalError>();
state.error<EvalError>("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow();
}
auto output = runProgram(program, true, commandArgs);
@ -582,7 +579,7 @@ struct CompareValues
if (v1->type() == nInt && v2->type() == nFloat)
return v1->integer < v2->fpoint;
if (v1->type() != v2->type())
state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow<EvalError>();
state.error<EvalError>("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow();
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
@ -610,7 +607,7 @@ struct CompareValues
}
}
default:
state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow<EvalError>();
state.error<EvalError>("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow();
#pragma GCC diagnostic pop
}
} catch (Error & e) {
@ -637,7 +634,7 @@ static Bindings::iterator getAttr(
{
Bindings::iterator value = attrSet->find(attrSym);
if (value == attrSet->end()) {
state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow<TypeError>();
state.error<TypeError>("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow();
}
return value;
}
@ -708,38 +705,53 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
.args = {"attrset"},
.arity = 1,
.doc = R"(
Take an *attrset* with values named `startSet` and `operator` in order to
return a *list of attrsets* by starting with the `startSet` and recursively
applying the `operator` function to each `item`. The *attrsets* in the
`startSet` and the *attrsets* produced by `operator` must contain a value
named `key` which is comparable. The result is produced by calling `operator`
for each `item` with a value for `key` that has not been called yet including
newly produced `item`s. The function terminates when no new `item`s are
produced. The resulting *list of attrsets* contains only *attrsets* with a
unique key. For example,
`builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function.
```
builtins.genericClosure {
startSet = [ {key = 5;} ];
operator = item: [{
key = if (item.key / 2 ) * 2 == item.key
then item.key / 2
else 3 * item.key + 1;
}];
}
```
evaluates to
```
[ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
```
It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets:
- `startSet`:
The initial list of attribute sets.
- `operator`:
A function that takes an attribute set and returns a list of attribute sets.
It defines how each item in the current set is processed and expanded into more items.
Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison.
The value of `key` can be one of the following types:
`key` can be one of the following types:
- [Number](@docroot@/language/values.md#type-number)
- [Boolean](@docroot@/language/values.md#type-boolean)
- [String](@docroot@/language/values.md#type-string)
- [Path](@docroot@/language/values.md#type-path)
- [List](@docroot@/language/values.md#list)
The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added.
Items are compared by their `key` attribute.
Common usages are:
- Generating unique collections of items, such as dependency graphs.
- Traversing through structures that may contain cycles or loops.
- Processing data structures with complex internal relationships.
> **Example**
>
> ```nix
> builtins.genericClosure {
> startSet = [ {key = 5;} ];
> operator = item: [{
> key = if (item.key / 2 ) * 2 == item.key
> then item.key / 2
> else 3 * item.key + 1;
> }];
> }
> ```
>
> evaluates to
>
> ```nix
> [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
> ```
)",
.fun = prim_genericClosure,
});
@ -757,21 +769,12 @@ static RegisterPrimOp primop_break({
if (state.debugRepl && !state.debugTraces.empty()) {
auto error = Error(ErrorInfo {
.level = lvlInfo,
.msg = hintfmt("breakpoint reached"),
.errPos = state.positions[pos],
.msg = HintFmt("breakpoint reached"),
.pos = state.positions[pos],
});
auto & dt = state.debugTraces.front();
state.runDebugRepl(&error, dt.env, dt.expr);
if (state.debugQuit) {
// If the user elects to quit the repl, throw an exception.
throw Error(ErrorInfo{
.level = lvlInfo,
.msg = hintfmt("quit the debugger"),
.errPos = nullptr,
});
}
}
// Return the value we were passed.
@ -790,7 +793,7 @@ static RegisterPrimOp primop_abort({
NixStringContext context;
auto s = state.coerceToString(pos, *args[0], context,
"while evaluating the error message passed to builtins.abort").toOwned();
state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s));
state.error<Abort>("evaluation aborted with the following error message: '%1%'", s).debugThrow();
}
});
@ -809,7 +812,7 @@ static RegisterPrimOp primop_throw({
NixStringContext context;
auto s = state.coerceToString(pos, *args[0], context,
"while evaluating the error message passed to builtin.throw").toOwned();
state.debugThrowLastTrace(ThrownError(s));
state.error<ThrownError>(s).debugThrow();
}
});
@ -823,7 +826,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
auto message = state.coerceToString(pos, *args[0], context,
"while evaluating the error message passed to builtins.addErrorContext",
false, false).toOwned();
e.addTrace(nullptr, hintfmt(message), true);
e.addTrace(nullptr, HintFmt(message));
throw;
}
}
@ -882,7 +885,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
/* increment state.trylevel, and decrement it when this function returns. */
MaintainCount trylevel(state.trylevel);
void (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
ReplExitStatus (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry)
{
/* to prevent starting the repl from exceptions withing a tryEval, null it. */
@ -998,6 +1001,10 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu
printError("trace: %1%", args[0]->string_view());
else
printError("trace: %1%", ValuePrinter(state, *args[0]));
if (evalSettings.builtinsTraceDebugger && state.debugRepl && !state.debugTraces.empty()) {
const DebugTrace & last = state.debugTraces.front();
state.runDebugRepl(nullptr, last.env, last.expr);
}
state.forceValue(*args[1], pos);
v = *args[1];
}
@ -1009,6 +1016,12 @@ static RegisterPrimOp primop_trace({
Evaluate *e1* and print its abstract syntax representation on
standard error. Then return *e2*. This function is useful for
debugging.
If the
[`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace)
option is set to `true` and the `--debugger` flag is given, the
interactive debugger will be started when `trace` is called (like
[`break`](@docroot@/language/builtins.md#builtins-break)).
)",
.fun = prim_trace,
});
@ -1074,10 +1087,10 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
* often results from the composition of several functions
* (derivationStrict, derivation, mkDerivation, mkPythonModule, etc.)
*/
e.addTrace(nullptr, hintfmt(
e.addTrace(nullptr, HintFmt(
"while evaluating derivation '%s'\n"
" whose name attribute is located at %s",
drvName, pos), true);
drvName, pos));
throw;
}
}
@ -1088,9 +1101,10 @@ drvName, Bindings * attrs, Value & v)
/* Check whether attributes should be passed as a JSON file. */
using nlohmann::json;
std::optional<json> jsonObject;
auto pos = v.determinePos(noPos);
auto attr = attrs->find(state.sStructuredAttrs);
if (attr != attrs->end() &&
state.forceBool(*attr->value, noPos,
state.forceBool(*attr->value, pos,
"while evaluating the `__structuredAttrs` "
"attribute passed to builtins.derivationStrict"))
jsonObject = json::object();
@ -1099,7 +1113,7 @@ drvName, Bindings * attrs, Value & v)
bool ignoreNulls = false;
attr = attrs->find(state.sIgnoreNulls);
if (attr != attrs->end())
ignoreNulls = state.forceBool(*attr->value, noPos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict");
ignoreNulls = state.forceBool(*attr->value, pos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict");
/* Build the derivation expression by processing the attributes. */
Derivation drv;
@ -1124,41 +1138,40 @@ drvName, Bindings * attrs, Value & v)
auto handleHashMode = [&](const std::string_view s) {
if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
else if (s == "text") {
else if (s == "git") {
experimentalFeatureSettings.require(Xp::GitHashing);
ingestionMethod = FileIngestionMethod::Git;
} else if (s == "text") {
experimentalFeatureSettings.require(Xp::DynamicDerivations);
ingestionMethod = TextIngestionMethod {};
} else
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
.errPos = state.positions[noPos]
}));
state.error<EvalError>(
"invalid value '%s' for 'outputHashMode' attribute", s
).atPos(v).debugThrow();
};
auto handleOutputs = [&](const Strings & ss) {
outputs.clear();
for (auto & j : ss) {
if (outputs.find(j) != outputs.end())
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("duplicate derivation output '%1%'", j),
.errPos = state.positions[noPos]
}));
state.error<EvalError>("duplicate derivation output '%1%'", j)
.atPos(v)
.debugThrow();
/* !!! Check whether j is a valid attribute
name. */
/* Derivations cannot be named ‘drv’, because
then we'd have an attribute ‘drvPath’ in
the resulting set. */
if (j == "drv")
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("invalid derivation output name 'drv'" ),
.errPos = state.positions[noPos]
}));
state.error<EvalError>("invalid derivation output name 'drv'")
.atPos(v)
.debugThrow();
outputs.insert(j);
}
if (outputs.empty())
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("derivation cannot have an empty set of outputs"),
.errPos = state.positions[noPos]
}));
state.error<EvalError>("derivation cannot have an empty set of outputs")
.atPos(v)
.debugThrow();
};
try {
@ -1167,16 +1180,16 @@ drvName, Bindings * attrs, Value & v)
const std::string_view context_below("");
if (ignoreNulls) {
state.forceValue(*i->value, noPos);
state.forceValue(*i->value, pos);
if (i->value->type() == nNull) continue;
}
if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) {
if (i->name == state.sContentAddressed && state.forceBool(*i->value, pos, context_below)) {
contentAddressed = true;
experimentalFeatureSettings.require(Xp::CaDerivations);
}
else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) {
else if (i->name == state.sImpure && state.forceBool(*i->value, pos, context_below)) {
isImpure = true;
experimentalFeatureSettings.require(Xp::ImpureDerivations);
}
@ -1184,9 +1197,9 @@ drvName, Bindings * attrs, Value & v)
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
else if (i->name == state.sArgs) {
state.forceList(*i->value, noPos, context_below);
state.forceList(*i->value, pos, context_below);
for (auto elem : i->value->listItems()) {
auto s = state.coerceToString(noPos, *elem, context,
auto s = state.coerceToString(pos, *elem, context,
"while evaluating an element of the argument list",
true).toOwned();
drv.args.push_back(s);
@ -1201,29 +1214,29 @@ drvName, Bindings * attrs, Value & v)
if (i->name == state.sStructuredAttrs) continue;
(*jsonObject)[key] = printValueAsJSON(state, true, *i->value, noPos, context);
(*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context);
if (i->name == state.sBuilder)
drv.builder = state.forceString(*i->value, context, noPos, context_below);
drv.builder = state.forceString(*i->value, context, pos, context_below);
else if (i->name == state.sSystem)
drv.platform = state.forceStringNoCtx(*i->value, noPos, context_below);
drv.platform = state.forceStringNoCtx(*i->value, pos, context_below);
else if (i->name == state.sOutputHash)
outputHash = state.forceStringNoCtx(*i->value, noPos, context_below);
outputHash = state.forceStringNoCtx(*i->value, pos, context_below);
else if (i->name == state.sOutputHashAlgo)
outputHashAlgo = state.forceStringNoCtx(*i->value, noPos, context_below);
outputHashAlgo = state.forceStringNoCtx(*i->value, pos, context_below);
else if (i->name == state.sOutputHashMode)
handleHashMode(state.forceStringNoCtx(*i->value, noPos, context_below));
handleHashMode(state.forceStringNoCtx(*i->value, pos, context_below));
else if (i->name == state.sOutputs) {
/* Require ‘outputs’ to be a list of strings. */
state.forceList(*i->value, noPos, context_below);
state.forceList(*i->value, pos, context_below);
Strings ss;
for (auto elem : i->value->listItems())
ss.emplace_back(state.forceStringNoCtx(*elem, noPos, context_below));
ss.emplace_back(state.forceStringNoCtx(*elem, pos, context_below));
handleOutputs(ss);
}
} else {
auto s = state.coerceToString(noPos, *i->value, context, context_below, true).toOwned();
auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned();
drv.env.emplace(key, s);
if (i->name == state.sBuilder) drv.builder = std::move(s);
else if (i->name == state.sSystem) drv.platform = std::move(s);
@ -1238,8 +1251,7 @@ drvName, Bindings * attrs, Value & v)
} catch (Error & e) {
e.addTrace(state.positions[i->pos],
hintfmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName),
true);
HintFmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName));
throw;
}
}
@ -1281,16 +1293,14 @@ drvName, Bindings * attrs, Value & v)
/* Do we have all required attributes? */
if (drv.builder == "")
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("required attribute 'builder' missing"),
.errPos = state.positions[noPos]
}));
state.error<EvalError>("required attribute 'builder' missing")
.atPos(v)
.debugThrow();
if (drv.platform == "")
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("required attribute 'system' missing"),
.errPos = state.positions[noPos]
}));
state.error<EvalError>("required attribute 'system' missing")
.atPos(v)
.debugThrow();
/* Check whether the derivation name is valid. */
if (isDerivation(drvName) &&
@ -1298,10 +1308,10 @@ drvName, Bindings * attrs, Value & v)
outputs.size() == 1 &&
*(outputs.begin()) == "out"))
{
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension),
.errPos = state.positions[noPos]
}));
state.error<EvalError>(
"derivation names are allowed to end in '%s' only if they produce a single derivation file",
drvExtension
).atPos(v).debugThrow();
}
if (outputHash) {
@ -1310,10 +1320,9 @@ drvName, Bindings * attrs, Value & v)
Ignore `__contentAddressed` because fixed output derivations are
already content addressed. */
if (outputs.size() != 1 || *(outputs.begin()) != "out")
state.debugThrowLastTrace(Error({
.msg = hintfmt("multiple outputs are not supported in fixed-output derivations"),
.errPos = state.positions[noPos]
}));
state.error<EvalError>(
"multiple outputs are not supported in fixed-output derivations"
).atPos(v).debugThrow();
auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo));
@ -1332,10 +1341,8 @@ drvName, Bindings * attrs, Value & v)
else if (contentAddressed || isImpure) {
if (contentAddressed && isImpure)
throw EvalError({
.msg = hintfmt("derivation cannot be both content-addressed and impure"),
.errPos = state.positions[noPos]
});
state.error<EvalError>("derivation cannot be both content-addressed and impure")
.atPos(v).debugThrow();
auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
@ -1376,10 +1383,10 @@ drvName, Bindings * attrs, Value & v)
for (auto & i : outputs) {
auto h = get(hashModulo.hashes, i);
if (!h)
throw AssertionError({
.msg = hintfmt("derivation produced no hash for output '%s'", i),
.errPos = state.positions[noPos],
});
state.error<AssertionError>(
"derivation produced no hash for output '%s'",
i
).atPos(v).debugThrow();
auto outPath = state.store->makeOutputPath(i, *h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(
@ -1485,10 +1492,10 @@ static RegisterPrimOp primop_toPath({
static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
if (evalSettings.pureEval)
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"'%s' is not allowed in pure evaluation mode",
"builtins.storePath"
).atPos(pos).debugThrow();
NixStringContext context;
auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path;
@ -1498,10 +1505,8 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
if (!state.store->isStorePath(path.abs()))
path = CanonPath(canonPath(path.abs(), true));
if (!state.store->isInStore(path.abs()))
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("path '%1%' is not in the Nix store", path),
.errPos = state.positions[pos]
}));
state.error<EvalError>("path '%1%' is not in the Nix store", path)
.atPos(pos).debugThrow();
auto path2 = state.store->toStorePath(path.abs()).first;
if (!settings.readOnlyMode)
state.store->ensurePath(path2);
@ -1534,13 +1539,16 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
try {
auto & arg = *args[0];
auto path = realisePath(state, pos, arg);
/* SourcePath doesn't know about trailing slash. */
state.forceValue(arg, pos);
auto mustBeDir = arg.type() == nString
&& (arg.string_view().ends_with("/")
|| arg.string_view().ends_with("/."));
auto symlinkResolution =
mustBeDir ? SymlinkResolution::Full : SymlinkResolution::Ancestors;
auto path = realisePath(state, pos, arg, symlinkResolution);
auto st = path.maybeLstat();
auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
v.mkBool(exists);
@ -1616,7 +1624,10 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V
auto path = realisePath(state, pos, *args[0]);
auto s = path.readFile();
if (s.find((char) 0) != std::string::npos)
state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path));
state.error<EvalError>(
"the contents of the file '%1%' cannot be represented as a Nix string",
path
).atPos(pos).debugThrow();
StorePathSet refs;
if (state.store->isInStore(path.path.abs())) {
try {
@ -1673,10 +1684,11 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
auto rewrites = state.realiseContext(context);
path = rewriteStrings(path, rewrites);
} catch (InvalidPathError & e) {
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"cannot find '%1%', since path '%2%' is not valid",
path,
e.path
).atPos(pos).debugThrow();
}
searchPath.elements.emplace_back(SearchPath::Elem {
@ -1745,10 +1757,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V
auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
if (!ha)
state.debugThrowLastTrace(Error({
.msg = hintfmt("unknown hash algo '%1%'", algo),
.errPos = state.positions[pos]
}));
state.error<EvalError>("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow();
auto path = realisePath(state, pos, *args[1]);
@ -1777,7 +1786,7 @@ static std::string_view fileTypeToString(InputAccessor::Type type)
static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
auto path = realisePath(state, pos, *args[0], false);
auto path = realisePath(state, pos, *args[0], std::nullopt);
/* Retrieve the directory entry type and stringize it. */
v.mkString(fileTypeToString(path.lstat().type));
}
@ -1816,7 +1825,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va
// detailed node info quickly in this case we produce a thunk to
// query the file type lazily.
auto epath = state.allocValue();
epath->mkPath(path + name);
epath->mkPath(path / name);
if (!readFileType)
readFileType = &state.getBuiltin("readFileType");
attr.mkApp(readFileType, epath);
@ -2068,13 +2077,12 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
if (auto p = std::get_if<NixStringContextElem::Opaque>(&c.raw))
refs.insert(p->path);
else
state.debugThrowLastTrace(EvalError({
.msg = hintfmt(
"in 'toFile': the file named '%1%' must not contain a reference "
"to a derivation but contains (%2%)",
name, c.to_string()),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"files created by %1% may not reference derivations, but %2% references %3%",
"builtins.toFile",
name,
c.to_string()
).atPos(pos).debugThrow();
}
auto storePath = settings.readOnlyMode
@ -2084,7 +2092,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
})
: ({
StringSource s { contents };
state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
});
/* Note: we don't need to add `context' to the context of the
@ -2241,9 +2249,19 @@ static void addPath(
});
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = fetchToStore(*state.store, path.resolveSymlinks(), name, method, filter.get(), state.repair);
auto dstPath = fetchToStore(
*state.store,
path.resolveSymlinks(),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
name,
method,
filter.get(),
state.repair);
if (expectedHash && expectedStorePath != dstPath)
state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
state.error<EvalError>(
"store path mismatch in (possibly filtered) path added from '%s'",
path
).atPos(pos).debugThrow();
state.allowAndSetStorePathString(dstPath, v);
} else
state.allowAndSetStorePathString(*expectedStorePath, v);
@ -2343,16 +2361,15 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
else
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]),
.errPos = state.positions[attr.pos]
}));
state.error<EvalError>(
"unsupported argument '%1%' to 'addPath'",
state.symbols[attr.name]
).atPos(attr.pos).debugThrow();
}
if (!path)
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("missing required 'path' attribute in the first argument to builtins.path"),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"missing required 'path' attribute in the first argument to builtins.path"
).atPos(pos).debugThrow();
if (name.empty())
name = path->baseName();
@ -2770,10 +2787,7 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * arg
return;
}
if (!args[0]->isLambda())
state.debugThrowLastTrace(TypeError({
.msg = hintfmt("'functionArgs' requires a function"),
.errPos = state.positions[pos]
}));
state.error<TypeError>("'functionArgs' requires a function").atPos(pos).debugThrow();
if (!args[0]->lambda.fun->hasFormals()) {
v.mkAttrs(&state.emptyBindings);
@ -2943,10 +2957,10 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val
{
state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt");
if (n < 0 || (unsigned int) n >= list.listSize())
state.debugThrowLastTrace(Error({
.msg = hintfmt("list index %1% is out of bounds", n),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"list index %1% is out of bounds",
n
).atPos(pos).debugThrow();
state.forceValue(*list.listElems()[n], pos);
v = *list.listElems()[n];
}
@ -2991,10 +3005,7 @@ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value
{
state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail");
if (args[0]->listSize() == 0)
state.debugThrowLastTrace(Error({
.msg = hintfmt("'tail' called on an empty list"),
.errPos = state.positions[pos]
}));
state.error<EvalError>("'tail' called on an empty list").atPos(pos).debugThrow();
state.mkList(v, args[0]->listSize() - 1);
for (unsigned int n = 0; n < v.listSize(); ++n)
@ -3251,7 +3262,7 @@ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Va
auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList");
if (len < 0)
state.error("cannot create list of size %1%", len).debugThrow<EvalError>();
state.error<EvalError>("cannot create list of size %1%", len).atPos(pos).debugThrow();
// More strict than striclty (!) necessary, but acceptable
// as evaluating map without accessing any values makes little sense.
@ -3568,10 +3579,7 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value
NixFloat f2 = state.forceFloat(*args[1], pos, "while evaluating the second operand of the division");
if (f2 == 0)
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("division by zero"),
.errPos = state.positions[pos]
}));
state.error<EvalError>("division by zero").atPos(pos).debugThrow();
if (args[0]->type() == nFloat || args[1]->type() == nFloat) {
v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first operand of the division") / f2);
@ -3580,10 +3588,7 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value
NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division");
/* Avoid division overflow as it might raise SIGFPE. */
if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("overflow in integer division"),
.errPos = state.positions[pos]
}));
state.error<EvalError>("overflow in integer division").atPos(pos).debugThrow();
v.mkInt(i1 / i2);
}
@ -3714,10 +3719,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args,
int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring");
if (start < 0)
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("negative start position in 'substring'"),
.errPos = state.positions[pos]
}));
state.error<EvalError>("negative start position in 'substring'").atPos(pos).debugThrow();
int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
@ -3782,10 +3784,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args,
auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
if (!ha)
state.debugThrowLastTrace(Error({
.msg = hintfmt("unknown hash algo '%1%'", algo),
.errPos = state.positions[pos]
}));
state.error<EvalError>("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow();
NixStringContext context; // discarded
auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
@ -3951,15 +3950,13 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
} catch (std::regex_error & e) {
if (e.code() == std::regex_constants::error_space) {
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
.errPos = state.positions[pos]
}));
state.error<EvalError>("memory limit exceeded by regular expression '%s'", re)
.atPos(pos)
.debugThrow();
} else
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("invalid regular expression '%s'", re),
.errPos = state.positions[pos]
}));
state.error<EvalError>("invalid regular expression '%s'", re)
.atPos(pos)
.debugThrow();
}
}
@ -4055,15 +4052,13 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
} catch (std::regex_error & e) {
if (e.code() == std::regex_constants::error_space) {
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
.errPos = state.positions[pos]
}));
state.error<EvalError>("memory limit exceeded by regular expression '%s'", re)
.atPos(pos)
.debugThrow();
} else
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("invalid regular expression '%s'", re),
.errPos = state.positions[pos]
}));
state.error<EvalError>("invalid regular expression '%s'", re)
.atPos(pos)
.debugThrow();
}
}
@ -4139,7 +4134,9 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.replaceStrings");
state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.replaceStrings");
if (args[0]->listSize() != args[1]->listSize())
state.error("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths").atPos(pos).debugThrow<EvalError>();
state.error<EvalError>(
"'from' and 'to' arguments passed to builtins.replaceStrings have different lengths"
).atPos(pos).debugThrow();
std::vector<std::string> from;
from.reserve(args[0]->listSize());

50
src/libexpr/primops/context.cc
View file

@ -98,30 +98,30 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
auto contextSize = context.size();
if (contextSize != 1) {
throw EvalError({
.msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize),
.errPos = state.positions[pos]
});
state.error<EvalError>(
"context of string '%s' must have exactly one element, but has %d",
*s,
contextSize
).atPos(pos).debugThrow();
}
NixStringContext context2 {
(NixStringContextElem { std::visit(overloaded {
[&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep {
if (!c.path.isDerivation()) {
throw EvalError({
.msg = hintfmt("path '%s' is not a derivation",
state.store->printStorePath(c.path)),
.errPos = state.positions[pos],
});
state.error<EvalError>(
"path '%s' is not a derivation",
state.store->printStorePath(c.path)
).atPos(pos).debugThrow();
}
return NixStringContextElem::DrvDeep {
.drvPath = c.path,
};
},
[&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep {
throw EvalError({
.msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output),
.errPos = state.positions[pos],
});
state.error<EvalError>(
"`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'",
c.output
).atPos(pos).debugThrow();
},
[&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep {
/* Reuse original item because we want this to be idempotent. */
@ -261,10 +261,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
for (auto & i : *args[1]->attrs) {
const auto & name = state.symbols[i.name];
if (!state.store->isStorePath(name))
throw EvalError({
.msg = hintfmt("context key '%s' is not a store path", name),
.errPos = state.positions[i.pos]
});
state.error<EvalError>(
"context key '%s' is not a store path",
name
).atPos(i.pos).debugThrow();
auto namePath = state.store->parseStorePath(name);
if (!settings.readOnlyMode)
state.store->ensurePath(namePath);
@ -281,10 +281,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, iter->pos, "while evaluating the `allOutputs` attribute of a string context")) {
if (!isDerivation(name)) {
throw EvalError({
.msg = hintfmt("tried to add all-outputs context of %s, which is not a derivation, to a string", name),
.errPos = state.positions[i.pos]
});
state.error<EvalError>(
"tried to add all-outputs context of %s, which is not a derivation, to a string",
name
).atPos(i.pos).debugThrow();
}
context.emplace(NixStringContextElem::DrvDeep {
.drvPath = namePath,
@ -296,10 +296,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
if (iter != i.value->attrs->end()) {
state.forceList(*iter->value, iter->pos, "while evaluating the `outputs` attribute of a string context");
if (iter->value->listSize() && !isDerivation(name)) {
throw EvalError({
.msg = hintfmt("tried to add derivation output context of %s, which is not a derivation, to a string", name),
.errPos = state.positions[i.pos]
});
state.error<EvalError>(
"tried to add derivation output context of %s, which is not a derivation, to a string",
name
).atPos(i.pos).debugThrow();
}
for (auto elem : iter->value->listItems()) {
auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context");

44
src/libexpr/primops/fetchClosure.cc
View file

@ -23,20 +23,20 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor
auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath);
if (toPathMaybe && *toPathMaybe != rewrittenPath)
throw Error({
.msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
.msg = HintFmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
state.store->printStorePath(fromPath),
state.store->printStorePath(rewrittenPath),
state.store->printStorePath(*toPathMaybe)),
.errPos = state.positions[pos]
.pos = state.positions[pos]
});
if (!toPathMaybe)
throw Error({
.msg = hintfmt(
.msg = HintFmt(
"rewriting '%s' to content-addressed form yielded '%s'\n"
"Use this value for the 'toPath' attribute passed to 'fetchClosure'",
state.store->printStorePath(fromPath),
state.store->printStorePath(rewrittenPath)),
.errPos = state.positions[pos]
.pos = state.positions[pos]
});
}
@ -50,11 +50,11 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor
// We don't perform the rewriting when outPath already exists, as an optimisation.
// However, we can quickly detect a mistake if the toPath is input addressed.
throw Error({
.msg = hintfmt(
.msg = HintFmt(
"The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n"
"Set 'toPath' to an empty string to make Nix report the correct content-addressed path.",
state.store->printStorePath(toPath)),
.errPos = state.positions[pos]
.pos = state.positions[pos]
});
}
@ -73,14 +73,14 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos
if (!info->isContentAddressed(*state.store)) {
throw Error({
.msg = hintfmt(
.msg = HintFmt(
"The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n"
"If you do intend to fetch an input-addressed store path, add\n\n"
" inputAddressed = true;\n\n"
"to the 'fetchClosure' arguments.\n\n"
"Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.",
state.store->printStorePath(fromPath)),
.errPos = state.positions[pos]
.pos = state.positions[pos]
});
}
@ -99,11 +99,11 @@ static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosId
if (info->isContentAddressed(*state.store)) {
throw Error({
.msg = hintfmt(
.msg = HintFmt(
"The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n"
"Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed",
state.store->printStorePath(fromPath)),
.errPos = state.positions[pos]
.pos = state.positions[pos]
});
}
@ -153,15 +153,15 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
else
throw Error({
.msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName),
.errPos = state.positions[pos]
.msg = HintFmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName),
.pos = state.positions[pos]
});
}
if (!fromPath)
throw Error({
.msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
.errPos = state.positions[pos]
.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
.pos = state.positions[pos]
});
bool inputAddressed = inputAddressedMaybe.value_or(false);
@ -169,17 +169,17 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
if (inputAddressed) {
if (toPath)
throw Error({
.msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them",
.msg = HintFmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them",
"inputAddressed",
"toPath"),
.errPos = state.positions[pos]
.pos = state.positions[pos]
});
}
if (!fromStoreUrl)
throw Error({
.msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
.errPos = state.positions[pos]
.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
.pos = state.positions[pos]
});
auto parsedURL = parseURL(*fromStoreUrl);
@ -188,14 +188,14 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
parsedURL.scheme != "https" &&
!(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
throw Error({
.msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"),
.errPos = state.positions[pos]
.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"),
.pos = state.positions[pos]
});
if (!parsedURL.query.empty())
throw Error({
.msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
.errPos = state.positions[pos]
.msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
.pos = state.positions[pos]
});
auto fromStore = openStore(parsedURL.to_string());

10
src/libexpr/primops/fetchMercurial.cc
View file

@ -38,17 +38,11 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.fetchMercurial");
else
throw EvalError({
.msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]),
.errPos = state.positions[attr.pos]
});
state.error<EvalError>("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]).atPos(attr.pos).debugThrow();
}
if (url.empty())
throw EvalError({
.msg = hintfmt("'url' argument required"),
.errPos = state.positions[pos]
});
state.error<EvalError>("'url' argument required").atPos(pos).debugThrow();
} else
url = state.coerceToString(pos, *args[0], context,

73
src/libexpr/primops/fetchTree.cc
View file

@ -9,6 +9,7 @@
#include "tarball.hh"
#include "url.hh"
#include "value-to-json.hh"
#include "fetch-to-store.hh"
#include <ctime>
#include <iomanip>
@ -24,8 +25,6 @@ void emitTreeAttrs(
bool emptyRevFallback,
bool forceDirty)
{
assert(input.isLocked());
auto attrs = state.buildBindings(100);
state.mkStorePathString(storePath, attrs.alloc(state.sOutPath));
@ -100,16 +99,14 @@ static void fetchTree(
if (auto aType = args[0]->attrs->get(state.sType)) {
if (type)
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("unexpected attribute 'type'"),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"unexpected attribute 'type'"
).atPos(pos).debugThrow();
type = state.forceStringNoCtx(*aType->value, aType->pos, "while evaluating the `type` attribute passed to builtins.fetchTree");
} else if (!type)
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"attribute 'type' is missing in call to 'fetchTree'"
).atPos(pos).debugThrow();
attrs.emplace("type", type.value());
@ -132,8 +129,8 @@ static void fetchTree(
attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump());
}
else
state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
state.symbols[attr.name], showType(*attr.value)));
state.error<TypeError>("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
state.symbols[attr.name], showType(*attr.value)).debugThrow();
}
if (params.isFetchGit && !attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
@ -142,10 +139,9 @@ static void fetchTree(
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"attribute 'name' isn’t supported in call to 'fetchTree'"
).atPos(pos).debugThrow();
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
@ -163,10 +159,9 @@ static void fetchTree(
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"
).atPos(pos).debugThrow();
input = fetchers::Input::fromURL(url);
}
}
@ -175,10 +170,14 @@ static void fetchTree(
input = lookupInRegistries(state.store, input).first;
if (evalSettings.pureEval && !input.isLocked()) {
auto fetcher = "fetchTree";
if (params.isFetchGit)
state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchGit' requires a locked input, at %s", state.positions[pos]));
else
state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));
fetcher = "fetchGit";
state.error<EvalError>(
"in pure evaluation mode, '%s' will not fetch unlocked input '%s'",
fetcher, input.to_string()
).atPos(pos).debugThrow();
}
state.checkURI(input.toURLString());
@ -432,17 +431,13 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
else
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("unsupported argument '%s' to '%s'", n, who),
.errPos = state.positions[attr.pos]
}));
state.error<EvalError>("unsupported argument '%s' to '%s'", n, who)
.atPos(pos).debugThrow();
}
if (!url)
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("'url' argument required"),
.errPos = state.positions[pos]
}));
state.error<EvalError>(
"'url' argument required").atPos(pos).debugThrow();
} else
url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch");
@ -455,7 +450,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
name = baseNameOf(*url);
if (evalSettings.pureEval && !expectedHash)
state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who));
state.error<EvalError>("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow();
// early exit if pinned and already in the store
if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) {
@ -477,16 +472,22 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
// https://github.com/NixOS/nix/issues/4313
auto storePath =
unpack
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name)
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
if (expectedHash) {
auto hash = unpack
? state.store->queryPathInfo(storePath)->narHash
: hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath));
if (hash != *expectedHash)
state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)));
if (hash != *expectedHash) {
state.error<EvalError>(
"hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url,
expectedHash->to_string(HashFormat::Nix32, true),
hash.to_string(HashFormat::Nix32, true)
).withExitStatus(102)
.debugThrow();
}
}
state.allowAndSetStorePathString(storePath, v);

5
src/libexpr/primops/fromTOML.cc
View file

@ -83,10 +83,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
try {
visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */));
} catch (std::exception & e) { // TODO: toml::syntax_error
throw EvalError({
.msg = hintfmt("while parsing a TOML string: %s", e.what()),
.errPos = state.positions[pos]
});
state.error<EvalError>("while parsing TOML: %s", e.what()).atPos(pos).debugThrow();
}
}

22
src/libexpr/print-options.hh
View file

@ -17,24 +17,29 @@ struct PrintOptions
* If true, output ANSI color sequences.
*/
bool ansiColors = false;
/**
* If true, force values.
*/
bool force = false;
/**
* If true and `force` is set, print derivations as
* `«derivation /nix/store/...»` instead of as attribute sets.
*/
bool derivationPaths = false;
/**
* If true, track which values have been printed and skip them on
* subsequent encounters. Useful for self-referential values.
*/
bool trackRepeated = true;
/**
* Maximum depth to evaluate to.
*/
size_t maxDepth = std::numeric_limits<size_t>::max();
/**
* Maximum number of attributes in attribute sets to print.
*
@ -42,6 +47,7 @@ struct PrintOptions
* attribute set encountered.
*/
size_t maxAttrs = std::numeric_limits<size_t>::max();
/**
* Maximum number of list items to print.
*
@ -49,10 +55,26 @@ struct PrintOptions
* list encountered.
*/
size_t maxListItems = std::numeric_limits<size_t>::max();
/**
* Maximum string length to print.
*/
size_t maxStringLength = std::numeric_limits<size_t>::max();
/**
* Indentation width for pretty-printing.
*
* If set to 0 (the default), values are not pretty-printed.
*/
size_t prettyIndent = 0;
/**
* True if pretty-printing is enabled.
*/
inline bool shouldPrettyPrint()
{
return prettyIndent > 0;
}
};
/**

133
src/libexpr/print.cc
View file

@ -152,7 +152,8 @@ struct ImportantFirstAttrNameCmp
}
};
typedef std::set<Value *> ValuesSeen;
typedef std::set<const void *> ValuesSeen;
typedef std::vector<std::pair<std::string, Value *>> AttrVec;
class Printer
{
@ -163,6 +164,37 @@ private:
std::optional<ValuesSeen> seen;
size_t attrsPrinted = 0;
size_t listItemsPrinted = 0;
std::string indent;
void increaseIndent()
{
if (options.shouldPrettyPrint()) {
indent.append(options.prettyIndent, ' ');
}
}
void decreaseIndent()
{
if (options.shouldPrettyPrint()) {
assert(indent.size() >= options.prettyIndent);
indent.resize(indent.size() - options.prettyIndent);
}
}
/**
* Print a space (for separating items or attributes).
*
* If pretty-printing is enabled, a newline and the current `indent` is
* printed instead.
*/
void printSpace(bool prettyPrint)
{
if (prettyPrint) {
output << "\n" << indent;
} else {
output << " ";
}
}
void printRepeated()
{
@ -255,14 +287,36 @@ private:
output << "»";
if (options.ansiColors)
output << ANSI_NORMAL;
} catch (BaseError & e) {
} catch (Error & e) {
printError_(e);
}
}
bool shouldPrettyPrintAttrs(AttrVec & v)
{
if (!options.shouldPrettyPrint() || v.empty()) {
return false;
}
// Pretty-print attrsets with more than one item.
if (v.size() > 1) {
return true;
}
auto item = v[0].second;
if (!item) {
return true;
}
// Pretty-print single-item attrsets only if they contain nested
// structures.
auto itemType = item->type();
return itemType == nList || itemType == nAttrs || itemType == nThunk;
}
void printAttrs(Value & v, size_t depth)
{
if (seen && !seen->insert(&v).second) {
if (seen && !seen->insert(v.attrs).second) {
printRepeated();
return;
}
@ -270,9 +324,10 @@ private:
if (options.force && options.derivationPaths && state.isDerivation(v)) {
printDerivation(v);
} else if (depth < options.maxDepth) {
output << "{ ";
increaseIndent();
output << "{";
std::vector<std::pair<std::string, Value *>> sorted;
AttrVec sorted;
for (auto & i : *v.attrs)
sorted.emplace_back(std::pair(state.symbols[i.name], i.value));
@ -281,7 +336,11 @@ private:
else
std::sort(sorted.begin(), sorted.end(), ImportantFirstAttrNameCmp());
auto prettyPrint = shouldPrettyPrintAttrs(sorted);
for (auto & i : sorted) {
printSpace(prettyPrint);
if (attrsPrinted >= options.maxAttrs) {
printElided(sorted.size() - attrsPrinted, "attribute", "attributes");
break;
@ -290,13 +349,38 @@ private:
printAttributeName(output, i.first);
output << " = ";
print(*i.second, depth + 1);
output << "; ";
output << ";";
attrsPrinted++;
}
decreaseIndent();
printSpace(prettyPrint);
output << "}";
} else
} else {
output << "{ ... }";
}
}
bool shouldPrettyPrintList(std::span<Value * const> list)
{
if (!options.shouldPrettyPrint() || list.empty()) {
return false;
}
// Pretty-print lists with more than one item.
if (list.size() > 1) {
return true;
}
auto item = list[0];
if (!item) {
return true;
}
// Pretty-print single-item lists only if they contain nested
// structures.
auto itemType = item->type();
return itemType == nList || itemType == nAttrs || itemType == nThunk;
}
void printList(Value & v, size_t depth)
@ -306,11 +390,16 @@ private:
return;
}
output << "[ ";
if (depth < options.maxDepth) {
for (auto elem : v.listItems()) {
increaseIndent();
output << "[";
auto listItems = v.listItems();
auto prettyPrint = shouldPrettyPrintList(listItems);
for (auto elem : listItems) {
printSpace(prettyPrint);
if (listItemsPrinted >= options.maxListItems) {
printElided(v.listSize() - listItemsPrinted, "item", "items");
printElided(listItems.size() - listItemsPrinted, "item", "items");
break;
}
@ -319,13 +408,15 @@ private:
} else {
printNullptr();
}
output << " ";
listItemsPrinted++;
}
decreaseIndent();
printSpace(prettyPrint);
output << "]";
} else {
output << "[ ... ]";
}
else
output << "... ";
output << "]";
}
void printFunction(Value & v)
@ -405,11 +496,11 @@ private:
output << ANSI_NORMAL;
}
void printError_(BaseError & e)
void printError_(Error & e)
{
if (options.ansiColors)
output << ANSI_RED;
output << "«" << e.msg() << "»";
output << "«error: " << filterANSIEscapes(e.info().msg.str(), true) << "»";
if (options.ansiColors)
output << ANSI_NORMAL;
}
@ -422,7 +513,7 @@ private:
if (options.force) {
try {
state.forceValue(v, v.determinePos(noPos));
} catch (BaseError & e) {
} catch (Error & e) {
printError_(e);
return;
}
@ -488,6 +579,7 @@ public:
{
attrsPrinted = 0;
listItemsPrinted = 0;
indent.clear();
if (options.trackRepeated) {
seen.emplace();
@ -511,4 +603,11 @@ std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer)
return output;
}
template<>
HintFmt & HintFmt::operator%(const ValuePrinter & value)
{
fmt % value;
return *this;
}
}

10
src/libexpr/print.hh
View file

@ -9,6 +9,7 @@
#include <iostream>
#include "fmt.hh"
#include "print-options.hh"
namespace nix {
@ -78,4 +79,13 @@ public:
};
std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer);
/**
* `ValuePrinter` does its own ANSI formatting, so we don't color it
* magenta.
*/
template<>
HintFmt & HintFmt::operator%(const ValuePrinter & value);
}

20
src/libexpr/repl-exit-status.hh Normal file
View file

@ -0,0 +1,20 @@
#pragma once
namespace nix {
/**
* Exit status returned from the REPL.
*/
enum class ReplExitStatus {
/**
* The user exited with `:quit`. The program (e.g., if the REPL was acting
* as the debugger) should exit.
*/
QuitAll,
/**
* The user exited with `:continue`. The program should continue running.
*/
Continue,
};
}

22
src/libexpr/value-to-json.cc
View file

@ -64,7 +64,7 @@ json printValueAsJSON(EvalState & state, bool strict,
out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore);
} catch (Error & e) {
e.addTrace(state.positions[a.pos],
hintfmt("while evaluating attribute '%1%'", j));
HintFmt("while evaluating attribute '%1%'", j));
throw;
}
}
@ -80,8 +80,8 @@ json printValueAsJSON(EvalState & state, bool strict,
try {
out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore));
} catch (Error & e) {
e.addTrace({},
hintfmt("while evaluating list element at index %1%", i));
e.addTrace(state.positions[pos],
HintFmt("while evaluating list element at index %1%", i));
throw;
}
i++;
@ -99,13 +99,12 @@ json printValueAsJSON(EvalState & state, bool strict,
case nThunk:
case nFunction:
auto e = TypeError({
.msg = hintfmt("cannot convert %1% to JSON", showType(v)),
.errPos = state.positions[v.determinePos(pos)]
});
e.addTrace(state.positions[pos], hintfmt("message for the trace"));
state.debugThrowLastTrace(e);
throw e;
state.error<TypeError>(
"cannot convert %1% to JSON",
showType(v)
)
.atPos(v.determinePos(pos))
.debugThrow();
}
return out;
}
@ -119,7 +118,8 @@ void printValueAsJSON(EvalState & state, bool strict,
json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
NixStringContext & context, bool copyToStore) const
{
state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType()));
state.error<TypeError>("cannot convert %1% to JSON", showType())
.debugThrow();
}

2
src/libexpr/value.hh
View file

@ -105,7 +105,7 @@ class ExternalValueBase
* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
* error.
*/
virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const;
virtual std::string coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const;
/**
* Compare to another value of the same type. Defaults to uncomparable,

4
src/libexpr/value/context.hh
View file

@ -19,8 +19,8 @@ public:
: Error("")
{
raw = raw_;
auto hf = hintfmt(args...);
err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw);
auto hf = HintFmt(args...);
err.msg = HintFmt("Bad String Context element: %1%: %2%", Uncolored(hf.str()), raw);
}
};

5
src/libfetchers/attrs.cc
View file

@ -104,4 +104,9 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
return query;
}
Hash getRevAttr(const Attrs & attrs, const std::string & name)
{
return Hash::parseAny(getStrAttr(attrs, name), HashAlgorithm::SHA1);
}
}

2
src/libfetchers/attrs.hh
View file

@ -39,4 +39,6 @@ bool getBoolAttr(const Attrs & attrs, const std::string & name);
std::map<std::string, std::string> attrsToQuery(const Attrs & attrs);
Hash getRevAttr(const Attrs & attrs, const std::string & name);
}

26
src/libfetchers/fetch-to-store.cc
View file

@ -7,6 +7,7 @@ namespace nix {
StorePath fetchToStore(
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name,
ContentAddressMethod method,
PathFilter * filter,
@ -21,23 +22,9 @@ StorePath fetchToStore(
cacheKey = fetchers::Attrs{
{"_what", "fetchToStore"},
{"store", store.storeDir},
{"name", std::string(name)},
{"name", std::string{name}},
{"fingerprint", *path.accessor->fingerprint},
{
"method",
std::visit(overloaded {
[](const TextIngestionMethod &) {
return "text";
},
[](const FileIngestionMethod & fim) {
switch (fim) {
case FileIngestionMethod::Flat: return "flat";
case FileIngestionMethod::Recursive: return "nar";
default: assert(false);
}
},
}, method.raw),
},
{"method", std::string{method.render()}},
{"path", path.path.abs()}
};
if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
@ -47,18 +34,19 @@ StorePath fetchToStore(
} else
debug("source path '%s' is uncacheable", path);
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path));
Activity act(*logger, lvlChatty, actUnknown,
fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path));
auto filter2 = filter ? *filter : defaultPathFilter;
auto storePath =
settings.readOnlyMode
mode == FetchMode::DryRun
? store.computeStorePath(
name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first
: store.addToStore(
name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair);
if (cacheKey)
if (cacheKey && mode == FetchMode::Copy)
fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
return storePath;

3
src/libfetchers/fetch-to-store.hh
View file

@ -8,12 +8,15 @@
namespace nix {
enum struct FetchMode { DryRun, Copy };
/**
* Copy the `path` to the Nix store.
*/
StorePath fetchToStore(
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name = "source",
ContentAddressMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr,

13
src/libfetchers/fetchers.cc
View file

@ -45,12 +45,8 @@ static void fixupInput(Input & input)
// Check common attributes.
input.getType();
input.getRef();
if (input.getRev())
input.locked = true;
input.getRevCount();
input.getLastModified();
if (input.getNarHash())
input.locked = true;
}
Input Input::fromURL(const ParsedURL & url, bool requireTree)
@ -140,6 +136,11 @@ bool Input::isDirect() const
return !scheme || scheme->isDirect(*this);
}
bool Input::isLocked() const
{
return scheme && scheme->isLocked(*this);
}
Attrs Input::toAttrs() const
{
return attrs;
@ -222,8 +223,6 @@ std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
input.to_string(), *prevRevCount);
}
input.locked = true;
return {std::move(storePath), input};
}
@ -376,7 +375,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
{
auto [accessor, input2] = getAccessor(store, input);
auto storePath = fetchToStore(*store, SourcePath(accessor), input2.getName());
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, input2.getName());
return {storePath, input2};
}

12
src/libfetchers/fetchers.hh
View file

@ -29,7 +29,6 @@ struct Input
std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs;
bool locked = false;
/**
* path of the parent of this input, used for relative path resolution
@ -71,7 +70,7 @@ public:
* Check whether this is a "locked" input, that is,
* one that contains a commit hash or content hash.
*/
bool isLocked() const { return locked; }
bool isLocked() const;
bool operator ==(const Input & other) const;
@ -121,7 +120,6 @@ public:
std::optional<std::string> getFingerprint(ref<Store> store) const;
};
/**
* The `InputScheme` represents a type of fetcher. Each fetcher
* registers with nix at startup time. When processing an `Input`,
@ -196,6 +194,14 @@ struct InputScheme
*/
virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
{ return std::nullopt; }
/**
* Return `true` if this input is considered "locked", i.e. it has
* attributes like a Git revision or NAR hash that uniquely
* identify its contents.
*/
virtual bool isLocked(const Input & input) const
{ return false; }
};
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

30
src/libfetchers/filtering-input-accessor.cc
View file

@ -5,26 +5,26 @@ namespace nix {
std::string FilteringInputAccessor::readFile(const CanonPath & path)
{
checkAccess(path);
return next->readFile(prefix + path);
return next->readFile(prefix / path);
}
bool FilteringInputAccessor::pathExists(const CanonPath & path)
{
return isAllowed(path) && next->pathExists(prefix + path);
return isAllowed(path) && next->pathExists(prefix / path);
}
std::optional<InputAccessor::Stat> FilteringInputAccessor::maybeLstat(const CanonPath & path)
{
checkAccess(path);
return next->maybeLstat(prefix + path);
return next->maybeLstat(prefix / path);
}
InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path)
{
checkAccess(path);
DirEntries entries;
for (auto & entry : next->readDirectory(prefix + path)) {
if (isAllowed(path + entry.first))
for (auto & entry : next->readDirectory(prefix / path)) {
if (isAllowed(path / entry.first))
entries.insert(std::move(entry));
}
return entries;
@ -33,12 +33,12 @@ InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath
std::string FilteringInputAccessor::readLink(const CanonPath & path)
{
checkAccess(path);
return next->readLink(prefix + path);
return next->readLink(prefix / path);
}
std::string FilteringInputAccessor::showPath(const CanonPath & path)
{
return next->showPath(prefix + path);
return next->showPath(prefix / path);
}
void FilteringInputAccessor::checkAccess(const CanonPath & path)
@ -51,33 +51,33 @@ void FilteringInputAccessor::checkAccess(const CanonPath & path)
struct AllowListInputAccessorImpl : AllowListInputAccessor
{
std::set<CanonPath> allowedPaths;
std::set<CanonPath> allowedPrefixes;
AllowListInputAccessorImpl(
ref<InputAccessor> next,
std::set<CanonPath> && allowedPaths,
std::set<CanonPath> && allowedPrefixes,
MakeNotAllowedError && makeNotAllowedError)
: AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError))
, allowedPaths(std::move(allowedPaths))
, allowedPrefixes(std::move(allowedPrefixes))
{ }
bool isAllowed(const CanonPath & path) override
{
return path.isAllowed(allowedPaths);
return path.isAllowed(allowedPrefixes);
}
void allowPath(CanonPath path) override
void allowPrefix(CanonPath prefix) override
{
allowedPaths.insert(std::move(path));
allowedPrefixes.insert(std::move(prefix));
}
};
ref<AllowListInputAccessor> AllowListInputAccessor::create(
ref<InputAccessor> next,
std::set<CanonPath> && allowedPaths,
std::set<CanonPath> && allowedPrefixes,
MakeNotAllowedError && makeNotAllowedError)
{
return make_ref<AllowListInputAccessorImpl>(next, std::move(allowedPaths), std::move(makeNotAllowedError));
return make_ref<AllowListInputAccessorImpl>(next, std::move(allowedPrefixes), std::move(makeNotAllowedError));
}
bool CachingFilteringInputAccessor::isAllowed(const CanonPath & path)

9
src/libfetchers/filtering-input-accessor.hh
View file

@ -54,18 +54,19 @@ struct FilteringInputAccessor : InputAccessor
};
/**
* A wrapping `InputAccessor` that checks paths against an allow-list.
* A wrapping `InputAccessor` that checks paths against a set of
* allowed prefixes.
*/
struct AllowListInputAccessor : public FilteringInputAccessor
{
/**
* Grant access to the specified path.
* Grant access to the specified prefix.
*/
virtual void allowPath(CanonPath path) = 0;
virtual void allowPrefix(CanonPath prefix) = 0;
static ref<AllowListInputAccessor> create(
ref<InputAccessor> next,
std::set<CanonPath> && allowedPaths,
std::set<CanonPath> && allowedPrefixes,
MakeNotAllowedError && makeNotAllowedError);
using FilteringInputAccessor::FilteringInputAccessor;

64
src/libfetchers/fs-input-accessor.cc
View file

@ -6,72 +6,30 @@ namespace nix {
struct FSInputAccessor : InputAccessor, PosixSourceAccessor
{
CanonPath root;
FSInputAccessor(const CanonPath & root)
: root(root)
{
displayPrefix = root.isRoot() ? "" : root.abs();
}
void readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback) override
{
auto absPath = makeAbsPath(path);
PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
}
bool pathExists(const CanonPath & path) override
{
return PosixSourceAccessor::pathExists(makeAbsPath(path));
}
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
return PosixSourceAccessor::maybeLstat(makeAbsPath(path));
}
DirEntries readDirectory(const CanonPath & path) override
{
DirEntries res;
for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path)))
res.emplace(entry);
return res;
}
std::string readLink(const CanonPath & path) override
{
return PosixSourceAccessor::readLink(makeAbsPath(path));
}
CanonPath makeAbsPath(const CanonPath & path)
{
return root + path;
}
std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override
{
return makeAbsPath(path);
}
using PosixSourceAccessor::PosixSourceAccessor;
};
ref<InputAccessor> makeFSInputAccessor(const CanonPath & root)
ref<InputAccessor> makeFSInputAccessor()
{
return make_ref<FSInputAccessor>(root);
return make_ref<FSInputAccessor>();
}
ref<InputAccessor> makeFSInputAccessor(std::filesystem::path root)
{
return make_ref<FSInputAccessor>(std::move(root));
}
ref<InputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath)
{
return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)));
// FIXME: should use `store->getFSAccessor()`
return makeFSInputAccessor(std::filesystem::path { store->toRealPath(storePath) });
}
SourcePath getUnfilteredRootPath(CanonPath path)
{
static auto rootFS = makeFSInputAccessor(CanonPath::root);
static auto rootFS = makeFSInputAccessor();
return {rootFS, path};
}

5
src/libfetchers/fs-input-accessor.hh
View file

@ -8,8 +8,9 @@ namespace nix {
class StorePath;
class Store;
ref<InputAccessor> makeFSInputAccessor(
const CanonPath & root);
ref<InputAccessor> makeFSInputAccessor();
ref<InputAccessor> makeFSInputAccessor(std::filesystem::path root);
ref<InputAccessor> makeStorePathAccessor(
ref<Store> store,

288
src/libfetchers/git-utils.cc
View file

@ -2,12 +2,13 @@
#include "fs-input-accessor.hh"
#include "input-accessor.hh"
#include "filtering-input-accessor.hh"
#include "memory-input-accessor.hh"
#include "cache.hh"
#include "finally.hh"
#include "processes.hh"
#include "signals.hh"
#include <boost/core/span.hpp>
#include "users.hh"
#include "fs-sink.hh"
#include <git2/attr.h>
#include <git2/blob.h>
@ -28,6 +29,7 @@
#include <unordered_set>
#include <queue>
#include <regex>
#include <span>
namespace std {
@ -140,15 +142,15 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type)
struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
{
/** Location of the repository on disk. */
CanonPath path;
std::filesystem::path path;
Repository repo;
GitRepoImpl(CanonPath _path, bool create, bool bare)
GitRepoImpl(std::filesystem::path _path, bool create, bool bare)
: path(std::move(_path))
{
initLibGit2();
if (pathExists(path.abs())) {
if (pathExists(path.native())) {
if (git_repository_open(Setter(repo), path.c_str()))
throw Error("opening Git repository '%s': %s", path, git_error_last()->message);
} else {
@ -221,10 +223,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
return toHash(*oid);
}
std::vector<Submodule> parseSubmodules(const CanonPath & configFile)
std::vector<Submodule> parseSubmodules(const std::filesystem::path & configFile)
{
GitConfig config;
if (git_config_open_ondisk(Setter(config), configFile.abs().c_str()))
if (git_config_open_ondisk(Setter(config), configFile.c_str()))
throw Error("parsing .gitmodules file: %s", git_error_last()->message);
ConfigIterator it;
@ -295,8 +297,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
throw Error("getting working directory status: %s", git_error_last()->message);
/* Get submodule info. */
auto modulesFile = path + ".gitmodules";
if (pathExists(modulesFile.abs()))
auto modulesFile = path / ".gitmodules";
if (pathExists(modulesFile))
info.submodules = parseSubmodules(modulesFile);
return info;
@ -356,6 +358,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
ref<InputAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
ref<GitFileSystemObjectSink> getFileSystemObjectSink() override;
static int sidebandProgressCallback(const char * str, int len, void * payload)
{
auto act = (Activity *) payload;
@ -389,10 +393,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
auto dir = this->path;
Strings gitArgs;
if (shallow) {
gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec };
gitArgs = { "-C", dir, "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec };
}
else {
gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--", url, refspec };
gitArgs = { "-C", dir, "fetch", "--quiet", "--force", "--", url, refspec };
}
runProgram(RunOptions {
@ -438,7 +442,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
.args = {
"-c",
"gpg.ssh.allowedSignersFile=" + allowedSignersFile,
"-C", path.abs(),
"-C", path,
"verify-commit",
rev.gitRev()
},
@ -463,9 +467,25 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
else
throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
}
Hash treeHashToNarHash(const Hash & treeHash) override
{
auto accessor = getAccessor(treeHash, false);
fetchers::Attrs cacheKey({{"_what", "treeHashToNarHash"}, {"treeHash", treeHash.gitRev()}});
if (auto res = fetchers::getCache()->lookup(cacheKey))
return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
auto narHash = accessor->hashPath(CanonPath::root);
fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
return narHash;
}
};
ref<GitRepo> GitRepo::openRepo(const CanonPath & path, bool create, bool bare)
ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare)
{
return make_ref<GitRepoImpl>(path, create, bare);
}
@ -576,20 +596,61 @@ struct GitInputAccessor : InputAccessor
/* Recursively look up 'path' relative to the root. */
git_tree_entry * lookup(const CanonPath & path)
{
if (path.isRoot()) return nullptr;
auto i = lookupCache.find(path);
if (i == lookupCache.end()) {
TreeEntry entry;
if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) {
if (err != GIT_ENOTFOUND)
throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
}
if (i != lookupCache.end()) return i->second.get();
i = lookupCache.emplace(path, std::move(entry)).first;
auto parent = path.parent();
if (!parent) return nullptr;
auto name = path.baseName().value();
auto parentTree = lookupTree(*parent);
if (!parentTree) return nullptr;
auto count = git_tree_entrycount(parentTree->get());
git_tree_entry * res = nullptr;
/* Add all the tree entries to the cache to speed up
subsequent lookups. */
for (size_t n = 0; n < count; ++n) {
auto entry = git_tree_entry_byindex(parentTree->get(), n);
TreeEntry copy;
if (git_tree_entry_dup(Setter(copy), entry))
throw Error("dupping tree entry: %s", git_error_last()->message);
auto entryName = std::string_view(git_tree_entry_name(entry));
if (entryName == name)
res = copy.get();
auto path2 = *parent;
path2.push(entryName);
lookupCache.emplace(path2, std::move(copy)).first->second.get();
}
return &*i->second;
return res;
}
std::optional<Tree> lookupTree(const CanonPath & path)
{
if (path.isRoot()) {
Tree tree;
if (git_tree_dup(Setter(tree), root.get()))
throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message);
return tree;
}
auto entry = lookup(path);
if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_TREE)
return std::nullopt;
Tree tree;
if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry))
throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
return tree;
}
git_tree_entry * need(const CanonPath & path)
@ -729,6 +790,154 @@ struct GitExportIgnoreInputAccessor : CachingFilteringInputAccessor {
};
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
{
ref<GitRepoImpl> repo;
struct PendingDir
{
std::string name;
TreeBuilder builder;
};
std::vector<PendingDir> pendingDirs;
size_t componentsToStrip = 1;
void pushBuilder(std::string name)
{
git_treebuilder * b;
if (git_treebuilder_new(&b, *repo, nullptr))
throw Error("creating a tree builder: %s", git_error_last()->message);
pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) });
};
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo) : repo(repo)
{
pushBuilder("");
}
std::pair<git_oid, std::string> popBuilder()
{
assert(!pendingDirs.empty());
auto pending = std::move(pendingDirs.back());
git_oid oid;
if (git_treebuilder_write(&oid, pending.builder.get()))
throw Error("creating a tree object: %s", git_error_last()->message);
pendingDirs.pop_back();
return {oid, pending.name};
};
void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode)
{
assert(!pendingDirs.empty());
auto & pending = pendingDirs.back();
if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode))
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
};
void updateBuilders(std::span<const std::string> names)
{
// Find the common prefix of pendingDirs and names.
size_t prefixLen = 0;
for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen)
if (names[prefixLen] != pendingDirs[prefixLen + 1].name)
break;
// Finish the builders that are not part of the common prefix.
for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) {
auto [oid, name] = popBuilder();
addToTree(name, oid, GIT_FILEMODE_TREE);
}
// Create builders for the new directories.
for (auto n = prefixLen; n < names.size(); ++n)
pushBuilder(names[n]);
};
bool prepareDirs(const std::vector<std::string> & pathComponents, bool isDir)
{
std::span<const std::string> pathComponents2{pathComponents};
if (pathComponents2.size() <= componentsToStrip) return false;
pathComponents2 = pathComponents2.subspan(componentsToStrip);
updateBuilders(
isDir
? pathComponents2
: pathComponents2.first(pathComponents2.size() - 1));
return true;
}
void createRegularFile(
const Path & path,
std::function<void(CreateRegularFileSink &)> func) override
{
auto pathComponents = tokenizeString<std::vector<std::string>>(path, "/");
if (!prepareDirs(pathComponents, false)) return;
git_writestream * stream = nullptr;
if (git_blob_create_from_stream(&stream, *repo, nullptr))
throw Error("creating a blob stream object: %s", git_error_last()->message);
struct CRF : CreateRegularFileSink {
const Path & path;
GitFileSystemObjectSinkImpl & back;
git_writestream * stream;
bool executable = false;
CRF(const Path & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream)
: path(path), back(back), stream(stream)
{}
void operator () (std::string_view data) override
{
if (stream->write(stream, data.data(), data.size()))
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
}
void isExecutable() override
{
executable = true;
}
} crf { path, *this, stream };
func(crf);
git_oid oid;
if (git_blob_create_from_stream_commit(&oid, stream))
throw Error("creating a blob object for tarball member '%s': %s", path, git_error_last()->message);
addToTree(*pathComponents.rbegin(), oid,
crf.executable
? GIT_FILEMODE_BLOB_EXECUTABLE
: GIT_FILEMODE_BLOB);
}
void createDirectory(const Path & path) override
{
auto pathComponents = tokenizeString<std::vector<std::string>>(path, "/");
(void) prepareDirs(pathComponents, true);
}
void createSymlink(const Path & path, const std::string & target) override
{
auto pathComponents = tokenizeString<std::vector<std::string>>(path, "/");
if (!prepareDirs(pathComponents, false)) return;
git_oid oid;
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
}
Hash sync() override {
updateBuilders({});
auto [oid, _name] = popBuilder();
return toHash(oid);
}
};
ref<GitInputAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
{
auto self = ref<GitRepoImpl>(shared_from_this());
@ -750,17 +959,26 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
ref<InputAccessor> GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
{
auto self = ref<GitRepoImpl>(shared_from_this());
/* In case of an empty workdir, return an empty in-memory tree. We
cannot use AllowListInputAccessor because it would return an
error for the root (and we can't add the root to the allow-list
since that would allow access to all its children). */
ref<InputAccessor> fileAccessor =
AllowListInputAccessor::create(
makeFSInputAccessor(path),
std::set<CanonPath> { wd.files },
std::move(makeNotAllowedError));
if (exportIgnore) {
wd.files.empty()
? makeEmptyInputAccessor()
: AllowListInputAccessor::create(
makeFSInputAccessor(path),
std::set<CanonPath> { wd.files },
std::move(makeNotAllowedError)).cast<InputAccessor>();
if (exportIgnore)
return make_ref<GitExportIgnoreInputAccessor>(self, fileAccessor, std::nullopt);
}
else {
else
return fileAccessor;
}
}
ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink()
{
return make_ref<GitFileSystemObjectSinkImpl>(ref<GitRepoImpl>(shared_from_this()));
}
std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev, bool exportIgnore)
@ -781,7 +999,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
auto rawAccessor = getRawAccessor(rev);
for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) {
for (auto & submodule : parseSubmodules(pathTemp)) {
auto rev = rawAccessor->getSubmoduleRev(submodule.path);
result.push_back({std::move(submodule), rev});
}
@ -789,5 +1007,11 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
return result;
}
ref<GitRepo> getTarballCache()
{
static auto repoDir = std::filesystem::path(getCacheDir()) / "nix" / "tarball-cache";
return GitRepo::openRepo(repoDir, true, true);
}
}

27
src/libfetchers/git-utils.hh
View file

@ -2,17 +2,26 @@
#include "filtering-input-accessor.hh"
#include "input-accessor.hh"
#include "fs-sink.hh"
namespace nix {
namespace fetchers { struct PublicKey; }
struct GitFileSystemObjectSink : FileSystemObjectSink
{
/**
* Flush builder and return a final Git hash.
*/
virtual Hash sync() = 0;
};
struct GitRepo
{
virtual ~GitRepo()
{ }
static ref<GitRepo> openRepo(const CanonPath & path, bool create = false, bool bare = false);
static ref<GitRepo> openRepo(const std::filesystem::path & path, bool create = false, bool bare = false);
virtual uint64_t getRevCount(const Hash & rev) = 0;
@ -64,18 +73,14 @@ struct GitRepo
const std::string & url,
const std::string & base) = 0;
struct TarballInfo
{
Hash treeHash;
time_t lastModified;
};
virtual bool hasObject(const Hash & oid) = 0;
virtual ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) = 0;
virtual ref<InputAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0;
virtual void fetch(
const std::string & url,
const std::string & refspec,
@ -88,6 +93,14 @@ struct GitRepo
virtual void verifyCommit(
const Hash & rev,
const std::vector<fetchers::PublicKey> & publicKeys) = 0;
/**
* Given a Git tree hash, compute the hash of its NAR
* serialisation. This is memoised on-disk.
*/
virtual Hash treeHashToNarHash(const Hash & treeHash) = 0;
};
ref<GitRepo> getTarballCache();
}

37
src/libfetchers/git.cc
View file

@ -158,6 +158,8 @@ std::vector<PublicKey> getPublicKeys(const Attrs & attrs)
} // end namespace
static const Hash nullRev{HashAlgorithm::SHA1};
struct GitInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
@ -319,7 +321,7 @@ struct GitInputScheme : InputScheme
if (!repoInfo.isLocal)
throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
writeFile((CanonPath(repoInfo.url) + path).abs(), contents);
writeFile((CanonPath(repoInfo.url) / path).abs(), contents);
auto result = runProgram(RunOptions {
.program = "git",
@ -415,7 +417,7 @@ struct GitInputScheme : InputScheme
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo();
repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo();
return repoInfo;
}
@ -429,7 +431,7 @@ struct GitInputScheme : InputScheme
if (auto res = cache->lookup(key))
return getIntAttr(*res, "lastModified");
auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev);
auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev);
cache->upsert(key, Attrs{{"lastModified", lastModified}});
@ -447,7 +449,7 @@ struct GitInputScheme : InputScheme
Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url));
auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev);
auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev);
cache->upsert(key, Attrs{{"revCount", revCount}});
@ -457,7 +459,7 @@ struct GitInputScheme : InputScheme
std::string getDefaultRef(const RepoInfo & repoInfo) const
{
auto head = repoInfo.isLocal
? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef()
? GitRepo::openRepo(repoInfo.url)->getWorkdirRef()
: readHeadCached(repoInfo.url);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url);
@ -510,7 +512,7 @@ struct GitInputScheme : InputScheme
if (repoInfo.isLocal) {
repoDir = repoInfo.url;
if (!input.getRev())
input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
} else {
Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input));
repoDir = cacheDir;
@ -519,7 +521,7 @@ struct GitInputScheme : InputScheme
createDirs(dirOf(cacheDir));
PathLocks cacheDirLock({cacheDir});
auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true);
auto repo = GitRepo::openRepo(cacheDir, true, true);
Path localRefFile =
ref.compare(0, 5, "refs/") == 0
@ -588,7 +590,7 @@ struct GitInputScheme : InputScheme
// cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
}
auto repo = GitRepo::openRepo(CanonPath(repoDir));
auto repo = GitRepo::openRepo(repoDir);
auto isShallow = repo->isShallow();
@ -664,7 +666,7 @@ struct GitInputScheme : InputScheme
for (auto & submodule : repoInfo.workdirInfo.submodules)
repoInfo.workdirInfo.files.insert(submodule.path);
auto repo = GitRepo::openRepo(CanonPath(repoInfo.url), false, false);
auto repo = GitRepo::openRepo(repoInfo.url, false, false);
auto exportIgnore = getExportIgnoreAttr(input);
@ -680,7 +682,7 @@ struct GitInputScheme : InputScheme
std::map<CanonPath, nix::ref<InputAccessor>> mounts;
for (auto & submodule : repoInfo.workdirInfo.submodules) {
auto submodulePath = CanonPath(repoInfo.url) + submodule.path;
auto submodulePath = CanonPath(repoInfo.url) / submodule.path;
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", submodulePath.abs());
@ -703,15 +705,17 @@ struct GitInputScheme : InputScheme
}
if (!repoInfo.workdirInfo.isDirty) {
auto repo = GitRepo::openRepo(CanonPath(repoInfo.url));
auto repo = GitRepo::openRepo(repoInfo.url);
if (auto ref = repo->getWorkdirRef())
input.attrs.insert_or_assign("ref", *ref);
auto rev = repoInfo.workdirInfo.headRev.value();
/* Return a rev of 000... if there are no commits yet. */
auto rev = repoInfo.workdirInfo.headRev.value_or(nullRev);
input.attrs.insert_or_assign("rev", rev.gitRev());
input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev));
input.attrs.insert_or_assign("revCount",
rev == nullRev ? 0 : getRevCount(repoInfo, repoInfo.url, rev));
verifyCommit(input, repo);
} else {
@ -733,8 +737,6 @@ struct GitInputScheme : InputScheme
? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev)
: 0);
input.locked = true; // FIXME
return {accessor, std::move(input)};
}
@ -771,6 +773,11 @@ struct GitInputScheme : InputScheme
else
return std::nullopt;
}
bool isLocked(const Input & input) const override
{
return (bool) input.getRev();
}
};
static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });

139
src/libfetchers/github.cc
View file

@ -8,6 +8,8 @@
#include "fetchers.hh"
#include "fetch-settings.hh"
#include "tarball.hh"
#include "tarfile.hh"
#include "git-utils.hh"
#include <optional>
#include <nlohmann/json.hpp>
@ -180,49 +182,111 @@ struct GitArchiveInputScheme : InputScheme
return headers;
}
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
struct RefInfo
{
Hash rev;
std::optional<Hash> treeHash;
};
virtual RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
struct TarballInfo
{
Input input(_input);
Hash treeHash;
time_t lastModified;
};
std::pair<Input, TarballInfo> downloadArchive(ref<Store> store, Input input) const
{
if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD");
std::optional<Hash> upstreamTreeHash;
auto rev = input.getRev();
if (!rev) rev = getRevFromRef(store, input);
if (!rev) {
auto refInfo = getRevFromRef(store, input);
rev = refInfo.rev;
upstreamTreeHash = refInfo.treeHash;
debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev());
}
input.attrs.erase("ref");
input.attrs.insert_or_assign("rev", rev->gitRev());
Attrs lockedAttrs({
{"type", "git-tarball"},
{"rev", rev->gitRev()},
});
auto cache = getCache();
if (auto res = getCache()->lookup(*store, lockedAttrs)) {
input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
return {std::move(res->second), input};
Attrs treeHashKey{{"_what", "gitRevToTreeHash"}, {"rev", rev->gitRev()}};
Attrs lastModifiedKey{{"_what", "gitRevToLastModified"}, {"rev", rev->gitRev()}};
if (auto treeHashAttrs = cache->lookup(treeHashKey)) {
if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) {
auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
if (getTarballCache()->hasObject(treeHash))
return {std::move(input), TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }};
else
debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev());
}
}
/* Stream the tarball into the tarball cache. */
auto url = getDownloadUrl(input);
auto result = downloadTarball(store, url.url, input.getName(), true, url.headers);
auto source = sinkToSource([&](Sink & sink) {
FileTransferRequest req(url.url);
req.headers = url.headers;
getFileTransfer()->download(std::move(req), sink);
});
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
TarArchive archive { *source };
auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
getCache()->add(
*store,
lockedAttrs,
{
{"rev", rev->gitRev()},
{"lastModified", uint64_t(result.lastModified)}
},
result.storePath,
true);
TarballInfo tarballInfo {
.treeHash = parseSink->sync(),
.lastModified = lastModified
};
return {result.storePath, input};
cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}});
cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}});
#if 0
if (upstreamTreeHash != tarballInfo.treeHash)
warn(
"Git tree hash mismatch for revision '%s' of '%s': "
"expected '%s', got '%s'. "
"This can happen if the Git repository uses submodules.",
rev->gitRev(), input.to_string(), upstreamTreeHash->gitRev(), tarballInfo.treeHash.gitRev());
#endif
return {std::move(input), tarballInfo};
}
std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
{
auto [input, tarballInfo] = downloadArchive(store, _input);
input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev());
input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false);
accessor->setPathDisplay("«" + input.to_string() + "»");
accessor->fingerprint = input.getFingerprint(store);
return {accessor, input};
}
bool isLocked(const Input & input) const override
{
/* Since we can't verify the integrity of the tarball from the
Git revision alone, we also require a NAR hash for
locking. FIXME: in the future, we may want to require a Git
tree hash instead of a NAR hash. */
return input.getRev().has_value() && input.getNarHash().has_value();
}
std::optional<ExperimentalFeature> experimentalFeature() const override
@ -269,7 +333,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
return getStrAttr(input.attrs, "repo");
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto host = getHost(input);
auto url = fmt(
@ -284,9 +348,11 @@ struct GitHubInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
return RefInfo {
.rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
.treeHash = Hash::parseAny(std::string { json["commit"]["tree"]["sha"] }, HashAlgorithm::SHA1)
};
}
DownloadUrl getDownloadUrl(const Input & input) const override
@ -343,7 +409,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1));
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// See rate limiting note below
@ -356,9 +422,10 @@ struct GitLabInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
return RefInfo {
.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)
};
}
DownloadUrl getDownloadUrl(const Input & input) const override
@ -402,7 +469,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
// Once it is implemented, however, should work as expected.
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
// TODO: In the future, when the sourcehut graphql API is implemented for mercurial
// and with anonymous access, this method should use it instead.
@ -445,12 +512,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
id = parsedLine->target;
}
if(!id)
if (!id)
throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
auto rev = Hash::parseAny(*id, HashAlgorithm::SHA1);
debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
return rev;
return RefInfo {
.rev = Hash::parseAny(*id, HashAlgorithm::SHA1)
};
}
DownloadUrl getDownloadUrl(const Input & input) const override

6
src/libfetchers/memory-input-accessor.cc
View file

@ -20,4 +20,10 @@ ref<MemoryInputAccessor> makeMemoryInputAccessor()
return make_ref<MemoryInputAccessorImpl>();
}
ref<InputAccessor> makeEmptyInputAccessor()
{
static auto empty = makeMemoryInputAccessor().cast<InputAccessor>();
return empty;
}
}

2
src/libfetchers/memory-input-accessor.hh
View file

@ -13,4 +13,6 @@ struct MemoryInputAccessor : InputAccessor
ref<MemoryInputAccessor> makeMemoryInputAccessor();
ref<InputAccessor> makeEmptyInputAccessor();
}

7
src/libfetchers/mercurial.cc
View file

@ -141,7 +141,7 @@ struct MercurialInputScheme : InputScheme
if (!isLocal)
throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string());
auto absPath = CanonPath(repoPath) + path;
auto absPath = CanonPath(repoPath) / path;
writeFile(absPath.abs(), contents);
@ -347,6 +347,11 @@ struct MercurialInputScheme : InputScheme
return makeResult(infoAttrs, std::move(storePath));
}
bool isLocked(const Input & input) const override
{
return (bool) input.getRev();
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())

7
src/libfetchers/path.cc
View file

@ -84,7 +84,12 @@ struct PathInputScheme : InputScheme
std::string_view contents,
std::optional<std::string> commitMsg) const override
{
writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents);
writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents);
}
bool isLocked(const Input & input) const override
{
return (bool) input.getNarHash();
}
CanonPath getAbsPath(const Input & input) const

186
src/libfetchers/tarball.cc
View file

@ -9,6 +9,9 @@
#include "types.hh"
#include "split.hh"
#include "posix-source-accessor.hh"
#include "fs-input-accessor.hh"
#include "store-api.hh"
#include "git-utils.hh"
namespace nix::fetchers {
@ -57,10 +60,8 @@ DownloadFileResult downloadFile(
throw;
}
// FIXME: write to temporary file.
Attrs infoAttrs({
{"etag", res.etag},
{"url", res.effectiveUri},
});
if (res.immutableUrl)
@ -91,96 +92,102 @@ DownloadFileResult downloadFile(
storePath = std::move(info.path);
}
getCache()->add(
*store,
inAttrs,
infoAttrs,
*storePath,
locked);
if (url != res.effectiveUri)
/* Cache metadata for all URLs in the redirect chain. */
for (auto & url : res.urls) {
inAttrs.insert_or_assign("url", url);
infoAttrs.insert_or_assign("url", *res.urls.rbegin());
getCache()->add(
*store,
{
{"type", "file"},
{"url", res.effectiveUri},
{"name", name},
},
inAttrs,
infoAttrs,
*storePath,
locked);
}
return {
.storePath = std::move(*storePath),
.etag = res.etag,
.effectiveUrl = res.effectiveUri,
.effectiveUrl = *res.urls.rbegin(),
.immutableUrl = res.immutableUrl,
};
}
DownloadTarballResult downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers)
{
Attrs inAttrs({
{"type", "tarball"},
{"_what", "tarballCache"},
{"url", url},
{"name", name},
});
auto cached = getCache()->lookupExpired(*store, inAttrs);
auto cached = getCache()->lookupExpired(inAttrs);
auto attrsToResult = [&](const Attrs & infoAttrs)
{
auto treeHash = getRevAttr(infoAttrs, "treeHash");
return DownloadTarballResult {
.treeHash = treeHash,
.lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
.immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
.accessor = getTarballCache()->getAccessor(treeHash, false),
};
};
if (cached && !getTarballCache()->hasObject(getRevAttr(cached->infoAttrs, "treeHash")))
cached.reset();
if (cached && !cached->expired)
return {
.storePath = std::move(cached->storePath),
.lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"),
.immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
};
/* We previously downloaded this tarball and it's younger than
`tarballTtl`, so no need to check the server. */
return attrsToResult(cached->infoAttrs);
auto res = downloadFile(store, url, name, locked, headers);
auto _res = std::make_shared<Sync<FileTransferResult>>();
std::optional<StorePath> unpackedStorePath;
time_t lastModified;
if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) {
unpackedStorePath = std::move(cached->storePath);
lastModified = getIntAttr(cached->infoAttrs, "lastModified");
} else {
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
unpackTarfile(store->toRealPath(res.storePath), tmpDir);
auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
lastModified = lstat(topDir).st_mtime;
PosixSourceAccessor accessor;
unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, defaultPathFilter, NoRepair);
}
Attrs infoAttrs({
{"lastModified", uint64_t(lastModified)},
{"etag", res.etag},
auto source = sinkToSource([&](Sink & sink) {
FileTransferRequest req(url);
req.expectedETag = cached ? getStrAttr(cached->infoAttrs, "etag") : "";
getFileTransfer()->download(std::move(req), sink,
[_res](FileTransferResult r)
{
*_res->lock() = r;
});
});
if (res.immutableUrl)
infoAttrs.emplace("immutableUrl", *res.immutableUrl);
// TODO: fall back to cached value if download fails.
getCache()->add(
*store,
inAttrs,
infoAttrs,
*unpackedStorePath,
locked);
/* Note: if the download is cached, `importTarball()` will receive
no data, which causes it to import an empty tarball. */
TarArchive archive { *source };
auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
return {
.storePath = std::move(*unpackedStorePath),
.lastModified = lastModified,
.immutableUrl = res.immutableUrl,
};
auto res(_res->lock());
Attrs infoAttrs;
if (res->cached) {
/* The server says that the previously downloaded version is
still current. */
infoAttrs = cached->infoAttrs;
} else {
infoAttrs.insert_or_assign("etag", res->etag);
infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev());
infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified));
if (res->immutableUrl)
infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl);
}
/* Insert a cache entry for every URL in the redirect chain. */
for (auto & url : res->urls) {
inAttrs.insert_or_assign("url", url);
getCache()->upsert(inAttrs, infoAttrs);
}
// FIXME: add a cache entry for immutableUrl? That could allow
// cache poisoning.
return attrsToResult(infoAttrs);
}
// An input scheme corresponding to a curl-downloadable resource.
@ -198,6 +205,8 @@ struct CurlInputScheme : InputScheme
virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0;
static const std::set<std::string> specialParams;
std::optional<Input> inputFromURL(const ParsedURL & _url, bool requireTree) const override
{
if (!isValidURL(_url, requireTree))
@ -220,8 +229,17 @@ struct CurlInputScheme : InputScheme
if (auto n = string2Int<uint64_t>(*i))
input.attrs.insert_or_assign("revCount", *n);
url.query.erase("rev");
url.query.erase("revCount");
if (auto i = get(url.query, "lastModified"))
if (auto n = string2Int<uint64_t>(*i))
input.attrs.insert_or_assign("lastModified", *n);
/* The URL query parameters serve two roles: specifying fetch
settings for Nix itself, and arbitrary data as part of the
HTTP request. Now that we've processed the Nix-specific
attributes above, remove them so we don't also send them as
part of the HTTP request. */
for (auto & param : allowedAttrs())
url.query.erase(param);
input.attrs.insert_or_assign("type", std::string { schemeName() });
input.attrs.insert_or_assign("url", url.to_string());
@ -260,6 +278,11 @@ struct CurlInputScheme : InputScheme
url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
return url;
}
bool isLocked(const Input & input) const override
{
return (bool) input.getNarHash();
}
};
struct FileInputScheme : CurlInputScheme
@ -275,10 +298,24 @@ struct FileInputScheme : CurlInputScheme
: (!requireTree && !hasTarballExtension(url.path)));
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
{
auto input(_input);
/* Unlike TarballInputScheme, this stores downloaded files in
the Nix store directly, since there is little deduplication
benefit in using the Git cache for single big files like
tarballs. */
auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false);
return {std::move(file.storePath), input};
auto narHash = store->queryPathInfo(file.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
auto accessor = makeStorePathAccessor(store, file.storePath);
accessor->setPathDisplay("«" + input.to_string() + "»");
return {accessor, input};
}
};
@ -296,11 +333,13 @@ struct TarballInputScheme : CurlInputScheme
: (requireTree || hasTarballExtension(url.path)));
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
{
Input input(_input);
auto url = getStrAttr(input.attrs, "url");
auto result = downloadTarball(store, url, input.getName(), false);
auto input(_input);
auto result = downloadTarball(getStrAttr(input.attrs, "url"), {});
result.accessor->setPathDisplay("«" + input.to_string() + "»");
if (result.immutableUrl) {
auto immutableInput = Input::fromURL(*result.immutableUrl);
@ -314,7 +353,10 @@ struct TarballInputScheme : CurlInputScheme
if (result.lastModified && !input.attrs.contains("lastModified"))
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
return {result.storePath, std::move(input)};
input.attrs.insert_or_assign("narHash",
getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true));
return {result.accessor, input};
}
};

12
src/libfetchers/tarball.hh
View file

@ -2,11 +2,13 @@
#include "types.hh"
#include "path.hh"
#include "hash.hh"
#include <optional>
namespace nix {
class Store;
struct InputAccessor;
}
namespace nix::fetchers {
@ -28,16 +30,18 @@ DownloadFileResult downloadFile(
struct DownloadTarballResult
{
StorePath storePath;
Hash treeHash;
time_t lastModified;
std::optional<std::string> immutableUrl;
ref<InputAccessor> accessor;
};
/**
* Download and import a tarball into the Git cache. The result is the
* Git tree hash of the root directory.
*/
DownloadTarballResult downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
}

4
src/libmain/shared.cc
View file

@ -340,7 +340,7 @@ int handleExceptions(const std::string & programName, std::function<void()> fun)
return 1;
} catch (BaseError & e) {
logError(e.info());
return e.status;
return e.info().status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
@ -408,6 +408,4 @@ PrintFreed::~PrintFreed()
showBytes(results.bytesFreed));
}
Exit::~Exit() { }
}

10
src/libmain/shared.hh
View file

@ -7,6 +7,7 @@
#include "common-args.hh"
#include "path.hh"
#include "derived-path.hh"
#include "exit.hh"
#include <signal.h>
@ -15,15 +16,6 @@
namespace nix {
class Exit : public std::exception
{
public:
int status;
Exit() : status(0) { }
Exit(int status) : status(status) { }
virtual ~Exit();
};
int handleExceptions(const std::string & programName, std::function<void()> fun);
/**

32
src/libstore/binary-cache-store.cc
View file

@ -235,14 +235,14 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
std::regex regex2("^[0-9a-f]{38}\\.debug$");
for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) {
auto dir = buildIdDir + s1;
auto dir = buildIdDir / s1;
if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory
|| !std::regex_match(s1, regex1))
continue;
for (auto & [s2, _type] : narAccessor->readDirectory(dir)) {
auto debugPath = dir + s2;
auto debugPath = dir / s2;
if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular
|| !std::regex_match(s2, regex2))
@ -305,7 +305,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
StorePath BinaryCacheStore::addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method,
FileSerialisationMethod dumpMethod,
ContentAddressMethod hashMethod,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair)
@ -313,17 +314,27 @@ StorePath BinaryCacheStore::addToStoreFromDump(
std::optional<Hash> caHash;
std::string nar;
// Calculating Git hash from NAR stream not yet implemented. May not
// be possible to implement in single-pass if the NAR is in an
// inconvenient order. Could fetch after uploading, however.
if (hashMethod.getFileIngestionMethod() == FileIngestionMethod::Git)
unsupported("addToStoreFromDump");
if (auto * dump2p = dynamic_cast<StringSource *>(&dump)) {
auto & dump2 = *dump2p;
// Hack, this gives us a "replayable" source so we can compute
// multiple hashes more easily.
caHash = hashString(HashAlgorithm::SHA256, dump2.s);
switch (method.getFileIngestionMethod()) {
case FileIngestionMethod::Recursive:
//
// Only calculate if the dump is in the right format, however.
if (static_cast<FileIngestionMethod>(dumpMethod) == hashMethod.getFileIngestionMethod())
caHash = hashString(HashAlgorithm::SHA256, dump2.s);
switch (dumpMethod) {
case FileSerialisationMethod::Recursive:
// The dump is already NAR in this case, just use it.
nar = dump2.s;
break;
case FileIngestionMethod::Flat:
case FileSerialisationMethod::Flat:
{
// The dump is Flat, so we need to convert it to NAR with a
// single file.
StringSink s;
@ -331,10 +342,11 @@ StorePath BinaryCacheStore::addToStoreFromDump(
nar = std::move(s.s);
break;
}
}
} else {
// Otherwise, we have to do th same hashing as NAR so our single
// hash will suffice for both purposes.
if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
unsupported("addToStoreFromDump");
}
StringSource narDump { nar };
@ -349,7 +361,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(
*this,
name,
ContentAddressWithReferences::fromParts(
method,
hashMethod,
caHash ? *caHash : nar.first,
{
.others = references,
@ -450,7 +462,7 @@ StorePath BinaryCacheStore::addToStore(
non-recursive+sha256 so we can just use the default
implementation of this method in terms of addToStoreFromDump. */
auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter);
auto source = sinkToSource([&](Sink & sink) {
accessor.dumpPath(path, sink, filter);

5
src/libstore/binary-cache-store.hh
View file

@ -125,7 +125,8 @@ public:
StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method,
FileSerialisationMethod dumpMethod,
ContentAddressMethod hashMethod,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair) override;
@ -147,7 +148,7 @@ public:
void narFromPath(const StorePath & path, Sink & sink) override;
ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;

20
src/libstore/build/derivation-goal.cc
View file

@ -708,7 +708,7 @@ void DerivationGoal::tryToBuild()
if (!outputLocks.lockPaths(lockFiles, "", false)) {
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
fmt("waiting for lock on %s", Magenta(showPaths(lockFiles))));
worker.waitForAWhile(shared_from_this());
return;
}
@ -762,7 +762,7 @@ void DerivationGoal::tryToBuild()
the wake-up timeout expires. */
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
worker.waitForAWhile(shared_from_this());
outputLocks.unlock();
return;
@ -780,9 +780,13 @@ void DerivationGoal::tryToBuild()
void DerivationGoal::tryLocalBuild() {
throw Error(
"unable to build with a primary store that isn't a local store; "
"either pass a different '--store' or enable remote builds."
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
R"(
Unable to build with a primary store that isn't a local store;
either pass a different '--store' or enable remote builds.
For more information check 'man nix.conf' and search for '/machines'.
)"
);
}
@ -891,7 +895,7 @@ void runPostBuildHook(
if (hook == "")
return;
Activity act(logger, lvlInfo, actPostBuildHook,
Activity act(logger, lvlTalkative, actPostBuildHook,
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{store.printStorePath(drvPath)});
PushActivity pact(act.id);
@ -987,7 +991,7 @@ void DerivationGoal::buildDone()
diskFull |= cleanupDecideWhetherDiskFull();
auto msg = fmt("builder for '%s' %s",
yellowtxt(worker.store.printStorePath(drvPath)),
Magenta(worker.store.printStorePath(drvPath)),
statusToString(status));
if (!logger->isVerbose() && !logTail.empty()) {
@ -1523,7 +1527,7 @@ void DerivationGoal::done(
outputLocks.unlock();
buildResult.status = status;
if (ex)
buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));
buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg));
if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true;
if (buildResult.status == BuildResult::PermanentFailure)

4
src/libstore/build/entry-points.cc
View file

@ -33,7 +33,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
}
if (failed.size() == 1 && ex) {
ex->status = worker.failingExitStatus();
ex->withExitStatus(worker.failingExitStatus());
throw std::move(*ex);
} else if (!failed.empty()) {
if (ex) logError(ex->info());
@ -104,7 +104,7 @@ void Store::ensurePath(const StorePath & path)
if (goal->exitCode != Goal::ecSuccess) {
if (goal->ex) {
goal->ex->status = worker.failingExitStatus();
goal->ex->withExitStatus(worker.failingExitStatus());
throw std::move(*goal->ex);
} else
throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));

56
src/libstore/build/local-derivation-goal.cc
View file

@ -8,6 +8,7 @@
#include "finally.hh"
#include "util.hh"
#include "archive.hh"
#include "git.hh"
#include "compression.hh"
#include "daemon.hh"
#include "topo-sort.hh"
@ -92,7 +93,7 @@ void handleDiffHook(
} catch (Error & error) {
ErrorInfo ei = error.info();
// FIXME: wrap errors.
ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str());
ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str());
logError(ei);
}
}
@ -232,7 +233,7 @@ void LocalDerivationGoal::tryLocalBuild()
if (!buildUser) {
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath))));
worker.waitForAWhile(shared_from_this());
return;
}
@ -1311,12 +1312,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method,
FileSerialisationMethod dumpMethod,
ContentAddressMethod hashMethod,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair) override
{
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair);
auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair);
goal.addDependency(path);
return path;
}
@ -2130,16 +2132,17 @@ void LocalDerivationGoal::runChild()
try {
logger = makeJSONLogger(*logger);
BasicDerivation & drv2(*drv);
for (auto & e : drv2.env)
e.second = rewriteStrings(e.second, inputRewrites);
std::map<std::string, Path> outputs;
for (auto & e : drv->outputs)
outputs.insert_or_assign(e.first,
worker.store.printStorePath(scratchOutputs.at(e.first)));
if (drv->builder == "builtin:fetchurl")
builtinFetchurl(drv2, netrcData);
builtinFetchurl(*drv, outputs, netrcData);
else if (drv->builder == "builtin:buildenv")
builtinBuildenv(drv2);
builtinBuildenv(*drv, outputs);
else if (drv->builder == "builtin:unpack-channel")
builtinUnpackChannel(drv2);
builtinUnpackChannel(*drv, outputs);
else
throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8));
_exit(0);
@ -2456,15 +2459,28 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
rewriteOutput(outputRewrites);
/* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath->hashPart() };
auto got = ({
HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
auto got = [&]{
PosixSourceAccessor accessor;
dumpPath(
accessor, CanonPath { actualPath },
caSink,
outputHash.method.getFileIngestionMethod());
caSink.finish().first;
});
auto fim = outputHash.method.getFileIngestionMethod();
switch (fim) {
case FileIngestionMethod::Flat:
case FileIngestionMethod::Recursive:
{
HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
auto fim = outputHash.method.getFileIngestionMethod();
dumpPath(
accessor, CanonPath { actualPath },
caSink,
(FileSerialisationMethod) fim);
return caSink.finish().first;
}
case FileIngestionMethod::Git: {
return git::dumpHash(
outputHash.hashAlgo, accessor,
CanonPath { tmpDir + "/tmp" }).hash;
}
}
}();
ValidPathInfo newInfo0 {
worker.store,
@ -2490,7 +2506,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
PosixSourceAccessor accessor;
HashResult narHashAndSize = hashPath(
accessor, CanonPath { actualPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
FileSerialisationMethod::Recursive, HashAlgorithm::SHA256);
newInfo0.narHash = narHashAndSize.first;
newInfo0.narSize = narHashAndSize.second;
}
@ -2514,7 +2530,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
PosixSourceAccessor accessor;
HashResult narHashAndSize = hashPath(
accessor, CanonPath { actualPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
FileSerialisationMethod::Recursive, HashAlgorithm::SHA256);
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
newInfo0.narSize = narHashAndSize.second;
auto refs = rewriteRefs();

2
src/libstore/build/local-derivation-goal.hh
View file

@ -106,7 +106,7 @@ struct LocalDerivationGoal : public DerivationGoal
RedirectedOutputs redirectedOutputs;
/**
* The outputs paths used during the build.
* The output paths used during the build.
*
* - Input-addressed derivations or fixed content-addressed outputs are
* sometimes built when some of their outputs already exist, and can not

26
src/libstore/build/worker.cc
View file

@ -331,13 +331,23 @@ void Worker::run(const Goals & _topGoals)
if (awake.empty() && 0U == settings.maxBuildJobs)
{
if (getMachines().empty())
throw Error("unable to start any build; either increase '--max-jobs' "
"or enable remote builds."
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
throw Error(
R"(
Unable to start any build;
either increase '--max-jobs' or enable remote builds.
For more information run 'man nix.conf' and search for '/machines'.
)"
);
else
throw Error("unable to start any build; remote machines may not have "
"all required system features."
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
throw Error(
R"(
Unable to start any build;
remote machines may not have all required system features.
For more information run 'man nix.conf' and search for '/machines'.
)"
);
}
assert(!awake.empty());
@ -519,11 +529,11 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
HashResult current = hashPath(
Hash current = hashPath(
*store.getFSAccessor(), CanonPath { store.printStorePath(path) },
FileIngestionMethod::Recursive, info->narHash.algo);
Hash nullHash(HashAlgorithm::SHA256);
res = info->narHash == nullHash || info->narHash == current.first;
res = info->narHash == nullHash || info->narHash == current;
}
pathContentsGoodCache.insert_or_assign(path, res);
if (!res)

10
src/libstore/builtins.hh
View file

@ -6,7 +6,13 @@
namespace nix {
// TODO: make pluggable.
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
void builtinUnpackChannel(const BasicDerivation & drv);
void builtinFetchurl(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs,
const std::string & netrcData);
void builtinUnpackChannel(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs);
}

6
src/libstore/builtins/buildenv.cc
View file

@ -161,7 +161,9 @@ void buildProfile(const Path & out, Packages && pkgs)
debug("created %d symlinks in user environment", state.symlinks);
}
void builtinBuildenv(const BasicDerivation & drv)
void builtinBuildenv(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs)
{
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
@ -169,7 +171,7 @@ void builtinBuildenv(const BasicDerivation & drv)
return i->second;
};
Path out = getAttr("out");
auto out = outputs.at("out");
createDirs(out);
/* Convert the stuff we get from the environment back into a

4
src/libstore/builtins/buildenv.hh
View file

@ -45,6 +45,8 @@ typedef std::vector<Package> Packages;
void buildProfile(const Path & out, Packages && pkgs);
void builtinBuildenv(const BasicDerivation & drv);
void builtinBuildenv(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs);
}

27
src/libstore/builtins/fetchurl.cc
View file

@ -6,7 +6,10 @@
namespace nix {
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
void builtinFetchurl(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs,
const std::string & netrcData)
{
/* Make the host's netrc data available. Too bad curl requires
this to be stored in a file. It would be nice if we could just
@ -16,14 +19,15 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
writeFile(settings.netrcFile, netrcData, 0600);
}
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;
};
auto out = get(drv.outputs, "out");
if (!out)
throw Error("'builtin:fetchurl' requires an 'out' output");
Path storePath = getAttr("out");
auto mainUrl = getAttr("url");
if (!(drv.type().isFixed() || drv.type().isImpure()))
throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation");
auto storePath = outputs.at("out");
auto mainUrl = drv.env.at("url");
bool unpack = getOr(drv.env, "unpack", "") == "1";
/* Note: have to use a fresh fileTransfer here because we're in
@ -59,13 +63,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
};
/* Try the hashed mirrors first. */
if (getAttr("outputHashMode") == "flat")
auto dof = std::get_if<DerivationOutput::CAFixed>(&out->raw);
if (dof && dof->ca.method.getFileIngestionMethod() == FileIngestionMethod::Flat)
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
std::optional<HashAlgorithm> ht = parseHashAlgoOpt(getAttr("outputHashAlgo"));
Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false));
fetch(hashedMirror + printHashAlgo(dof->ca.hash.algo) + "/" + dof->ca.hash.to_string(HashFormat::Base16, false));
return;
} catch (Error & e) {
debug(e.what());

6
src/libstore/builtins/unpack-channel.cc
View file

@ -3,7 +3,9 @@
namespace nix {
void builtinUnpackChannel(const BasicDerivation & drv)
void builtinUnpackChannel(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs)
{
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
@ -11,7 +13,7 @@ void builtinUnpackChannel(const BasicDerivation & drv)
return i->second;
};
Path out = getAttr("out");
auto out = outputs.at("out");
auto channelName = getAttr("channelName");
auto src = getAttr("src");

48
src/libstore/content-address.cc
View file

@ -4,22 +4,44 @@
namespace nix {
std::string makeFileIngestionPrefix(FileIngestionMethod m)
std::string_view makeFileIngestionPrefix(FileIngestionMethod m)
{
switch (m) {
case FileIngestionMethod::Flat:
return "";
case FileIngestionMethod::Recursive:
return "r:";
case FileIngestionMethod::Git:
experimentalFeatureSettings.require(Xp::GitHashing);
return "git:";
default:
throw Error("impossible, caught both cases");
}
}
std::string ContentAddressMethod::renderPrefix() const
std::string_view ContentAddressMethod::render() const
{
return std::visit(overloaded {
[](TextIngestionMethod) -> std::string { return "text:"; },
[](TextIngestionMethod) -> std::string_view { return "text"; },
[](FileIngestionMethod m2) {
/* Not prefixed for back compat with things that couldn't produce text before. */
return renderFileIngestionMethod(m2);
},
}, raw);
}
ContentAddressMethod ContentAddressMethod::parse(std::string_view m)
{
if (m == "text")
return TextIngestionMethod {};
else
return parseFileIngestionMethod(m);
}
std::string_view ContentAddressMethod::renderPrefix() const
{
return std::visit(overloaded {
[](TextIngestionMethod) -> std::string_view { return "text:"; },
[](FileIngestionMethod m2) {
/* Not prefixed for back compat with things that couldn't produce text before. */
return makeFileIngestionPrefix(m2);
@ -32,13 +54,17 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
if (splitPrefix(m, "r:")) {
return FileIngestionMethod::Recursive;
}
else if (splitPrefix(m, "git:")) {
experimentalFeatureSettings.require(Xp::GitHashing);
return FileIngestionMethod::Git;
}
else if (splitPrefix(m, "text:")) {
return TextIngestionMethod {};
}
return FileIngestionMethod::Flat;
}
std::string ContentAddressMethod::render(HashAlgorithm ha) const
std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const
{
return std::visit(overloaded {
[&](const TextIngestionMethod & th) {
@ -92,10 +118,10 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
}
auto parseHashAlgorithm_ = [&](){
auto hashTypeRaw = splitPrefixTo(rest, ':');
if (!hashTypeRaw)
auto hashAlgoRaw = splitPrefixTo(rest, ':');
if (!hashAlgoRaw)
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw);
HashAlgorithm hashAlgo = parseHashAlgo(*hashAlgoRaw);
return hashAlgo;
};
@ -112,6 +138,10 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
auto method = FileIngestionMethod::Flat;
if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive;
else if (splitPrefix(rest, "git:")) {
experimentalFeatureSettings.require(Xp::GitHashing);
method = FileIngestionMethod::Git;
}
HashAlgorithm hashAlgo = parseHashAlgorithm_();
return {
std::move(method),
@ -133,7 +163,7 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
};
}
std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parse(std::string_view caMethod)
std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parseWithAlgo(std::string_view caMethod)
{
std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference
@ -155,7 +185,7 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
std::string ContentAddress::printMethodAlgo() const
{
return method.renderPrefix()
return std::string { method.renderPrefix() }
+ printHashAlgo(hash.algo);
}

28
src/libstore/content-address.hh
View file

@ -36,7 +36,7 @@ struct TextIngestionMethod : std::monostate { };
* Compute the prefix to the hash algorithm which indicates how the
* files were ingested.
*/
std::string makeFileIngestionPrefix(FileIngestionMethod m);
std::string_view makeFileIngestionPrefix(FileIngestionMethod m);
/**
* An enumeration of all the ways we can content-address store objects.
@ -59,6 +59,20 @@ struct ContentAddressMethod
MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod);
/**
* Parse a content addressing method (name).
*
* The inverse of `render`.
*/
static ContentAddressMethod parse(std::string_view rawCaMethod);
/**
* Render a content addressing method (name).
*
* The inverse of `parse`.
*/
std::string_view render() const;
/**
* Parse the prefix tag which indicates how the files
* were ingested, with the fixed output case not prefixed for back
@ -74,20 +88,20 @@ struct ContentAddressMethod
*
* The rough inverse of `parsePrefix()`.
*/
std::string renderPrefix() const;
std::string_view renderPrefix() const;
/**
* Parse a content addressing method and hash type.
* Parse a content addressing method and hash algorithm.
*/
static std::pair<ContentAddressMethod, HashAlgorithm> parse(std::string_view rawCaMethod);
static std::pair<ContentAddressMethod, HashAlgorithm> parseWithAlgo(std::string_view rawCaMethod);
/**
* Render a content addressing method and hash type in a
* Render a content addressing method and hash algorithm in a
* nicer way, prefixing both cases.
*
* The rough inverse of `parse()`.
*/
std::string render(HashAlgorithm ht) const;
std::string renderWithAlgo(HashAlgorithm ha) const;
/**
* Get the underlying way to content-address file system objects.
@ -113,7 +127,7 @@ struct ContentAddressMethod
* ‘text:sha256:<sha256 hash of file contents>’
*
* - `FixedIngestionMethod`:
* ‘fixed:<r?>:<hash type>:<hash of file contents>’
* ‘fixed:<r?>:<hash algorithm>:<hash of file contents>’
*/
struct ContentAddress
{

58
src/libstore/daemon.cc
View file

@ -13,6 +13,7 @@
#include "archive.hh"
#include "derivations.hh"
#include "args.hh"
#include "git.hh"
namespace nix::daemon {
@ -119,7 +120,7 @@ struct TunnelLogger : public Logger
if (GET_PROTOCOL_MINOR(clientVersion) >= 26) {
to << STDERR_ERROR << *ex;
} else {
to << STDERR_ERROR << ex->what() << ex->status;
to << STDERR_ERROR << ex->what() << ex->info().status;
}
}
}
@ -400,11 +401,23 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork();
auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
auto hashAlgo = hashAlgo_; // work around clang bug
auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr);
FramedSource source(from);
FileSerialisationMethod dumpMethod;
switch (contentAddressMethod.getFileIngestionMethod()) {
case FileIngestionMethod::Flat:
dumpMethod = FileSerialisationMethod::Flat;
break;
case FileIngestionMethod::Recursive:
dumpMethod = FileSerialisationMethod::Recursive;
break;
case FileIngestionMethod::Git:
// Use NAR; Git is not a serialization method
dumpMethod = FileSerialisationMethod::Recursive;
break;
}
// TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair);
auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair);
return store->queryPathInfo(path);
}();
logger->stopWork();
@ -430,30 +443,23 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
hashAlgo = parseHashAlgo(hashAlgoRaw);
}
// Old protocol always sends NAR, regardless of hashing method
auto dumpSource = sinkToSource([&](Sink & saved) {
if (method == FileIngestionMethod::Recursive) {
/* We parse the NAR dump through into `saved` unmodified,
so why all this extra work? We still parse the NAR so
that we aren't sending arbitrary data to `saved`
unwittingly`, and we know when the NAR ends so we don't
consume the rest of `from` and can't parse another
command. (We don't trust `addToStoreFromDump` to not
eagerly consume the entire stream it's given, past the
length of the Nar. */
TeeSource savedNARSource(from, saved);
NullFileSystemObjectSink sink; /* just parse the NAR */
parseDump(sink, savedNARSource);
} else {
/* Incrementally parse the NAR file, stripping the
metadata, and streaming the sole file we expect into
`saved`. */
RegularFileSink savedRegular { saved };
parseDump(savedRegular, from);
if (!savedRegular.regular) throw Error("regular file expected");
}
/* We parse the NAR dump through into `saved` unmodified,
so why all this extra work? We still parse the NAR so
that we aren't sending arbitrary data to `saved`
unwittingly`, and we know when the NAR ends so we don't
consume the rest of `from` and can't parse another
command. (We don't trust `addToStoreFromDump` to not
eagerly consume the entire stream it's given, past the
length of the Nar. */
TeeSource savedNARSource(from, saved);
NullFileSystemObjectSink sink; /* just parse the NAR */
parseDump(sink, savedNARSource);
});
logger->startWork();
auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
auto path = store->addToStoreFromDump(
*dumpSource, baseName, FileSerialisationMethod::Recursive, method, hashAlgo);
logger->stopWork();
to << store->printStorePath(path);
@ -485,7 +491,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto path = ({
StringSource source { s };
store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
});
logger->stopWork();
to << store->printStorePath(path);

16
src/libstore/derivations.cc
View file

@ -150,7 +150,7 @@ StorePath writeDerivation(Store & store,
})
: ({
StringSource s { contents };
store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
});
}
@ -601,7 +601,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
},
[&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo));
s += ','; printUnquotedString(s, std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo));
s += ','; printUnquotedString(s, "");
},
[&](const DerivationOutput::Deferred &) {
@ -612,7 +612,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
[&](const DerivationOutput::Impure & doi) {
// FIXME
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo));
s += ','; printUnquotedString(s, std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo));
s += ','; printUnquotedString(s, "impure");
}
}, i.second.raw);
@ -701,7 +701,7 @@ DerivationType BasicDerivation::type() const
floatingHashAlgo = dof.hashAlgo;
} else {
if (*floatingHashAlgo != dof.hashAlgo)
throw Error("all floating outputs must use the same hash type");
throw Error("all floating outputs must use the same hash algorithm");
}
},
[&](const DerivationOutput::Deferred &) {
@ -984,7 +984,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
},
[&](const DerivationOutput::CAFloating & dof) {
out << ""
<< (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo))
<< (std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo))
<< "";
},
[&](const DerivationOutput::Deferred &) {
@ -994,7 +994,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
},
[&](const DerivationOutput::Impure & doi) {
out << ""
<< (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo))
<< (std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo))
<< "impure";
},
}, i.second.raw);
@ -1221,11 +1221,11 @@ nlohmann::json DerivationOutput::toJSON(
// FIXME print refs?
},
[&](const DerivationOutput::CAFloating & dof) {
res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo);
res["hashAlgo"] = std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo);
},
[&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) {
res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo);
res["hashAlgo"] = std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo);
res["impure"] = true;
},
}, raw);

3
src/libstore/dummy-store.cc
View file

@ -61,7 +61,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
virtual StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method = FileIngestionMethod::Recursive,
FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
const StorePathSet & references = StorePathSet(),
RepairFlag repair = NoRepair) override

33
src/libstore/filetransfer.cc
View file

@ -106,6 +106,8 @@ struct curlFileTransfer : public FileTransfer
this->result.data.append(data);
})
{
result.urls.push_back(request.uri);
requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
if (!request.expectedETag.empty())
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
@ -182,6 +184,14 @@ struct curlFileTransfer : public FileTransfer
return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
}
void appendCurrentUrl()
{
char * effectiveUriCStr = nullptr;
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
if (effectiveUriCStr && *result.urls.rbegin() != effectiveUriCStr)
result.urls.push_back(effectiveUriCStr);
}
size_t headerCallback(void * contents, size_t size, size_t nmemb)
{
size_t realSize = size * nmemb;
@ -196,6 +206,7 @@ struct curlFileTransfer : public FileTransfer
statusMsg = trim(match.str(1));
acceptRanges = false;
encoding = "";
appendCurrentUrl();
} else {
auto i = line.find(':');
@ -360,14 +371,11 @@ struct curlFileTransfer : public FileTransfer
{
auto httpStatus = getHTTPStatus();
char * effectiveUriCStr = nullptr;
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
if (effectiveUriCStr)
result.effectiveUri = effectiveUriCStr;
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
request.verb(), request.uri, code, httpStatus, result.bodySize);
appendCurrentUrl();
if (decompressionSink) {
try {
decompressionSink->finish();
@ -779,7 +787,10 @@ FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
return enqueueFileTransfer(request).get();
}
void FileTransfer::download(FileTransferRequest && request, Sink & sink)
void FileTransfer::download(
FileTransferRequest && request,
Sink & sink,
std::function<void(FileTransferResult)> resultCallback)
{
/* Note: we can't call 'sink' via request.dataCallback, because
that would cause the sink to execute on the fileTransfer
@ -829,11 +840,13 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
};
enqueueFileTransfer(request,
{[_state](std::future<FileTransferResult> fut) {
{[_state, resultCallback{std::move(resultCallback)}](std::future<FileTransferResult> fut) {
auto state(_state->lock());
state->quit = true;
try {
fut.get();
auto res = fut.get();
if (resultCallback)
resultCallback(std::move(res));
} catch (...) {
state->exc = std::current_exception();
}
@ -882,12 +895,12 @@ template<typename... Args>
FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args)
: Error(args...), error(error), response(response)
{
const auto hf = hintfmt(args...);
const auto hf = HintFmt(args...);
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
// to print different messages for different verbosity levels. For now
// we add some heuristics for detecting when we want to show the response.
if (response && (response->size() < 1024 || response->find("<html>") != std::string::npos))
err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response));
err.msg = HintFmt("%1%\n\nresponse body:\n\n%2%", Uncolored(hf.str()), chomp(*response));
else
err.msg = hf;
}

33
src/libstore/filetransfer.hh
View file

@ -75,14 +75,34 @@ struct FileTransferRequest
struct FileTransferResult
{
/**
* Whether this is a cache hit (i.e. the ETag supplied in the
* request is still valid). If so, `data` is empty.
*/
bool cached = false;
/**
* The ETag of the object.
*/
std::string etag;
std::string effectiveUri;
/**
* All URLs visited in the redirect chain.
*/
std::vector<std::string> urls;
/**
* The response body.
*/
std::string data;
uint64_t bodySize = 0;
/* An "immutable" URL for this resource (i.e. one whose contents
will never change), as returned by the `Link: <url>;
rel="immutable"` header. */
/**
* An "immutable" URL for this resource (i.e. one whose contents
* will never change), as returned by the `Link: <url>;
* rel="immutable"` header.
*/
std::optional<std::string> immutableUrl;
};
@ -116,7 +136,10 @@ struct FileTransfer
* Download a file, writing its data to a sink. The sink will be
* invoked on the thread of the caller.
*/
void download(FileTransferRequest && request, Sink & sink);
void download(
FileTransferRequest && request,
Sink & sink,
std::function<void(FileTransferResult)> resultCallback = {});
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
};

160
src/libstore/globals.hh
View file

@ -180,14 +180,21 @@ public:
getDefaultCores(),
"cores",
R"(
Sets the value of the `NIX_BUILD_CORES` environment variable in the
invocation of builders. Builders can use this variable at their
discretion to control the maximum amount of parallelism. For
instance, in Nixpkgs, if the derivation attribute
`enableParallelBuilding` is set to `true`, the builder passes the
`-jN` flag to GNU Make. It can be overridden using the `--cores`
command line switch and defaults to `1`. The value `0` means that
the builder should use all available CPU cores in the system.
Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation.
The `builder` executable can use this variable to control its own maximum amount of parallelism.
<!--
FIXME(@fricklerhandwerk): I don't think this should even be mentioned here.
A very generic example using `derivation` and `xargs` may be more appropriate to explain the mechanism.
Using `mkDerivation` as an example requires being aware of that there are multiple independent layers that are completely opaque here.
-->
For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it will pass the `-j${NIX_BUILD_CORES}` flag to GNU Make.
The value `0` means that the `builder` should use all available CPU cores in the system.
> **Note**
>
> The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting.
)",
{"build-cores"},
// Don't document the machine-specific default value
@ -270,9 +277,121 @@ public:
Setting<std::string> builders{
this, "@" + nixConfDir + "/machines", "builders",
R"(
A semicolon-separated list of build machines.
For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md)
)"};
A semicolon- or newline-separated list of build machines.
In addition to the [usual ways of setting configuration options](@docroot@/command-ref/conf-file.md), the value can be read from a file by prefixing its absolute path with `@`.
> **Example**
>
> This is the default setting:
>
> ```
> builders = @/etc/nix/machines
> ```
Each machine specification consists of the following elements, separated by spaces.
Only the first element is required.
To leave a field at its default, set it to `-`.
1. The URI of the remote store in the format `ssh://[username@]hostname`.
> **Example**
>
> `ssh://nix@mac`
For backward compatibility, `ssh://` may be omitted.
The hostname may be an alias defined in `~/.ssh/config`.
2. A comma-separated list of [Nix system types](@docroot@/contributing/hacking.md#system-type).
If omitted, this defaults to the local platform type.
> **Example**
>
> `aarch64-darwin`
It is possible for a machine to support multiple platform types.
> **Example**
>
> `i686-linux,x86_64-linux`
3. The SSH identity file to be used to log in to the remote machine.
If omitted, SSH will use its regular identities.
> **Example**
>
> `/home/user/.ssh/id_mac`
4. The maximum number of builds that Nix will execute in parallel on the machine.
Typically this should be equal to the number of CPU cores.
5. The “speed factor”, indicating the relative speed of the machine as a positive integer.
If there are multiple machines of the right type, Nix will prefer the fastest, taking load into account.
6. A comma-separated list of supported [system features](#conf-system-features).
A machine will only be used to build a derivation if all the features in the derivation's [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute are supported by that machine.
7. A comma-separated list of required [system features](#conf-system-features).
A machine will only be used to build a derivation if all of the machine’s required features appear in the derivation’s [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute.
8. The (base64-encoded) public host key of the remote machine.
If omitted, SSH will use its regular `known_hosts` file.
The value for this field can be obtained via `base64 -w0`.
> **Example**
>
> Multiple builders specified on the command line:
>
> ```console
> --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
> ```
> **Example**
>
> This specifies several machines that can perform `i686-linux` builds:
>
> ```
> nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 1 kvm
> nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 2
> nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 1 2 kvm benchmark
> ```
>
> However, `poochie` will only build derivations that have the attribute
>
> ```nix
> requiredSystemFeatures = [ "benchmark" ];
> ```
>
> or
>
> ```nix
> requiredSystemFeatures = [ "benchmark" "kvm" ];
> ```
>
> `itchy` cannot do builds that require `kvm`, but `scratchy` does support such builds.
> For regular builds, `itchy` will be preferred over `scratchy` because it has a higher speed factor.
For Nix to use substituters, the calling user must be in the [`trusted-users`](#conf-trusted-users) list.
> **Note**
>
> A build machine must be accessible via SSH and have Nix installed.
> `nix` must be available in `$PATH` for the user connecting over SSH.
> **Warning**
>
> If you are building via the Nix daemon (default), the Nix daemon user account on the local machine (that is, `root`) requires access to a user account on the remote machine (not necessarily `root`).
>
> If you can’t or don’t want to configure `root` to be able to access the remote machine, set [`store`](#conf-store) to any [local store](@docroot@/store/types/local-store.html), e.g. by passing `--store /tmp` to the command on the local machine.
To build only on remote machines and disable local builds, set [`max-jobs`](#conf-max-jobs) to 0.
If you want the remote machines to use substituters, set [`builders-use-substitutes`](#conf-builders-use-substituters) to `true`.
)",
{}, false};
Setting<bool> alwaysAllowSubstitutes{
this, false, "always-allow-substitutes",
@ -793,10 +912,17 @@ public:
Setting<unsigned int> ttlNegativeNarInfoCache{
this, 3600, "narinfo-cache-negative-ttl",
R"(
The TTL in seconds for negative lookups. If a store path is queried
from a substituter but was not found, there will be a negative
lookup cached in the local disk cache database for the specified
duration.
The TTL in seconds for negative lookups.
If a store path is queried from a [substituter](#conf-substituters) but was not found, there will be a negative lookup cached in the local disk cache database for the specified duration.
Set to `0` to force updating the lookup cache.
To wipe the lookup cache completely:
```shell-session
$ rm $HOME/.cache/nix/binary-cache-v*.sqlite*
# rm /root/.cache/nix/binary-cache-v*.sqlite*
```
)"};
Setting<unsigned int> ttlPositiveNarInfoCache{
@ -968,8 +1094,8 @@ public:
this, {}, "hashed-mirrors",
R"(
A list of web servers used by `builtins.fetchurl` to obtain files by
hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
download the file from *hashed-mirror*/*ht*/*h*. This allows files to
hash. Given a hash algorithm *ha* and a base-16 hash *h*, Nix will try to
download the file from *hashed-mirror*/*ha*/*h*. This allows files to
be downloaded even if they have disappeared from their original URI.
For example, given an example mirror `http://tarballs.nixos.org/`,
when building the derivation

3
src/libstore/legacy-ssh-store.hh
View file

@ -72,7 +72,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
virtual StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method = FileIngestionMethod::Recursive,
FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
const StorePathSet & references = StorePathSet(),
RepairFlag repair = NoRepair) override

2
src/libstore/local-fs-store.cc
View file

@ -28,7 +28,7 @@ struct LocalStoreAccessor : PosixSourceAccessor
auto [storePath, rest] = store->toStorePath(path.abs());
if (requireValidPath && !store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest);
return CanonPath(store->getRealStoreDir()) / storePath.to_string() / CanonPath(rest);
}
std::optional<Stat> maybeLstat(const CanonPath & path) override

2
src/libstore/local-fs-store.hh
View file

@ -43,7 +43,7 @@ public:
LocalFSStore(const Params & params);
void narFromPath(const StorePath & path, Sink & sink) override;
ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
/**
* Creates symlink from the `gcRoot` to the `storePath` and

78
src/libstore/local-store.cc
View file

@ -1,5 +1,6 @@
#include "local-store.hh"
#include "globals.hh"
#include "git.hh"
#include "archive.hh"
#include "pathlocks.hh"
#include "worker-protocol.hh"
@ -1103,19 +1104,29 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (info.ca) {
auto & specified = *info.ca;
auto actualHash = ({
HashModuloSink caSink {
specified.hash.algo,
std::string { info.path.hashPart() },
};
PosixSourceAccessor accessor;
dumpPath(
*getFSAccessor(false),
CanonPath { printStorePath(info.path) },
caSink,
specified.method.getFileIngestionMethod());
auto accessor = getFSAccessor(false);
CanonPath path { printStorePath(info.path) };
Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++
auto fim = specified.method.getFileIngestionMethod();
switch (fim) {
case FileIngestionMethod::Flat:
case FileIngestionMethod::Recursive:
{
HashModuloSink caSink {
specified.hash.algo,
std::string { info.path.hashPart() },
};
dumpPath(*accessor, path, caSink, (FileSerialisationMethod) fim);
h = caSink.finish().first;
break;
}
case FileIngestionMethod::Git:
h = git::dumpHash(specified.hash.algo, *accessor, path).hash;
break;
}
ContentAddress {
.method = specified.method,
.hash = caSink.finish().first,
.hash = std::move(h),
};
});
if (specified.hash != actualHash.hash) {
@ -1143,7 +1154,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
StorePath LocalStore::addToStoreFromDump(
Source & source0,
std::string_view name,
ContentAddressMethod method,
FileSerialisationMethod dumpMethod,
ContentAddressMethod hashMethod,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair)
@ -1196,7 +1208,13 @@ StorePath LocalStore::addToStoreFromDump(
Path tempDir;
AutoCloseFD tempDirFd;
if (!inMemory) {
bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod;
/* If the methods don't match, our streaming hash of the dump is the
wrong sort, and we need to rehash. */
bool inMemoryAndDontNeedRestore = inMemory && methodsMatch;
if (!inMemoryAndDontNeedRestore) {
/* Drain what we pulled so far, and then keep on pulling */
StringSource dumpSource { dump };
ChainSource bothSource { dumpSource, source };
@ -1205,17 +1223,23 @@ StorePath LocalStore::addToStoreFromDump(
delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x";
restorePath(tempPath, bothSource, method.getFileIngestionMethod());
restorePath(tempPath, bothSource, dumpMethod);
dumpBuffer.reset();
dump = {};
}
auto [hash, size] = hashSink->finish();
auto [dumpHash, size] = hashSink->finish();
PosixSourceAccessor accessor;
auto desc = ContentAddressWithReferences::fromParts(
method,
hash,
hashMethod,
methodsMatch
? dumpHash
: hashPath(
accessor, CanonPath { tempPath },
hashMethod.getFileIngestionMethod(), hashAlgo),
{
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
@ -1241,10 +1265,20 @@ StorePath LocalStore::addToStoreFromDump(
autoGC();
if (inMemory) {
if (inMemoryAndDontNeedRestore) {
StringSource dumpSource { dump };
/* Restore from the buffer in memory. */
restorePath(realPath, dumpSource, method.getFileIngestionMethod());
auto fim = hashMethod.getFileIngestionMethod();
switch (fim) {
case FileIngestionMethod::Flat:
case FileIngestionMethod::Recursive:
restorePath(realPath, dumpSource, (FileSerialisationMethod) fim);
break;
case FileIngestionMethod::Git:
// doesn't correspond to serialization method, so
// this should be unreachable
assert(false);
}
} else {
/* Move the temporary path we restored above. */
moveFile(tempPath, realPath);
@ -1252,8 +1286,8 @@ StorePath LocalStore::addToStoreFromDump(
/* For computing the nar hash. In recursive SHA-256 mode, this
is the same as the store hash, so no need to do it again. */
auto narHash = std::pair { hash, size };
if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) {
auto narHash = std::pair { dumpHash, size };
if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) {
HashSink narSink { HashAlgorithm::SHA256 };
dumpPath(realPath, narSink);
narHash = narSink.finish();
@ -1345,7 +1379,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
PosixSourceAccessor accessor;
std::string hash = hashPath(
accessor, CanonPath { linkPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false);
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false);
if (hash != link.name) {
printError("link '%s' was modified! expected hash '%s', got '%s'",
linkPath, link.name, hash);

3
src/libstore/local-store.hh
View file

@ -180,7 +180,8 @@ public:
StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method,
FileSerialisationMethod dumpMethod,
ContentAddressMethod hashMethod,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair) override;

2
src/libstore/nar-accessor.cc
View file

@ -277,7 +277,7 @@ json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
json &res2 = obj["entries"];
for (const auto & [name, type] : accessor->readDirectory(path)) {
if (recurse) {
res2[name] = listNar(accessor, path + name, true);
res2[name] = listNar(accessor, path / name, true);
} else
res2[name] = json::object();
}

2
src/libstore/nar-info-disk-cache.cc
View file

@ -209,7 +209,7 @@ public:
{
auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority));
assert(r.next());
if (!r.next()) { abort(); }
ret.id = (int) r.getInt(0);
}

4
src/libstore/optimise-store.cc
View file

@ -151,7 +151,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
PosixSourceAccessor accessor;
hashPath(
accessor, CanonPath { path },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first;
});
debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
@ -166,7 +166,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
PosixSourceAccessor accessor;
hashPath(
accessor, CanonPath { linkPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first;
})))
{
// XXX: Consider overwriting linkPath with our valid version.

23
src/libstore/remote-store.cc
View file

@ -13,6 +13,7 @@
#include "derivations.hh"
#include "pool.hh"
#include "finally.hh"
#include "git.hh"
#include "logging.hh"
#include "callback.hh"
#include "filetransfer.hh"
@ -435,7 +436,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
conn->to
<< WorkerProto::Op::AddToStore
<< name
<< caMethod.render(hashAlgo);
<< caMethod.renderWithAlgo(hashAlgo);
WorkerProto::write(*this, *conn, references);
conn->to << repair;
@ -508,12 +509,28 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
StorePath RemoteStore::addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method,
FileSerialisationMethod dumpMethod,
ContentAddressMethod hashMethod,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair)
{
return addCAToStore(dump, name, method, hashAlgo, references, repair)->path;
FileSerialisationMethod fsm;
switch (hashMethod.getFileIngestionMethod()) {
case FileIngestionMethod::Flat:
fsm = FileSerialisationMethod::Flat;
break;
case FileIngestionMethod::Recursive:
fsm = FileSerialisationMethod::Recursive;
break;
case FileIngestionMethod::Git:
// Use NAR; Git is not a serialization method
fsm = FileSerialisationMethod::Recursive;
break;
}
if (fsm != dumpMethod)
unsupported("RemoteStore::addToStoreFromDump doesn't support this `dumpMethod` `hashMethod` combination");
return addCAToStore(dump, name, hashMethod, hashAlgo, references, repair)->path;
}

5
src/libstore/remote-store.hh
View file

@ -87,7 +87,8 @@ public:
StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method = FileIngestionMethod::Recursive,
FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
const StorePathSet & references = StorePathSet(),
RepairFlag repair = NoRepair) override;
@ -184,7 +185,7 @@ protected:
friend struct ConnectionHandle;
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
virtual void narFromPath(const StorePath & path, Sink & sink) override;

Some files were not shown because too many files have changed in this diff Show more