wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-30 11:43:15 +02:00
Code Activity

Merge remote-tracking branch 'origin/master' into pr-shell-env

Browse source
This commit is contained in:
Eelco Dolstra 2024-01-12 12:56:26 +01:00
commit 66bd1b0298
541 changed files with 10850 additions and 5364 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/libcmd/built-path.cc
View file

@ -12,9 +12,9 @@ namespace nix {
bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
{ \
const MY_TYPE* me = this; \
auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
auto fields1 = std::tie(*me->drvPath, me->FIELD); \
me = &other; \
auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
auto fields2 = std::tie(*me->drvPath, me->FIELD); \
return fields1 COMPARATOR fields2; \
}
#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
@ -80,7 +80,7 @@ SingleDerivedPath SingleBuiltPath::discardOutputPath() const
);
}
nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const
{
nlohmann::json res;
res["drvPath"] = drvPath->toJSON(store);
@ -90,7 +90,7 @@ nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
return res;
}
nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const
{
nlohmann::json res;
res["drvPath"] = drvPath->toJSON(store);
@ -100,14 +100,14 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
return res;
}
nlohmann::json SingleBuiltPath::toJSON(const Store & store) const
nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
{
return std::visit([&](const auto & buildable) {
return buildable.toJSON(store);
}, raw());
}
nlohmann::json BuiltPath::toJSON(const Store & store) const
nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
{
return std::visit([&](const auto & buildable) {
return buildable.toJSON(store);

18
src/libcmd/built-path.hh
View file

@ -14,9 +14,9 @@ struct SingleBuiltPathBuilt {
SingleDerivedPathBuilt discardOutputPath() const;
std::string to_string(const Store & store) const;
static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
nlohmann::json toJSON(const Store & store) const;
std::string to_string(const StoreDirConfig & store) const;
static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
nlohmann::json toJSON(const StoreDirConfig & store) const;
DECLARE_CMP(SingleBuiltPathBuilt);
};
@ -41,8 +41,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
SingleDerivedPath discardOutputPath() const;
static SingleBuiltPath parse(const Store & store, std::string_view);
nlohmann::json toJSON(const Store & store) const;
static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view);
nlohmann::json toJSON(const StoreDirConfig & store) const;
};
static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
@ -59,9 +59,9 @@ struct BuiltPathBuilt {
ref<SingleBuiltPath> drvPath;
std::map<std::string, StorePath> outputs;
std::string to_string(const Store & store) const;
static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
nlohmann::json toJSON(const Store & store) const;
std::string to_string(const StoreDirConfig & store) const;
static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
nlohmann::json toJSON(const StoreDirConfig & store) const;
DECLARE_CMP(BuiltPathBuilt);
};
@ -89,7 +89,7 @@ struct BuiltPath : _BuiltPathRaw {
StorePathSet outPaths() const;
RealisedPath::Set toRealisedPaths(Store & store) const;
nlohmann::json toJSON(const Store & store) const;
nlohmann::json toJSON(const StoreDirConfig & store) const;
};
typedef std::vector<BuiltPath> BuiltPaths;

14
src/libcmd/command.cc
View file

@ -1,4 +1,5 @@
#include "command.hh"
#include "markdown.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "derivations.hh"
@ -34,6 +35,19 @@ nlohmann::json NixMultiCommand::toJSON()
return MultiCommand::toJSON();
}
void NixMultiCommand::run()
{
if (!command) {
std::set<std::string> subCommandTextLines;
for (auto & [name, _] : commands)
subCommandTextLines.insert(fmt("- `%s`", name));
std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n",
commandName, concatStringsSep("\n", subCommandTextLines));
throw UsageError(renderMarkdownToTerminal(markdownError));
}
command->second->run();
}
StoreCommand::StoreCommand()
{
}

6
src/libcmd/command.hh
View file

@ -26,9 +26,13 @@ static constexpr Command::Category catNixInstallation = 102;
static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)";
struct NixMultiCommand : virtual MultiCommand, virtual Command
struct NixMultiCommand : MultiCommand, virtual Command
{
nlohmann::json toJSON() override;
using MultiCommand::MultiCommand;
virtual void run() override;
};
// For the overloaded run methods

2
src/libcmd/common-eval-args.cc
View file

@ -141,7 +141,7 @@ MixEvalArgs::MixEvalArgs()
.longName = "eval-store",
.description =
R"(
The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
)",
.category = category,

1
src/libcmd/editor-for.cc
View file

@ -1,5 +1,6 @@
#include "editor-for.hh"
#include "environment-variables.hh"
#include "source-path.hh"
namespace nix {

2
src/libcmd/editor-for.hh
View file

@ -2,7 +2,7 @@
///@file
#include "types.hh"
#include "input-accessor.hh"
#include "source-path.hh"
namespace nix {

2
src/libcmd/installable-flake.cc
View file

@ -52,7 +52,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
assert(aOutputs);
state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); });
state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));
return aOutputs->value;
}

3
src/libcmd/installable-value.cc
View file

@ -1,5 +1,6 @@
#include "installable-value.hh"
#include "eval-cache.hh"
#include "fetch-to-store.hh"
namespace nix {
@ -44,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
{
if (v.type() == nPath) {
auto storePath = v.path().fetchToStore(state->store);
auto storePath = fetchToStore(*state->store, v.path());
return {{
.path = DerivedPath::Opaque {
.path = std::move(storePath),

70
src/libcmd/installables.cc
View file

@ -47,6 +47,22 @@ MixFlakeOptions::MixFlakeOptions()
{
auto category = "Common flake-related options";
addFlag({
.longName = "recreate-lock-file",
.description = R"(
Recreate the flake's lock file from scratch.
> **DEPRECATED**
>
> Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
)",
.category = category,
.handler = {[&]() {
lockFlags.recreateLockFile = true;
warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead.");
}}
});
addFlag({
.longName = "no-update-lock-file",
.description = "Do not allow any updates to the flake's lock file.",
@ -63,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions()
addFlag({
.longName = "no-registries",
.description =
"Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.",
.description = R"(
Don't allow lookups in the flake registries.
> **DEPRECATED**
>
> Use [`--no-use-registries`](#opt-no-use-registries) instead.
)",
.category = category,
.handler = {[&]() {
lockFlags.useRegistries = false;
@ -79,6 +100,26 @@ MixFlakeOptions::MixFlakeOptions()
.handler = {&lockFlags.commitLockFile, true}
});
addFlag({
.longName = "update-input",
.description = R"(
Update a specific flake input (ignoring its previous entry in the lock file).
> **DEPRECATED**
>
> Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
)",
.category = category,
.labels = {"input-path"},
.handler = {[&](std::string s) {
warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version.");
lockFlags.inputUpdates.insert(flake::parseInputPath(s));
}},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
}}
});
addFlag({
.longName = "override-input",
.description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
@ -219,9 +260,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
evalSettings.pureEval = false;
auto state = getEvalState();
Expr *e = state->parseExprFromFile(
resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
);
auto e =
state->parseExprFromFile(
resolveExprPath(
lookupFileArg(*state, *file)));
Value root;
state->eval(e, root);
@ -673,7 +715,7 @@ BuiltPaths Installable::toBuiltPaths(
}
}
StorePathSet Installable::toStorePaths(
StorePathSet Installable::toStorePathSet(
ref<Store> evalStore,
ref<Store> store,
Realise mode, OperateOn operateOn,
@ -687,13 +729,27 @@ StorePathSet Installable::toStorePaths(
return outPaths;
}
StorePaths Installable::toStorePaths(
ref<Store> evalStore,
ref<Store> store,
Realise mode, OperateOn operateOn,
const Installables & installables)
{
StorePaths outPaths;
for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
auto thisOutPaths = path.outPaths();
outPaths.insert(outPaths.end(), thisOutPaths.begin(), thisOutPaths.end());
}
return outPaths;
}
StorePath Installable::toStorePath(
ref<Store> evalStore,
ref<Store> store,
Realise mode, OperateOn operateOn,
ref<Installable> installable)
{
auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
auto paths = toStorePathSet(evalStore, store, mode, operateOn, {installable});
if (paths.size() != 1)
throw Error("argument '%s' should evaluate to one store path", installable->what());

9
src/libcmd/installables.hh
View file

@ -165,7 +165,14 @@ struct Installable
const Installables & installables,
BuildMode bMode = bmNormal);
static std::set<StorePath> toStorePaths(
static std::set<StorePath> toStorePathSet(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
OperateOn operateOn,
const Installables & installables);
static std::vector<StorePath> toStorePaths(
ref<Store> evalStore,
ref<Store> store,
Realise mode,

4
src/libcmd/local.mk
View file

@ -8,8 +8,8 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
libcmd_LIBS = libstore libutil libexpr libmain libfetchers
$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

6
src/libcmd/markdown.cc
View file

@ -4,12 +4,15 @@
#include "terminal.hh"
#include <sys/queue.h>
#if HAVE_LOWDOWN
#include <lowdown.h>
#endif
namespace nix {
std::string renderMarkdownToTerminal(std::string_view markdown)
{
#if HAVE_LOWDOWN
int windowWidth = getWindowSize().second;
struct lowdown_opts opts {
@ -48,6 +51,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
throw Error("allocation error while rendering Markdown");
return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
#else
return std::string(markdown);
#endif
}
}

22
src/libcmd/repl.cc
View file

@ -5,7 +5,7 @@
#include <setjmp.h>
#ifdef READLINE
#ifdef USE_READLINE
#include <readline/history.h>
#include <readline/readline.h>
#else
@ -43,7 +43,6 @@ extern "C" {
#include "finally.hh"
#include "markdown.hh"
#include "local-fs-store.hh"
#include "progress-bar.hh"
#include "print.hh"
#if HAVE_BOEHMGC
@ -113,7 +112,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalS
: AbstractNixRepl(state)
, debugTraceIndex(0)
, getValues(getValues)
, staticEnv(new StaticEnv(false, state->staticBaseEnv.get()))
, staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
, historyFile(getDataDir() + "/nix/repl-history")
{
}
@ -222,7 +221,7 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
// prefer direct pos, but if noPos then try the expr.
auto pos = dt.pos
? dt.pos
: static_cast<std::shared_ptr<AbstractPos>>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]);
: positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
if (pos) {
out << pos;
@ -250,25 +249,23 @@ void NixRepl::mainLoop()
} catch (SysError & e) {
logWarning(e.info());
}
#ifndef READLINE
#ifndef USE_READLINE
el_hist_size = 1000;
#endif
read_history(historyFile.c_str());
auto oldRepl = curRepl;
curRepl = this;
Finally restoreRepl([&] { curRepl = oldRepl; });
#ifndef READLINE
#ifndef USE_READLINE
rl_set_complete_func(completionCallback);
rl_set_list_possib_func(listPossibleCallback);
#endif
/* Stop the progress bar because it interferes with the display of
the repl. */
stopProgressBar();
std::string input;
while (true) {
// Hide the progress bar while waiting for user input, so that it won't interfere.
logger->pause();
// When continuing input from previous lines, don't print a prompt, just align to the same
// number of chars as the prompt.
if (!getLine(input, input.empty() ? "nix-repl> " : " ")) {
@ -278,6 +275,7 @@ void NixRepl::mainLoop()
logger->cout("");
break;
}
logger->resume();
try {
if (!removeWhitespace(input).empty() && !processLine(input)) return;
} catch (ParseError & e) {
@ -890,7 +888,7 @@ void NixRepl::evalString(std::string s, Value & v)
{
Expr * e = parseString(s);
e->eval(*state, *env, v);
state->forceValue(v, [&]() { return v.determinePos(noPos); });
state->forceValue(v, v.determinePos(noPos));
}
@ -909,7 +907,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
str.flush();
checkInterrupt();
state->forceValue(v, [&]() { return v.determinePos(noPos); });
state->forceValue(v, v.determinePos(noPos));
switch (v.type()) {

6
src/libexpr/eval-cache.cc
View file

@ -22,7 +22,7 @@ struct AttrDb
{
std::atomic_bool failed{false};
const Store & cfg;
const StoreDirConfig & cfg;
struct State
{
@ -39,7 +39,7 @@ struct AttrDb
SymbolTable & symbols;
AttrDb(
const Store & cfg,
const StoreDirConfig & cfg,
const Hash & fingerprint,
SymbolTable & symbols)
: cfg(cfg)
@ -323,7 +323,7 @@ struct AttrDb
};
static std::shared_ptr<AttrDb> makeAttrDb(
const Store & cfg,
const StoreDirConfig & cfg,
const Hash & fingerprint,
SymbolTable & symbols)
{

20
src/libexpr/eval-inline.hh
View file

@ -73,8 +73,6 @@ Env & EvalState::allocEnv(size_t size)
#endif
env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
env->type = Env::Plain;
/* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
return *env;
@ -83,13 +81,6 @@ Env & EvalState::allocEnv(size_t size)
[[gnu::always_inline]]
void EvalState::forceValue(Value & v, const PosIdx pos)
{
forceValue(v, [&]() { return pos; });
}
template<typename Callable>
void EvalState::forceValue(Value & v, Callable getPos)
{
if (v.isThunk()) {
Env * env = v.thunk.env;
@ -100,13 +91,12 @@ void EvalState::forceValue(Value & v, Callable getPos)
expr->eval(*this, *env, v);
} catch (...) {
v.mkThunk(env, expr);
tryFixupBlackHolePos(v, pos);
throw;
}
}
else if (v.isApp())
callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.isBlackhole())
error("infinite recursion encountered").atPos(getPos()).template debugThrow<EvalError>();
callFunction(*v.app.left, *v.app.right, v, pos);
}
@ -121,9 +111,9 @@ template <typename Callable>
[[gnu::always_inline]]
inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx)
{
forceValue(v, noPos);
PosIdx pos = getPos();
forceValue(v, pos);
if (v.type() != nAttrs) {
PosIdx pos = getPos();
error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
}
}
@ -132,7 +122,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e
[[gnu::always_inline]]
inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx)
{
forceValue(v, noPos);
forceValue(v, pos);
if (!v.isList()) {
error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
}

6
src/libexpr/eval-settings.cc
View file

@ -89,6 +89,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url)
return std::string(url);
}
const std::string & EvalSettings::getCurrentSystem()
{
const auto & evalSystem = currentSystem.get();
return evalSystem != "" ? evalSystem : settings.thisSystem.get();
}
EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings);

28
src/libexpr/eval-settings.hh
View file

@ -27,6 +27,26 @@ struct EvalSettings : Config
[`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
)"};
Setting<std::string> currentSystem{
this, "", "eval-system",
R"(
This option defines
[`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
in the Nix language if it is set as a non-empty string.
Otherwise, if it is defined as the empty string (the default), the value of the
[`system` ](#conf-system)
configuration setting is used instead.
Unlike `system`, this setting does not change what kind of derivations can be built locally.
This is useful for evaluating Nix code on one system to produce derivations to be built on another type of system.
)"};
/**
* Implements the `eval-system` vs `system` defaulting logic
* described for `eval-system`.
*/
const std::string & getCurrentSystem();
Setting<bool> restrictEval{
this, false, "restrict-eval",
R"(
@ -68,6 +88,11 @@ struct EvalSettings : Config
evaluation mode. For example, when set to
`https://github.com/NixOS`, builtin functions such as `fetchGit` are
allowed to access `https://github.com/NixOS/patchelf.git`.
Access is granted when
- the URI is equal to the prefix,
- or the URI is a subpath of the prefix,
- or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme.
)"};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
@ -99,6 +124,9 @@ struct EvalSettings : Config
Setting<bool> traceVerbose{this, false, "trace-verbose",
"Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
Setting<unsigned int> maxCallDepth{this, 10000, "max-call-depth",
"The maximum function call depth to allow before erroring."};
};
extern EvalSettings evalSettings;

333
src/libexpr/eval.cc
View file

@ -1,6 +1,7 @@
#include "eval.hh"
#include "eval-settings.hh"
#include "hash.hh"
#include "primops.hh"
#include "types.hh"
#include "util.hh"
#include "store-api.hh"
@ -13,8 +14,12 @@
#include "profiles.hh"
#include "print.hh"
#include "fs-input-accessor.hh"
#include "filtering-input-accessor.hh"
#include "memory-input-accessor.hh"
#include "signals.hh"
#include "gc-small-vector.hh"
#include "url.hh"
#include "fetch-to-store.hh"
#include <algorithm>
#include <chrono>
@ -30,6 +35,7 @@
#include <sys/resource.h>
#include <nlohmann/json.hpp>
#include <boost/container/small_vector.hpp>
#if HAVE_BOEHMGC
@ -158,7 +164,17 @@ void Value::print(const SymbolTable &symbols, std::ostream &str,
break;
case tThunk:
case tApp:
str << "<CODE>";
if (!isBlackhole()) {
str << "<CODE>";
} else {
// Although we know for sure that it's going to be an infinite recursion
// when this value is accessed _in the current context_, it's likely
// that the user will misinterpret a simpler «infinite recursion» output
// as a definitive statement about the value, while in fact it may be
// a valid value after `builtins.trace` and perhaps some other steps
// have completed.
str << "«potential infinite recursion»";
}
break;
case tLambda:
str << "<LAMBDA>";
@ -175,15 +191,6 @@ void Value::print(const SymbolTable &symbols, std::ostream &str,
case tFloat:
str << fpoint;
break;
case tBlackhole:
// Although we know for sure that it's going to be an infinite recursion
// when this value is accessed _in the current context_, it's likely
// that the user will misinterpret a simpler «infinite recursion» output
// as a definitive statement about the value, while in fact it may be
// a valid value after `builtins.trace` and perhaps some other steps
// have completed.
str << "«potential infinite recursion»";
break;
default:
printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType);
abort();
@ -251,9 +258,8 @@ std::string showType(const Value & v)
case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name));
case tExternal: return v.external->showType();
case tThunk: return "a thunk";
case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk";
case tApp: return "a function application";
case tBlackhole: return "a black hole";
default:
return std::string(showType(v.type()));
}
@ -341,7 +347,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
} else {
Value nameValue;
name.expr->eval(state, env, nameValue);
state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name");
state.forceStringNoCtx(nameValue, name.expr->getPos(), "while evaluating an attribute name");
return state.symbols.create(nameValue.string_view());
}
}
@ -506,7 +512,16 @@ EvalState::EvalState(
, sOutputSpecified(symbols.create("outputSpecified"))
, repair(NoRepair)
, emptyBindings(0)
, rootFS(makeFSInputAccessor(CanonPath::root))
, rootFS(
evalSettings.restrictEval || evalSettings.pureEval
? ref<InputAccessor>(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {},
[](const CanonPath & path) -> RestrictedPathError {
auto modeInformation = evalSettings.pureEval
? "in pure evaluation mode (use '--impure' to override)"
: "in restricted mode";
throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
}))
: makeFSInputAccessor(CanonPath::root))
, corepkgsFS(makeMemoryInputAccessor())
, internalFS(makeMemoryInputAccessor())
, derivationInternal{corepkgsFS->addFile(
@ -529,14 +544,19 @@ EvalState::EvalState(
, env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
#endif
, baseEnv(allocEnv(128))
, staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
, staticBaseEnv{std::make_shared<StaticEnv>(nullptr, nullptr)}
{
corepkgsFS->setPathDisplay("<nix", ">");
internalFS->setPathDisplay("«nix-internal»", "");
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
assert(gcInitialised);
static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
vEmptyList.mkList(0);
/* Initialise the Nix expression search path. */
if (!evalSettings.pureEval) {
for (auto & i : _searchPath.elements)
@ -545,28 +565,10 @@ EvalState::EvalState(
searchPath.elements.emplace_back(SearchPath::Elem::parse(i));
}
if (evalSettings.restrictEval || evalSettings.pureEval) {
allowedPaths = PathSet();
for (auto & i : searchPath.elements) {
auto r = resolveSearchPathPath(i.path);
if (!r) continue;
auto path = std::move(*r);
if (store->isInStore(path)) {
try {
StorePathSet closure;
store->computeFSClosure(store->toStorePath(path).first, closure);
for (auto & path : closure)
allowPath(path);
} catch (InvalidPath &) {
allowPath(path);
}
} else
allowPath(path);
}
}
/* Allow access to all paths in the search path. */
if (rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
for (auto & i : searchPath.elements)
resolveSearchPathPath(i.path, true);
corepkgsFS->addFile(
CanonPath("fetchurl.nix"),
@ -584,14 +586,14 @@ EvalState::~EvalState()
void EvalState::allowPath(const Path & path)
{
if (allowedPaths)
allowedPaths->insert(path);
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
rootFS2->allowPath(CanonPath(path));
}
void EvalState::allowPath(const StorePath & storePath)
{
if (allowedPaths)
allowedPaths->insert(store->toRealPath(storePath));
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
rootFS2->allowPath(CanonPath(store->toRealPath(storePath)));
}
void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
@ -601,79 +603,57 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
mkStorePathString(storePath, v);
}
SourcePath EvalState::checkSourcePath(const SourcePath & path_)
inline static bool isJustSchemePrefix(std::string_view prefix)
{
// Don't check non-rootFS accessors, they're in a different namespace.
if (path_.accessor != ref<InputAccessor>(rootFS)) return path_;
if (!allowedPaths) return path_;
auto i = resolvedPaths.find(path_.path.abs());
if (i != resolvedPaths.end())
return i->second;
bool found = false;
/* First canonicalize the path without symlinks, so we make sure an
* attacker can't append ../../... to a path that would be in allowedPaths
* and thus leak symlink targets.
*/
Path abspath = canonPath(path_.path.abs());
for (auto & i : *allowedPaths) {
if (isDirOrInDir(abspath, i)) {
found = true;
break;
}
}
if (!found) {
auto modeInformation = evalSettings.pureEval
? "in pure eval mode (use '--impure' to override)"
: "in restricted mode";
throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation);
}
/* Resolve symlinks. */
debug("checking access to '%s'", abspath);
SourcePath path = rootPath(CanonPath(canonPath(abspath, true)));
for (auto & i : *allowedPaths) {
if (isDirOrInDir(path.path.abs(), i)) {
resolvedPaths.insert_or_assign(path_.path.abs(), path);
return path;
}
}
throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path);
return
!prefix.empty()
&& prefix[prefix.size() - 1] == ':'
&& isValidSchemeName(prefix.substr(0, prefix.size() - 1));
}
bool isAllowedURI(std::string_view uri, const Strings & allowedUris)
{
/* 'uri' should be equal to a prefix, or in a subdirectory of a
prefix. Thus, the prefix https://github.co does not permit
access to https://github.com. */
for (auto & prefix : allowedUris) {
if (uri == prefix
// Allow access to subdirectories of the prefix.
|| (uri.size() > prefix.size()
&& prefix.size() > 0
&& hasPrefix(uri, prefix)
&& (
// Allow access to subdirectories of the prefix.
prefix[prefix.size() - 1] == '/'
|| uri[prefix.size()] == '/'
// Allow access to whole schemes
|| isJustSchemePrefix(prefix)
)
))
return true;
}
return false;
}
void EvalState::checkURI(const std::string & uri)
{
if (!evalSettings.restrictEval) return;
/* 'uri' should be equal to a prefix, or in a subdirectory of a
prefix. Thus, the prefix https://github.co does not permit
access to https://github.com. Note: this allows 'http://' and
'https://' as prefixes for any http/https URI. */
for (auto & prefix : evalSettings.allowedUris.get())
if (uri == prefix ||
(uri.size() > prefix.size()
&& prefix.size() > 0
&& hasPrefix(uri, prefix)
&& (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/')))
return;
if (isAllowedURI(uri, evalSettings.allowedUris.get())) return;
/* If the URI is a path, then check it against allowedPaths as
well. */
if (hasPrefix(uri, "/")) {
checkSourcePath(rootPath(CanonPath(uri)));
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
rootFS2->checkAccess(CanonPath(uri));
return;
}
if (hasPrefix(uri, "file://")) {
checkSourcePath(rootPath(CanonPath(std::string(uri, 7))));
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
rootFS2->checkAccess(CanonPath(uri.substr(7)));
return;
}
@ -722,6 +702,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
}
void PrimOp::check()
{
if (arity > maxPrimOpArity) {
throw Error("primop arity must not exceed %1%", maxPrimOpArity);
}
}
void Value::mkPrimOp(PrimOp * p)
{
p->check();
clearValue();
internalType = tPrimOp;
primOp = p;
}
Value * EvalState::addPrimOp(PrimOp && primOp)
{
/* Hack to make constants lazy: turn them into a application of
@ -787,7 +784,7 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se)
// just for the current level of Env, not the whole chain.
void printWithBindings(const SymbolTable & st, const Env & env)
{
if (env.type == Env::HasWithAttrs) {
if (!env.values[0]->isThunk()) {
std::cout << "with: ";
std::cout << ANSI_MAGENTA;
Bindings::iterator j = env.values[0]->attrs->begin();
@ -841,7 +838,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En
if (env.up && se.up) {
mapStaticEnvBindings(st, *se.up, *env.up, vm);
if (env.type == Env::HasWithAttrs) {
if (!env.values[0]->isThunk()) {
// add 'with' bindings.
Bindings::iterator j = env.values[0]->attrs->begin();
while (j != env.values[0]->attrs->end()) {
@ -874,7 +871,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
? std::make_unique<DebugTraceStacker>(
*this,
DebugTrace {
.pos = error->info().errPos ? error->info().errPos : static_cast<std::shared_ptr<AbstractPos>>(positions[expr.getPos()]),
.pos = error->info().errPos ? error->info().errPos : positions[expr.getPos()],
.expr = expr,
.env = env,
.hint = error->info().msg,
@ -913,7 +910,7 @@ static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
EvalState & state,
Expr & expr,
Env & env,
std::shared_ptr<AbstractPos> && pos,
std::shared_ptr<Pos> && pos,
const char * s,
const std::string & s2)
{
@ -979,22 +976,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
if (!var.fromWith) return env->values[var.displ];
// This early exit defeats the `maybeThunk` optimization for variables from `with`,
// The added complexity of handling this appears to be similarly in cost, or
// the cases where applicable were insignificant in the first place.
if (noEval) return nullptr;
auto * fromWith = var.fromWith;
while (1) {
if (env->type == Env::HasWithExpr) {
if (noEval) return 0;
Value * v = allocValue();
evalAttrs(*env->up, (Expr *) env->values[0], *v, noPos, "<borked>");
env->values[0] = v;
env->type = Env::HasWithAttrs;
}
forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression");
Bindings::iterator j = env->values[0]->attrs->find(var.name);
if (j != env->values[0]->attrs->end()) {
if (countCalls) attrSelects[j->pos]++;
return j->value;
}
if (!env->prevWith)
if (!fromWith->parentWith)
error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow<UndefinedVarError>();
for (size_t l = env->prevWith; l; --l, env = env->up) ;
for (size_t l = fromWith->prevWith; l; --l, env = env->up) ;
fromWith = fromWith->parentWith;
}
}
@ -1158,10 +1156,8 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
}
void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial)
void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
{
auto path = checkSourcePath(path_);
FileEvalCache::iterator i;
if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
v = i->second;
@ -1182,7 +1178,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
e = j->second;
if (!e)
e = parseExprFromFile(checkSourcePath(resolvedPath));
e = parseExprFromFile(resolvedPath);
fileParseCache[resolvedPath] = e;
@ -1192,7 +1188,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
*this,
*e,
this->baseEnv,
e->getPos() ? static_cast<std::shared_ptr<AbstractPos>>(positions[e->getPos()]) : nullptr,
e->getPos() ? std::make_shared<Pos>(positions[e->getPos()]) : nullptr,
"while evaluating the file '%1%':", resolvedPath.to_string())
: nullptr;
@ -1392,6 +1388,15 @@ void ExprList::eval(EvalState & state, Env & env, Value & v)
}
Value * ExprList::maybeThunk(EvalState & state, Env & env)
{
if (elems.empty()) {
return &state.vEmptyList;
}
return Expr::maybeThunk(state, env);
}
void ExprVar::eval(EvalState & state, Env & env, Value & v)
{
Value * v2 = state.lookupVar(&env, *this, false);
@ -1491,7 +1496,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
e->eval(state, env, vTmp);
for (auto & i : attrPath) {
state.forceValue(*vAttrs, noPos);
state.forceValue(*vAttrs, getPos());
Bindings::iterator j;
auto name = getName(i, state, env);
if (vAttrs->type() != nAttrs ||
@ -1513,9 +1518,27 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
v.mkLambda(&env, this);
}
namespace {
/** Increments a count on construction and decrements on destruction.
*/
class CallDepth {
size_t & count;
public:
CallDepth(size_t & count) : count(count) {
++count;
}
~CallDepth() {
--count;
}
};
};
void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
{
if (callDepth > evalSettings.maxCallDepth)
error("stack overflow; max-call-depth exceeded").atPos(pos).template debugThrow<EvalError>();
CallDepth _level(callDepth);
auto trace = evalSettings.traceFunctionCalls
? std::make_unique<FunctionCallTrace>(positions[pos])
: nullptr;
@ -1654,15 +1677,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
return;
} else {
/* We have all the arguments, so call the primop. */
auto name = vCur.primOp->name;
auto * fn = vCur.primOp;
nrPrimOpCalls++;
if (countCalls) primOpCalls[name]++;
if (countCalls) primOpCalls[fn->name]++;
try {
vCur.primOp->fun(*this, noPos, args, vCur);
fn->fun(*this, vCur.determinePos(noPos), args, vCur);
} catch (Error & e) {
addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
throw;
}
@ -1691,7 +1714,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
/* We have all the arguments, so call the primop with
the previous and new arguments. */
Value * vArgs[arity];
Value * vArgs[maxPrimOpArity];
auto n = argsDone;
for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
vArgs[--n] = arg->primOpApp.right;
@ -1699,18 +1722,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
for (size_t i = 0; i < argsLeft; ++i)
vArgs[argsDone + i] = args[i];
auto name = primOp->primOp->name;
auto fn = primOp->primOp;
nrPrimOpCalls++;
if (countCalls) primOpCalls[name]++;
if (countCalls) primOpCalls[fn->name]++;
try {
// TODO:
// 1. Unify this and above code. Heavily redundant.
// 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc)
// so the debugger allows to inspect the wrong parameters passed to the builtin.
primOp->primOp->fun(*this, noPos, vArgs, vCur);
fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
} catch (Error & e) {
addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
throw;
}
@ -1748,11 +1771,17 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
Value vFun;
fun->eval(state, env, vFun);
Value * vArgs[args.size()];
// Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5}
// 2: over 4000
// 3: about 300
// 4: about 60
// 5: under 10
// This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
SmallValueVector<4> vArgs(args.size());
for (size_t i = 0; i < args.size(); ++i)
vArgs[i] = args[i]->maybeThunk(state, env);
state.callFunction(vFun, args.size(), vArgs, v, pos);
state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
}
@ -1810,7 +1839,7 @@ https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbo
}
}
callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos);
callFunction(fun, allocValue()->mkAttrs(attrs), res, pos);
}
@ -1818,9 +1847,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
{
Env & env2(state.allocEnv(1));
env2.up = &env;
env2.prevWith = prevWith;
env2.type = Env::HasWithExpr;
env2.values[0] = (Value *) attrs;
env2.values[0] = attrs->maybeThunk(state, env);
body->eval(state, env2, v);
}
@ -1846,7 +1873,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
{
v.mkBool(!state.evalBool(env, e, noPos, "in the argument of the not operator")); // XXX: FIXME: !
v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
}
@ -1991,8 +2018,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
return result;
};
Value values[es->size()];
Value * vTmpP = values;
// List of returned strings. References to these Values must NOT be persisted.
SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
Value * vTmpP = values.data();
for (auto & [i_pos, i] : *es) {
Value & vTmp = *vTmpP++;
@ -2057,6 +2085,29 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v)
}
void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
{
state.error("infinite recursion encountered")
.debugThrow<InfiniteRecursionError>();
}
// always force this to be separate, otherwise forceValue may inline it and take
// a massive perf hit
[[gnu::noinline]]
void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
{
if (!v.isBlackhole())
return;
auto e = std::current_exception();
try {
std::rethrow_exception(e);
} catch (InfiniteRecursionError & e) {
e.err.errPos = positions[pos];
} catch (...) {
}
}
void EvalState::forceValueDeep(Value & v)
{
std::set<const Value *> seen;
@ -2066,7 +2117,7 @@ void EvalState::forceValueDeep(Value & v)
recurse = [&](Value & v) {
if (!seen.insert(&v).second) return;
forceValue(v, [&]() { return v.determinePos(noPos); });
forceValue(v, v.determinePos(noPos));
if (v.type() == nAttrs) {
for (auto & i : *v.attrs)
@ -2286,7 +2337,7 @@ BackedStringView EvalState::coerceToString(
std::string result;
for (auto [n, v2] : enumerate(v.listItems())) {
try {
result += *coerceToString(noPos, *v2, context,
result += *coerceToString(pos, *v2, context,
"while evaluating one element of the list",
coerceMore, copyToStore, canonicalizePath);
} catch (Error & e) {
@ -2318,7 +2369,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
auto dstPath = i != srcToStore.end()
? i->second
: [&]() {
auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
auto dstPath = fetchToStore(*store, path, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
allowPath(dstPath);
srcToStore.insert_or_assign(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@ -2433,8 +2484,8 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
{
forceValue(v1, noPos);
forceValue(v2, noPos);
forceValue(v1, pos);
forceValue(v2, pos);
/* !!! Hack to support some old broken code that relies on pointer
equality tests between sets. (Specifically, builderDefs calls
@ -2458,7 +2509,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
return v1.boolean == v2.boolean;
case nString:
return v1.string_view().compare(v2.string_view()) == 0;
return strcmp(v1.c_str(), v2.c_str()) == 0;
case nPath:
return

72
src/libexpr/eval.hh
View file

@ -18,13 +18,18 @@
namespace nix {
/**
* We put a limit on primop arity because it lets us use a fixed size array on
* the stack. 8 is already an impractical number of arguments. Use an attrset
* argument for such overly complicated functions.
*/
constexpr size_t maxPrimOpArity = 8;
class Store;
class EvalState;
class StorePath;
struct SingleDerivedPath;
enum RepairFlag : bool;
struct FSInputAccessor;
struct MemoryInputAccessor;
@ -71,6 +76,12 @@ struct PrimOp
* Optional experimental for this to be gated on.
*/
std::optional<ExperimentalFeature> experimentalFeature;
/**
* Validity check to be performed by functions that introduce primops,
* such as RegisterPrimOp() and Value::mkPrimOp().
*/
void check();
};
/**
@ -105,11 +116,6 @@ struct Constant
struct Env
{
Env * up;
/**
* Number of of levels up to next `with` environment
*/
unsigned short prevWith:14;
enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
Value * values[0];
};
@ -136,7 +142,7 @@ struct RegexCache;
std::shared_ptr<RegexCache> makeRegexCache();
struct DebugTrace {
std::shared_ptr<AbstractPos> pos;
std::shared_ptr<Pos> pos;
const Expr & expr;
const Env & env;
hintformat hint;
@ -205,18 +211,17 @@ public:
*/
RepairFlag repair;
/**
* The allowed filesystem paths in restricted or pure evaluation
* mode.
*/
std::optional<PathSet> allowedPaths;
Bindings emptyBindings;
/**
* Empty list constant.
*/
Value vEmptyList;
/**
* The accessor for the root filesystem.
*/
const ref<FSInputAccessor> rootFS;
const ref<InputAccessor> rootFS;
/**
* The in-memory filesystem for <nix/...> paths.
@ -330,11 +335,6 @@ private:
std::map<std::string, std::optional<std::string>> searchPathResolved;
/**
* Cache used by checkSourcePath().
*/
std::unordered_map<Path, SourcePath> resolvedPaths;
/**
* Cache used by prim_match().
*/
@ -384,12 +384,6 @@ public:
*/
void allowAndSetStorePathString(const StorePath & storePath, Value & v);
/**
* Check whether access to a path is allowed and throw an error if
* not. Otherwise return the canonicalised path.
*/
SourcePath checkSourcePath(const SourcePath & path);
void checkURI(const std::string & uri);
/**
@ -433,13 +427,15 @@ public:
SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/**
* Try to resolve a search path value (not the optional key part)
* Try to resolve a search path value (not the optional key part).
*
* If the specified search path element is a URI, download it.
*
* If it is not found, return `std::nullopt`
*/
std::optional<std::string> resolveSearchPathPath(const SearchPath::Path & path);
std::optional<std::string> resolveSearchPathPath(
const SearchPath::Path & elem,
bool initAccessControl = false);
/**
* Evaluate an expression to normal form
@ -464,8 +460,7 @@ public:
*/
inline void forceValue(Value & v, const PosIdx pos);
template <typename Callable>
inline void forceValue(Value & v, Callable getPos);
void tryFixupBlackHolePos(Value & v, PosIdx pos);
/**
* Force a value, then recursively force list elements and
@ -627,6 +622,11 @@ private:
const SourcePath & basePath,
std::shared_ptr<StaticEnv> & staticEnv);
/**
* Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack.
*/
size_t callDepth = 0;
public:
/**
@ -744,6 +744,13 @@ public:
*/
[[nodiscard]] StringMap realiseContext(const NixStringContext & context);
/* Call the binary path filter predicate used builtins.path etc. */
bool callPathFilter(
Value * filterFun,
const SourcePath & path,
std::string_view pathArg,
PosIdx pos);
private:
/**
@ -827,7 +834,12 @@ std::string showType(const Value & v);
/**
* If `path` refers to a directory, then append "/default.nix".
*/
SourcePath resolveExprPath(const SourcePath & path);
SourcePath resolveExprPath(SourcePath path);
/**
* Whether a URI is allowed, assuming restrictEval is enabled
*/
bool isAllowedURI(std::string_view uri, const Strings & allowedPaths);
struct InvalidPathError : EvalError
{

25
src/libexpr/flake/flake.cc
View file

@ -212,8 +212,16 @@ static Flake getFlake(
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, originalRef, allowLookup, flakeCache);
// We need to guard against symlink attacks, but before we start doing
// filesystem operations we should make sure there's a flake.nix in the
// first place.
auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir;
auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix";
if (!pathExists(unsafeFlakeFile))
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
// Guard against symlink attacks.
auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true);
auto flakeDir = canonPath(unsafeFlakeDir, true);
auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
if (!isInDir(flakeFile, state.store->toRealPath(storePath)))
throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
@ -226,9 +234,6 @@ static Flake getFlake(
.storePath = storePath,
};
if (!pathExists(flakeFile))
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
Value vInfo;
state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack
@ -358,10 +363,13 @@ LockedFlake lockFlake(
debug("old lock file: %s", oldLockFile);
std::map<InputPath, FlakeInput> overrides;
std::set<InputPath> explicitCliOverrides;
std::set<InputPath> overridesUsed, updatesUsed;
for (auto & i : lockFlags.inputOverrides)
for (auto & i : lockFlags.inputOverrides) {
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
explicitCliOverrides.insert(i.first);
}
LockFile newLockFile;
@ -432,6 +440,7 @@ LockedFlake lockFlake(
ancestors? */
auto i = overrides.find(inputPath);
bool hasOverride = i != overrides.end();
bool hasCliOverride = explicitCliOverrides.contains(inputPath);
if (hasOverride) {
overridesUsed.insert(inputPath);
// Respect the “flakeness” of the input even if we
@ -467,7 +476,7 @@ LockedFlake lockFlake(
if (oldLock
&& oldLock->originalRef == *input.ref
&& !hasOverride)
&& !hasCliOverride)
{
debug("keeping existing input '%s'", inputPathS);
@ -547,7 +556,7 @@ LockedFlake lockFlake(
nuked the next time we update the lock
file. That is, overrides are sticky unless you
use --no-write-lock-file. */
auto ref = input2.ref ? *input2.ref : *input.ref;
auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref;
if (input.isFlake) {
Path localPath = parentPath;
@ -895,7 +904,7 @@ Fingerprint LockedFlake::getFingerprint() const
// FIXME: as an optimization, if the flake contains a lock file
// and we haven't changed it, then it's sufficient to use
// flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256,
return hashString(HashAlgorithm::SHA256,
fmt("%s;%s;%d;%d;%s",
flake.storePath.to_string(),
flake.lockedRef.subdir,

4
src/libexpr/flake/flakeref.cc
View file

@ -90,7 +90,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
fragment = percentDecode(url.substr(fragmentStart+1));
}
if (pathEnd != std::string::npos && fragmentStart != std::string::npos) {
query = decodeQuery(url.substr(pathEnd+1, fragmentStart));
query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1));
}
if (baseDir) {
@ -190,7 +190,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
static std::regex flakeRegex(
"((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+ "(?:#(" + queryRegex + "))?",
+ "(?:#(" + fragmentRegex + "))?",
std::regex::ECMAScript);
if (std::regex_match(url, match, flakeRegex)) {

48
src/libexpr/flake/url-name.cc Normal file
View file

@ -0,0 +1,48 @@
#include "url-name.hh"
#include <regex>
#include <iostream>
namespace nix {
static const std::string attributeNamePattern("[a-zA-Z0-9_-]+");
static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
static const std::regex gitProviderRegex("github|gitlab|sourcehut");
static const std::regex gitSchemeRegex("git($|\\+.*)");
static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)");
std::optional<std::string> getNameFromURL(const ParsedURL & url)
{
std::smatch match;
/* If there is a dir= argument, use its value */
if (url.query.count("dir") > 0)
return url.query.at("dir");
/* If the fragment isn't a "default" and contains two attribute elements, use the last one */
if (std::regex_match(url.fragment, match, lastAttributeRegex))
return match.str(1);
/* If this is a github/gitlab/sourcehut flake, use the repo name */
if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
return match.str(1);
/* If it is a regular git flake, use the directory name */
if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
return match.str(1);
/* If everything failed but there is a non-default fragment, use it in full */
if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
return url.fragment;
/* If there is no fragment, take the last element of the path */
if (std::regex_match(url.path, match, lastPathSegmentRegex))
return match.str(1);
/* If even that didn't work, the URL does not contain enough info to determine a useful name */
return {};
}
}

20
src/libexpr/flake/url-name.hh Normal file
View file

@ -0,0 +1,20 @@
#include "url.hh"
#include "url-parts.hh"
#include "util.hh"
#include "split.hh"
namespace nix {
/**
* Try to extract a reasonably unique and meaningful, human-readable
* name of a flake output from a parsed URL.
* When nullopt is returned, the callsite should use information available
* to it outside of the URL to determine a useful name.
* This is a heuristic approach intended for user interfaces.
* @return nullopt if the extracted name is not useful to identify a
* flake output, for example because it is empty or "default".
* Otherwise returns the extracted name.
*/
std::optional<std::string> getNameFromURL(const ParsedURL & url);
}

42
src/libexpr/gc-small-vector.hh Normal file
View file

@ -0,0 +1,42 @@
#pragma once
#include <boost/container/small_vector.hpp>
#if HAVE_BOEHMGC
#include <gc/gc.h>
#include <gc/gc_cpp.h>
#include <gc/gc_allocator.h>
#endif
namespace nix {
struct Value;
/**
* A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap.
*/
#if HAVE_BOEHMGC
template <typename T, size_t nItems>
using SmallVector = boost::container::small_vector<T, nItems, traceable_allocator<T>>;
#else
template <typename T, size_t nItems>
using SmallVector = boost::container::small_vector<T, nItems>;
#endif
/**
* A vector of value pointers. See `SmallVector`.
*/
template <size_t nItems>
using SmallValueVector = SmallVector<Value *, nItems>;
/**
* A vector of values that must not be referenced after the vector is destroyed.
*
* See also `SmallValueVector`.
*/
template <size_t nItems>
using SmallTemporaryValueVector = SmallVector<Value, nItems>;
}

4
src/libexpr/get-drvs.cc
View file

@ -198,7 +198,7 @@ StringSet DrvInfo::queryMetaNames()
bool DrvInfo::checkMeta(Value & v)
{
state->forceValue(v, [&]() { return v.determinePos(noPos); });
state->forceValue(v, v.determinePos(noPos));
if (v.type() == nList) {
for (auto elem : v.listItems())
if (!checkMeta(*elem)) return false;
@ -304,7 +304,7 @@ static bool getDerivation(EvalState & state, Value & v,
bool ignoreAssertionFailures)
{
try {
state.forceValue(v, [&]() { return v.determinePos(noPos); });
state.forceValue(v, v.determinePos(noPos));
if (!state.isDerivation(v)) return true;
/* Remove spurious duplicates (e.g., a set like `rec { x =

10
src/libexpr/lexer.l
View file

@ -1,4 +1,5 @@
%option reentrant bison-bridge bison-locations
%option align
%option noyywrap
%option never-interactive
%option stack
@ -35,9 +36,6 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
#define CUR_POS makeCurPos(*yylloc, data)
// backup to recover from yyless(0)
thread_local YYLTYPE prev_yylloc;
static void initLoc(YYLTYPE * loc)
{
loc->first_line = loc->last_line = 1;
@ -46,7 +44,7 @@ static void initLoc(YYLTYPE * loc)
static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
{
prev_yylloc = *loc;
loc->stash();
loc->first_line = loc->last_line;
loc->first_column = loc->last_column;
@ -230,7 +228,7 @@ or { return OR_KW; }
{HPATH_START}\$\{ {
PUSH_STATE(PATH_START);
yyless(0);
*yylloc = prev_yylloc;
yylloc->unstash();
}
<PATH_START>{PATH_SEG} {
@ -286,7 +284,7 @@ or { return OR_KW; }
context (it may be ')', ';', or something of that sort) */
POP_STATE();
yyless(0);
*yylloc = prev_yylloc;
yylloc->unstash();
return PATH_END;
}

12
src/libexpr/local.mk
View file

@ -16,9 +16,9 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
libexpr_LIBS = libutil libstore libfetchers
libexpr_LDFLAGS += -lboost_context -pthread
libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
ifdef HOST_LINUX
libexpr_LDFLAGS += -ldl
libexpr_LDFLAGS += -ldl
endif
# The dependency on libgc must be propagated (i.e. meaning that
@ -36,15 +36,15 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/value/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM =
$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM =

81
src/libexpr/nixexpr.cc
View file

@ -9,57 +9,7 @@
namespace nix {
struct PosAdapter : AbstractPos
{
Pos::Origin origin;
PosAdapter(Pos::Origin origin)
: origin(std::move(origin))
{
}
std::optional<std::string> getSource() const override
{
return std::visit(overloaded {
[](const Pos::none_tag &) -> std::optional<std::string> {
return std::nullopt;
},
[](const Pos::Stdin & s) -> std::optional<std::string> {
// Get rid of the null terminators added by the parser.
return std::string(s.source->c_str());
},
[](const Pos::String & s) -> std::optional<std::string> {
// Get rid of the null terminators added by the parser.
return std::string(s.source->c_str());
},
[](const SourcePath & path) -> std::optional<std::string> {
try {
return path.readFile();
} catch (Error &) {
return std::nullopt;
}
}
}, origin);
}
void print(std::ostream & out) const override
{
std::visit(overloaded {
[&](const Pos::none_tag &) { out << "«none»"; },
[&](const Pos::Stdin &) { out << "«stdin»"; },
[&](const Pos::String & s) { out << "«string»"; },
[&](const SourcePath & path) { out << path; }
}, origin);
}
};
Pos::operator std::shared_ptr<AbstractPos>() const
{
auto pos = std::make_shared<PosAdapter>(origin);
pos->line = line;
pos->column = column;
return pos;
}
ExprBlackHole eBlackHole;
// FIXME: remove, because *symbols* are abstract and do not have a single
// textual representation; see printIdentifier()
@ -266,17 +216,6 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
}
std::ostream & operator << (std::ostream & str, const Pos & pos)
{
if (auto pos2 = (std::shared_ptr<AbstractPos>) pos) {
str << *pos2;
} else
str << "undefined position";
return str;
}
std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
{
std::ostringstream out;
@ -331,6 +270,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
fromWith = nullptr;
/* Check whether the variable appears in the environment. If so,
set its level and displacement. */
const StaticEnv * curEnv;
@ -342,7 +283,6 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
} else {
auto i = curEnv->find(name);
if (i != curEnv->vars.end()) {
fromWith = false;
this->level = level;
displ = i->second;
return;
@ -358,7 +298,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
.msg = hintfmt("undefined variable '%1%'", es.symbols[name]),
.errPos = es.positions[pos]
});
fromWith = true;
for (auto * e = env.get(); e && !fromWith; e = e->up)
fromWith = e->isWith;
this->level = withLevel;
}
@ -391,7 +332,7 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
es.exprEnvs.insert(std::make_pair(this, env));
if (recursive) {
auto newEnv = std::make_shared<StaticEnv>(false, env.get(), recursive ? attrs.size() : 0);
auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), recursive ? attrs.size() : 0);
Displacement displ = 0;
for (auto & i : attrs)
@ -433,7 +374,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
es.exprEnvs.insert(std::make_pair(this, env));
auto newEnv = std::make_shared<StaticEnv>(
false, env.get(),
nullptr, env.get(),
(hasFormals() ? formals->formals.size() : 0) +
(!arg ? 0 : 1));
@ -469,7 +410,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
auto newEnv = std::make_shared<StaticEnv>(false, env.get(), attrs->attrs.size());
auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
Displacement displ = 0;
for (auto & i : attrs->attrs)
@ -488,6 +429,10 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
parentWith = nullptr;
for (auto * e = env.get(); e && !parentWith; e = e->up)
parentWith = e->isWith;
/* Does this `with' have an enclosing `with'? If so, record its
level so that `lookupVar' can look up variables in the previous
`with' if this one doesn't contain the desired attribute. */
@ -504,7 +449,7 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
es.exprEnvs.insert(std::make_pair(this, env));
attrs->bindVars(es, env);
auto newEnv = std::make_shared<StaticEnv>(true, env.get());
auto newEnv = std::make_shared<StaticEnv>(this, env.get());
body->bindVars(es, newEnv);
}

52
src/libexpr/nixexpr.hh
View file

@ -8,6 +8,7 @@
#include "symbol-table.hh"
#include "error.hh"
#include "chunked-vector.hh"
#include "position.hh"
namespace nix {
@ -21,25 +22,11 @@ MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError);
/**
* Position objects.
*/
struct Pos
class InfiniteRecursionError : public EvalError
{
uint32_t line;
uint32_t column;
struct none_tag { };
struct Stdin { ref<std::string> source; };
struct String { ref<std::string> source; };
typedef std::variant<none_tag, Stdin, String, SourcePath> Origin;
Origin origin;
explicit operator bool() const { return line > 0; }
operator std::shared_ptr<AbstractPos>() const;
friend class EvalState;
public:
using EvalError::EvalError;
};
class PosIdx {
@ -74,7 +61,7 @@ public:
mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
// Used for searching in PosTable::[].
explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {}
explicit Origin(uint32_t idx): idx(idx), origin{std::monostate()} {}
public:
const Pos::Origin origin;
@ -125,12 +112,11 @@ public:
inline PosIdx noPos = {};
std::ostream & operator << (std::ostream & str, const Pos & pos);
struct Env;
struct Value;
class EvalState;
struct ExprWith;
struct StaticEnv;
@ -219,8 +205,11 @@ struct ExprVar : Expr
Symbol name;
/* Whether the variable comes from an environment (e.g. a rec, let
or function argument) or from a "with". */
bool fromWith;
or function argument) or from a "with".
`nullptr`: Not from a `with`.
Valid pointer: the nearest, innermost `with` expression to query first. */
ExprWith * fromWith;
/* In the former case, the value is obtained by going `level`
levels up from the current environment and getting the
@ -292,6 +281,7 @@ struct ExprList : Expr
std::vector<Expr *> elems;
ExprList() { };
COMMON_METHODS
Value * maybeThunk(EvalState & state, Env & env) override;
PosIdx getPos() const override
{
@ -378,6 +368,7 @@ struct ExprWith : Expr
PosIdx pos;
Expr * attrs, * body;
size_t prevWith;
ExprWith * parentWith;
ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
PosIdx getPos() const override { return pos; }
COMMON_METHODS
@ -405,6 +396,7 @@ struct ExprOpNot : Expr
{
Expr * e;
ExprOpNot(Expr * e) : e(e) { };
PosIdx getPos() const override { return e->getPos(); }
COMMON_METHODS
};
@ -454,20 +446,30 @@ struct ExprPos : Expr
COMMON_METHODS
};
/* only used to mark thunks as black holes. */
struct ExprBlackHole : Expr
{
void show(const SymbolTable & symbols, std::ostream & str) const override {}
void eval(EvalState & state, Env & env, Value & v) override;
void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override {}
};
extern ExprBlackHole eBlackHole;
/* Static environments are used to map variable names onto (level,
displacement) pairs used to obtain the value of the variable at
runtime. */
struct StaticEnv
{
bool isWith;
ExprWith * isWith;
const StaticEnv * up;
// Note: these must be in sorted order.
typedef std::vector<std::pair<Symbol, Displacement>> Vars;
Vars vars;
StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
vars.reserve(expectedSize);
};

65
src/libexpr/parser.y
View file

@ -28,6 +28,31 @@
namespace nix {
#define YYLTYPE ::nix::ParserLocation
struct ParserLocation
{
int first_line, first_column;
int last_line, last_column;
// backup to recover from yyless(0)
int stashed_first_line, stashed_first_column;
int stashed_last_line, stashed_last_column;
void stash() {
stashed_first_line = first_line;
stashed_first_column = first_column;
stashed_last_line = last_line;
stashed_last_column = last_column;
}
void unstash() {
first_line = stashed_first_line;
first_column = stashed_first_column;
last_line = stashed_last_line;
last_column = stashed_last_column;
}
};
struct ParseData
{
EvalState & state;
@ -686,17 +711,26 @@ Expr * EvalState::parse(
}
SourcePath resolveExprPath(const SourcePath & path)
SourcePath resolveExprPath(SourcePath path)
{
unsigned int followCount = 0, maxFollow = 1024;
/* If `path' is a symlink, follow it. This is so that relative
path references work. */
auto path2 = path.resolveSymlinks();
while (!path.path.isRoot()) {
// Basic cycle/depth limit to avoid infinite loops.
if (++followCount >= maxFollow)
throw Error("too many symbolic links encountered while traversing the path '%s'", path);
auto p = path.parent().resolveSymlinks() + path.baseName();
if (p.lstat().type != InputAccessor::tSymlink) break;
path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))};
}
/* If `path' refers to a directory, append `/default.nix'. */
if (path2.lstat().type == InputAccessor::tDirectory)
return path2 + "default.nix";
if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory)
return path + "default.nix";
return path2;
return path;
}
@ -708,7 +742,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path)
Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
{
auto buffer = path.readFile();
auto buffer = path.resolveSymlinks().readFile();
// readFile hopefully have left some extra space for terminators
buffer.append("\0\0", 2);
return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
@ -775,7 +809,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
}
std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0)
std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl)
{
auto & value = value0.s;
auto i = searchPathResolved.find(value);
@ -792,7 +826,6 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
logWarning({
.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
});
res = std::nullopt;
}
}
@ -806,6 +839,20 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
else {
auto path = absPath(value);
/* Allow access to paths in the search path. */
if (initAccessControl) {
allowPath(path);
if (store->isInStore(path)) {
try {
StorePathSet closure;
store->computeFSClosure(store->toStorePath(path).first, closure);
for (auto & p : closure)
allowPath(p);
} catch (InvalidPath &) { }
}
}
if (pathExists(path))
res = { path };
else {
@ -821,7 +868,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
else
debug("failed to resolve search path element '%s'", value);
searchPathResolved[value] = res;
searchPathResolved.emplace(value, res);
return res;
}

268
src/libexpr/primops.cc
View file

@ -4,6 +4,7 @@
#include "eval-inline.hh"
#include "eval.hh"
#include "eval-settings.hh"
#include "gc-small-vector.hh"
#include "globals.hh"
#include "json-to-value.hh"
#include "names.hh"
@ -14,6 +15,8 @@
#include "value-to-json.hh"
#include "value-to-xml.hh"
#include "primops.hh"
#include "fs-input-accessor.hh"
#include "fetch-to-store.hh"
#include <boost/container/small_vector.hpp>
#include <nlohmann/json.hpp>
@ -29,7 +32,6 @@
#include <cmath>
namespace nix {
@ -83,16 +85,15 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
/* Build/substitute the context. */
std::vector<DerivedPath> buildReqs;
for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
store->buildPaths(buildReqs);
buildStore->buildPaths(buildReqs, bmNormal, store);
StorePathSet outputsToCopyAndAllow;
for (auto & drv : drvs) {
auto outputs = resolveDerivedPath(*store, drv);
auto outputs = resolveDerivedPath(*buildStore, drv, &*store);
for (auto & [outputName, outputPath] : outputs) {
/* Add the output of this derivations to the allowed
paths. */
if (allowedPaths) {
allowPath(outputPath);
}
outputsToCopyAndAllow.insert(outputPath);
/* Get all the output paths corresponding to the placeholders we had */
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
res.insert_or_assign(
@ -101,36 +102,35 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
.drvPath = drv.drvPath,
.output = outputName,
}).render(),
store->printStorePath(outputPath)
buildStore->printStorePath(outputPath)
);
}
}
}
if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
for (auto & outputPath : outputsToCopyAndAllow) {
/* Add the output of this derivations to the allowed
paths. */
allowPath(store->toRealPath(outputPath));
}
return res;
}
struct RealisePathFlags {
// Whether to check that the path is allowed in pure eval mode
bool checkForPureEval = true;
};
static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {})
static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bool resolveSymlinks = true)
{
NixStringContext context;
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
try {
if (!context.empty()) {
if (!context.empty() && path.accessor == state.rootFS) {
auto rewrites = state.realiseContext(context);
auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
return {path.accessor, CanonPath(realPath)};
path = {path.accessor, CanonPath(realPath)};
}
return flags.checkForPureEval
? state.checkSourcePath(path)
: path;
return resolveSymlinks ? path.resolveSymlinks() : path;
} catch (Error & e) {
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
throw;
@ -170,7 +170,7 @@ static void mkOutputString(
argument. */
static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
{
auto path = realisePath(state, pos, vPath);
auto path = realisePath(state, pos, vPath, false);
auto path2 = path.path.abs();
// FIXME
@ -222,7 +222,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
Env * env = &state.allocEnv(vScope->attrs->size());
env->up = &state.baseEnv;
auto staticEnv = std::make_shared<StaticEnv>(false, state.staticBaseEnv.get(), vScope->attrs->size());
auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv.get(), vScope->attrs->size());
unsigned int displ = 0;
for (auto & attr : *vScope->attrs) {
@ -446,9 +446,7 @@ static RegisterPrimOp primop_isNull({
.doc = R"(
Return `true` if *e* evaluates to `null`, and `false` otherwise.
> **Warning**
>
> This function is *deprecated*; just write `e == null` instead.
This is equivalent to `e == null`.
)",
.fun = prim_isNull,
});
@ -594,7 +592,7 @@ struct CompareValues
case nFloat:
return v1->fpoint < v2->fpoint;
case nString:
return v1->string_view().compare(v2->string_view()) < 0;
return strcmp(v1->c_str(), v2->c_str()) < 0;
case nPath:
// Note: we don't take the accessor into account
// since it's not obvious how to compare them in a
@ -1317,7 +1315,7 @@ drvName, Bindings * attrs, Value & v)
.errPos = state.positions[noPos]
}));
auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo));
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
@ -1339,7 +1337,7 @@ drvName, Bindings * attrs, Value & v)
.errPos = state.positions[noPos]
});
auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256);
auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
for (auto & i : outputs) {
@ -1348,13 +1346,13 @@ drvName, Bindings * attrs, Value & v)
drv.outputs.insert_or_assign(i,
DerivationOutput::Impure {
.method = method,
.hashType = ht,
.hashAlgo = ha,
});
else
drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating {
.method = method,
.hashType = ht,
.hashAlgo = ha,
});
}
}
@ -1493,7 +1491,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
}));
NixStringContext context;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path;
/* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */
@ -1533,29 +1531,19 @@ static RegisterPrimOp primop_storePath({
static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
auto & arg = *args[0];
/* We don’t check the path right now, because we don’t want to
throw if the path isn’t allowed, but just return false (and we
can’t just catch the exception here because we still want to
throw if something in the evaluation of `arg` tries to
access an unauthorized path). */
auto path = realisePath(state, pos, arg, { .checkForPureEval = false });
/* SourcePath doesn't know about trailing slash. */
auto mustBeDir = arg.type() == nString
&& (arg.string_view().ends_with("/")
|| arg.string_view().ends_with("/."));
try {
auto checked = state.checkSourcePath(path);
auto st = checked.maybeLstat();
auto & arg = *args[0];
auto path = realisePath(state, pos, arg);
/* SourcePath doesn't know about trailing slash. */
auto mustBeDir = arg.type() == nString
&& (arg.string_view().ends_with("/")
|| arg.string_view().ends_with("/."));
auto st = path.maybeLstat();
auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
v.mkBool(exists);
} catch (SysError & e) {
/* Don't give away info from errors while canonicalising
‘path’ in restricted mode. */
v.mkBool(false);
} catch (RestrictedPathError & e) {
v.mkBool(false);
}
@ -1699,7 +1687,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos)));
v.mkPath(state.findFile(searchPath, path, pos));
}
static RegisterPrimOp primop_findFile(PrimOp {
@ -1754,17 +1742,17 @@ static RegisterPrimOp primop_findFile(PrimOp {
/* Return the cryptographic hash of a file in base-16. */
static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
std::optional<HashType> ht = parseHashType(type);
if (!ht)
auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
if (!ha)
state.debugThrowLastTrace(Error({
.msg = hintfmt("unknown hash type '%1%'", type),
.msg = hintfmt("unknown hash algo '%1%'", algo),
.errPos = state.positions[pos]
}));
auto path = realisePath(state, pos, *args[1]);
v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false));
v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
}
static RegisterPrimOp primop_hashFile({
@ -1789,7 +1777,7 @@ static std::string_view fileTypeToString(InputAccessor::Type type)
static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
auto path = realisePath(state, pos, *args[0]);
auto path = realisePath(state, pos, *args[0], false);
/* Retrieve the directory entry type and stringize it. */
v.mkString(fileTypeToString(path.lstat().type));
}
@ -2090,8 +2078,14 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
}
auto storePath = settings.readOnlyMode
? state.store->computeStorePathForText(name, contents, refs)
: state.store->addTextToStore(name, contents, refs, state.repair);
? state.store->makeFixedOutputPathFromCA(name, TextInfo {
.hash = hashString(HashAlgorithm::SHA256, contents),
.references = std::move(refs),
})
: ({
StringSource s { contents };
state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
});
/* Note: we don't need to add `context' to the context of the
result, since `storePath' itself has references to the paths
@ -2178,11 +2172,35 @@ static RegisterPrimOp primop_toFile({
.fun = prim_toFile,
});
bool EvalState::callPathFilter(
Value * filterFun,
const SourcePath & path,
std::string_view pathArg,
PosIdx pos)
{
auto st = path.lstat();
/* Call the filter function. The first argument is the path, the
second is a string indicating the type of the file. */
Value arg1;
arg1.mkString(pathArg);
Value arg2;
// assert that type is not "unknown"
arg2.mkString(fileTypeToString(st.type));
Value * args []{&arg1, &arg2};
Value res;
callFunction(*filterFun, 2, args, res, pos);
return forceBool(res, pos, "while evaluating the return value of the path filter function");
}
static void addPath(
EvalState & state,
const PosIdx pos,
std::string_view name,
Path path,
SourcePath path,
Value * filterFun,
FileIngestionMethod method,
const std::optional<Hash> expectedHash,
@ -2190,48 +2208,29 @@ static void addPath(
const NixStringContext & context)
{
try {
// FIXME: handle CA derivation outputs (where path needs to
// be rewritten to the actual output).
auto rewrites = state.realiseContext(context);
path = state.toRealPath(rewriteStrings(path, rewrites), context);
StorePathSet refs;
if (state.store->isInStore(path)) {
if (path.accessor == state.rootFS && state.store->isInStore(path.path.abs())) {
// FIXME: handle CA derivation outputs (where path needs to
// be rewritten to the actual output).
auto rewrites = state.realiseContext(context);
path = {state.rootFS, CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))};
try {
auto [storePath, subPath] = state.store->toStorePath(path);
auto [storePath, subPath] = state.store->toStorePath(path.path.abs());
// FIXME: we should scanForReferences on the path before adding it
refs = state.store->queryPathInfo(storePath)->references;
path = state.store->toRealPath(storePath) + subPath;
path = {state.rootFS, CanonPath(state.store->toRealPath(storePath) + subPath)};
} catch (Error &) { // FIXME: should be InvalidPathError
}
}
path = evalSettings.pureEval && expectedHash
? path
: state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs();
PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path);
/* Call the filter function. The first argument is the path,
the second is a string indicating the type of the file. */
Value arg1;
arg1.mkString(path);
Value arg2;
arg2.mkString(
S_ISREG(st.st_mode) ? "regular" :
S_ISDIR(st.st_mode) ? "directory" :
S_ISLNK(st.st_mode) ? "symlink" :
"unknown" /* not supported, will fail! */);
Value * args []{&arg1, &arg2};
Value res;
state.callFunction(*filterFun, 2, args, res, pos);
return state.forceBool(res, pos, "while evaluating the return value of the path filter function");
}) : defaultPathFilter;
std::unique_ptr<PathFilter> filter;
if (filterFun)
filter = std::make_unique<PathFilter>([&](const Path & p) {
auto p2 = CanonPath(p);
return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos);
});
std::optional<StorePath> expectedStorePath;
if (expectedHash)
@ -2242,7 +2241,7 @@ static void addPath(
});
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
auto dstPath = fetchToStore(*state.store, path, name, method, filter.get(), state.repair);
if (expectedHash && expectedStorePath != dstPath)
state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
state.allowAndSetStorePathString(dstPath, v);
@ -2261,7 +2260,8 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
auto path = state.coerceToPath(pos, *args[1], context,
"while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
}
static RegisterPrimOp primop_filterSource({
@ -2341,7 +2341,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
else if (n == "recursive")
method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256);
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
else
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]),
@ -2356,7 +2356,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
if (name.empty())
name = path->baseName();
addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context);
addPath(state, pos, name, *path, filterFun, method, expectedHash, v, context);
}
static RegisterPrimOp primop_path({
@ -2375,7 +2375,7 @@ static RegisterPrimOp primop_path({
like `@`.
- filter\
A function of the type expected by `builtins.filterSource`,
A function of the type expected by [`builtins.filterSource`](#builtins-filterSource),
with the same semantics.
- recursive\
@ -2413,7 +2413,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args,
(v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]);
std::sort(v.listElems(), v.listElems() + n,
[](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; });
[](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; });
}
static RegisterPrimOp primop_attrNames({
@ -2550,6 +2550,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args
/* Get the attribute names to be removed.
We keep them as Attrs instead of Symbols so std::set_difference
can be used to remove them from attrs[0]. */
// 64: large enough to fit the attributes of a derivation
boost::container::small_vector<Attr, 64> names;
names.reserve(args[1]->listSize());
for (auto elem : args[1]->listItems()) {
@ -2729,8 +2730,8 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
Value * res[args[1]->listSize()];
unsigned int found = 0;
SmallValueVector<nonRecursiveStackReservation> res(args[1]->listSize());
size_t found = 0;
for (auto v2 : args[1]->listItems()) {
state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs");
@ -3064,9 +3065,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
// FIXME: putting this on the stack is risky.
Value * vs[args[1]->listSize()];
unsigned int k = 0;
SmallValueVector<nonRecursiveStackReservation> vs(args[1]->listSize());
size_t k = 0;
bool same = true;
for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
@ -3179,9 +3179,16 @@ static RegisterPrimOp primop_foldlStrict({
.doc = R"(
Reduce a list by applying a binary operator, from left to right,
e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6.
The return value of each application of `op` is evaluated immediately,
even for intermediate values.
...`.
For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates
to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {}
["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`.
The first argument of `op` is the accumulator whereas the second
argument is the current element being processed. The return value
of each application of `op` is evaluated immediately, even for
intermediate values.
)",
.fun = prim_foldlStrict,
});
@ -3191,10 +3198,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar
state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all"));
state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all"));
std::string_view errorCtx = any
? "while evaluating the return value of the function passed to builtins.any"
: "while evaluating the return value of the function passed to builtins.all";
Value vTmp;
for (auto elem : args[1]->listItems()) {
state.callFunction(*args[0], *elem, vTmp, pos);
bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all"));
bool res = state.forceBool(vTmp, pos, errorCtx);
if (res == any) {
v.mkBool(any);
return;
@ -3450,13 +3461,14 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
auto nrLists = args[1]->listSize();
Value lists[nrLists];
// List of returned lists before concatenation. References to these Values must NOT be persisted.
SmallTemporaryValueVector<conservativeStackReservation> lists(nrLists);
size_t len = 0;
for (unsigned int n = 0; n < nrLists; ++n) {
Value * vElem = args[1]->listElems()[n];
state.callFunction(*args[0], *vElem, lists[n], pos);
state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap");
state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap");
len += lists[n].listSize();
}
@ -3754,18 +3766,18 @@ static RegisterPrimOp primop_stringLength({
/* Return the cryptographic hash of a string in base-16. */
static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
std::optional<HashType> ht = parseHashType(type);
if (!ht)
auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
if (!ha)
state.debugThrowLastTrace(Error({
.msg = hintfmt("unknown hash type '%1%'", type),
.msg = hintfmt("unknown hash algo '%1%'", algo),
.errPos = state.positions[pos]
}));
NixStringContext context; // discarded
auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false));
v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
}
static RegisterPrimOp primop_hashString({
@ -3788,15 +3800,15 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args
auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'");
Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo"));
std::optional<HashType> ht = std::nullopt;
std::optional<HashAlgorithm> ha = std::nullopt;
if (iteratorHashAlgo != inputAttrs->end()) {
ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
}
Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'");
HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI));
v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
}
static RegisterPrimOp primop_convertHash({
@ -3825,7 +3837,8 @@ static RegisterPrimOp primop_convertHash({
The format of the resulting hash. Must be one of
- `"base16"`
- `"base32"`
- `"nix32"`
- `"base32"` (deprecated alias for `"nix32"`)
- `"base64"`
- `"sri"`
@ -4382,13 +4395,16 @@ void EvalState::createBaseEnv()
.impureOnly = true,
});
if (!evalSettings.pureEval) {
v.mkString(settings.thisSystem.get());
}
if (!evalSettings.pureEval)
v.mkString(evalSettings.getCurrentSystem());
addConstant("__currentSystem", v, {
.type = nString,
.doc = R"(
The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval).
The value of the
[`eval-system`](@docroot@/command-ref/conf-file.md#conf-eval-system)
or else
[`system`](@docroot@/command-ref/conf-file.md#conf-system)
configuration option.
It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression:
@ -4437,7 +4453,7 @@ void EvalState::createBaseEnv()
.doc = R"(
Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use.
This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md):
This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format):
```shell-session
$ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir

16
src/libexpr/primops.hh
View file

@ -8,6 +8,22 @@
namespace nix {
/**
* For functions where we do not expect deep recursion, we can use a sizable
* part of the stack a free allocation space.
*
* Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
*/
constexpr size_t nonRecursiveStackReservation = 128;
/**
* Functions that maybe applied to self-similar inputs, such as concatMap on a
* tree, should reserve a smaller part of the stack for allocation.
*
* Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
*/
constexpr size_t conservativeStackReservation = 16;
struct RegisterPrimOp
{
typedef std::vector<PrimOp> PrimOps;

1
src/libexpr/primops/fetchClosure.cc
View file

@ -1,5 +1,6 @@
#include "primops.hh"
#include "store-api.hh"
#include "realisation.hh"
#include "make-content-addressed.hh"
#include "url.hh"

4
src/libexpr/primops/fetchMercurial.cc
View file

@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
// be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial");
if (std::regex_match(value.begin(), value.end(), revRegex))
rev = Hash::parseAny(value, htSHA1);
rev = Hash::parseAny(value, HashAlgorithm::SHA1);
else
ref = value;
}
@ -79,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
attrs2.alloc("branch").mkString(*input2.getRef());
// Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2.getRev().value_or(Hash(htSHA1));
auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
attrs2.alloc("rev").mkString(rev2.gitRev());
attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
if (auto revCount = input2.getRevCount())

255
src/libexpr/primops/fetchTree.cc
View file

@ -46,7 +46,7 @@ void emitTreeAttrs(
attrs.alloc("shortRev").mkString(rev->gitShortRev());
} else if (emptyRevFallback) {
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(htSHA1);
auto emptyHash = Hash(HashAlgorithm::SHA1);
attrs.alloc("rev").mkString(emptyHash.gitRev());
attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
}
@ -187,45 +187,215 @@ static RegisterPrimOp primop_fetchTree({
.name = "fetchTree",
.args = {"input"},
.doc = R"(
Fetch a source tree or a plain file using one of the supported backends.
*input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax.
The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled.
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
> **Note**
- the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
- the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
*input* must be an attribute set with the following attributes:
- `type` (String, required)
One of the [supported source types](#source-types).
This determines other required and allowed input attributes.
- `narHash` (String, optional)
The `narHash` parameter can be used to substitute the source of the tree.
It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
That is, `fetchTree` is idempotent.
Downloads are cached in `$XDG_CACHE_HOME/nix`.
The remote source will be fetched from the network if both are true:
- A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
> **Note**
>
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
## Source types
The following source types and associated input attributes are supported.
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first paramter for
`type` or an attribute like `builtins.fetchTree.git`! -->
- `"file"`
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
- `url` (String, required)
Supported protocols:
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
- `"tarball"`
Download a tar archive and extract it into the Nix store.
This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
- `url` (String, required)
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
- `"git"`
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
- `url` (String, required)
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
- `ref` (String, optional)
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
Default: `false`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `allRefs` (Bool, optional)
If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache.
Otherwise, only the latest commit is fetched if it is not already cached.
Default: `false`
- `lastModified` (Integer, optional)
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `revCount` (Integer, optional)
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
The following input types are still subject to change:
- `"path"`
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
> **Example**
>
> The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix
> builtins.fetchTree {
> type = "github";
> owner = "NixOS";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> }
> ```
Here are some examples of how to use `fetchTree`:
- Fetch a GitHub repository using the attribute set representation:
```nix
builtins.fetchTree {
type = "github";
owner = "NixOS";
repo = "nixpkgs";
rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
}
```
This evaluates to the following attribute set:
```
{
lastModified = 1686503798;
lastModifiedDate = "20230611171638";
narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
shortRev = "ae2e6b3";
}
```
- Fetch the same GitHub repository using the URL-like syntax:
```
builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
```
> **Example**
>
> Fetch the same GitHub repository using the URL-like syntax:
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)",
.fun = prim_fetchTree,
.experimentalFeature = Xp::FetchTree,
@ -246,7 +416,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
if (n == "url")
url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch");
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256);
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
else
@ -276,7 +446,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who));
// early exit if pinned and already in the store
if (expectedHash && expectedHash->type == htSHA256) {
if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) {
auto expectedPath = state.store->makeFixedOutputPath(
name,
FixedOutputInfo {
@ -301,10 +471,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
if (expectedHash) {
auto hash = unpack
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, state.store->toRealPath(storePath));
: hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath));
if (hash != *expectedHash)
state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)));
*url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)));
}
state.allowAndSetStorePathString(storePath, v);
@ -425,7 +595,8 @@ static RegisterPrimOp primop_fetchGit({
- `shallow` (default: `false`)
A Boolean parameter that specifies whether fetching a shallow clone is allowed.
A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
This still performs a full clone of what is available on the remote.
- `allRefs`

68
src/libexpr/tests/derived-path.cc
View file

@ -1,68 +0,0 @@
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
#include <rapidcheck/gtest.h>
#include "tests/derived-path.hh"
#include "tests/libexpr.hh"
namespace nix {
// Testing of trivial expressions
class DerivedPathExpressionTest : public LibExprTest {};
// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is
// no a real fixture.
//
// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args
TEST_F(DerivedPathExpressionTest, force_init)
{
}
#ifndef COVERAGE
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_opaque_path_round_trip,
(const SingleDerivedPath::Opaque & o))
{
auto * v = state.allocValue();
state.mkStorePathString(o.path, *v);
auto d = state.coerceToSingleDerivedPath(noPos, *v, "");
RC_ASSERT(SingleDerivedPath { o } == d);
}
// TODO use DerivedPath::Built for parameter once it supports a single output
// path only.
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_derived_path_built_placeholder_round_trip,
(const SingleDerivedPath::Built & b))
{
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "ca-derivations");
auto * v = state.allocValue();
state.mkOutputString(*v, b, std::nullopt, mockXpSettings);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
RC_ASSERT(SingleDerivedPath { b } == d);
}
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_derived_path_built_out_path_round_trip,
(const SingleDerivedPath::Built & b, const StorePath & outPath))
{
auto * v = state.allocValue();
state.mkOutputString(*v, b, outPath);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
RC_ASSERT(SingleDerivedPath { b } == d);
}
#endif
} /* namespace nix */

1298
src/libexpr/tests/error_traces.cc
View file

File diff suppressed because it is too large Load diff

22
src/libexpr/tests/flakeref.cc
View file

@ -1,22 +0,0 @@
#include <gtest/gtest.h>
#include "flake/flakeref.hh"
namespace nix {
/* ----------- tests for flake/flakeref.hh --------------------------------------------------*/
/* ----------------------------------------------------------------------------
* to_string
* --------------------------------------------------------------------------*/
TEST(to_string, doesntReencodeUrl) {
auto s = "http://localhost:8181/test/+3d.tar.gz";
auto flakeref = parseFlakeRef(s);
auto parsed = flakeref.to_string();
auto expected = "http://localhost:8181/test/%2B3d.tar.gz";
ASSERT_EQ(parsed, expected);
}
}

68
src/libexpr/tests/json.cc
View file

@ -1,68 +0,0 @@
#include "tests/libexpr.hh"
#include "value-to-json.hh"
namespace nix {
// Testing the conversion to JSON
class JSONValueTest : public LibExprTest {
protected:
std::string getJSONValue(Value& value) {
std::stringstream ss;
NixStringContext ps;
printValueAsJSON(state, true, value, noPos, ss, ps);
return ss.str();
}
};
TEST_F(JSONValueTest, null) {
Value v;
v.mkNull();
ASSERT_EQ(getJSONValue(v), "null");
}
TEST_F(JSONValueTest, BoolFalse) {
Value v;
v.mkBool(false);
ASSERT_EQ(getJSONValue(v),"false");
}
TEST_F(JSONValueTest, BoolTrue) {
Value v;
v.mkBool(true);
ASSERT_EQ(getJSONValue(v), "true");
}
TEST_F(JSONValueTest, IntPositive) {
Value v;
v.mkInt(100);
ASSERT_EQ(getJSONValue(v), "100");
}
TEST_F(JSONValueTest, IntNegative) {
Value v;
v.mkInt(-100);
ASSERT_EQ(getJSONValue(v), "-100");
}
TEST_F(JSONValueTest, String) {
Value v;
v.mkString("test");
ASSERT_EQ(getJSONValue(v), "\"test\"");
}
TEST_F(JSONValueTest, StringQuotes) {
Value v;
v.mkString("test\"");
ASSERT_EQ(getJSONValue(v), "\"test\\\"\"");
}
// The dummy store doesn't support writing files. Fails with this exception message:
// C++ exception with description "error: operation 'addToStoreFromDump' is
// not supported by store 'dummy'" thrown in the test body.
TEST_F(JSONValueTest, DISABLED_Path) {
Value v;
v.mkPath(state.rootPath(CanonPath("/test")));
ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
}
} /* namespace nix */

143
src/libexpr/tests/libexpr.hh
View file

@ -1,143 +0,0 @@
#pragma once
///@file
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "value.hh"
#include "nixexpr.hh"
#include "eval.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "tests/libstore.hh"
namespace nix {
class LibExprTest : public LibStoreTest {
public:
static void SetUpTestSuite() {
LibStoreTest::SetUpTestSuite();
initGC();
}
protected:
LibExprTest()
: LibStoreTest()
, state({}, store)
{
}
Value eval(std::string input, bool forceValue = true) {
Value v;
Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root));
assert(e);
state.eval(e, v);
if (forceValue)
state.forceValue(v, noPos);
return v;
}
Symbol createSymbol(const char * value) {
return state.symbols.create(value);
}
EvalState state;
};
MATCHER(IsListType, "") {
return arg != nList;
}
MATCHER(IsList, "") {
return arg.type() == nList;
}
MATCHER(IsString, "") {
return arg.type() == nString;
}
MATCHER(IsNull, "") {
return arg.type() == nNull;
}
MATCHER(IsThunk, "") {
return arg.type() == nThunk;
}
MATCHER(IsAttrs, "") {
return arg.type() == nAttrs;
}
MATCHER_P(IsStringEq, s, fmt("The string is equal to \"%1%\"", s)) {
if (arg.type() != nString) {
return false;
}
return std::string_view(arg.c_str()) == s;
}
MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) {
if (arg.type() != nInt) {
return false;
}
return arg.integer == v;
}
MATCHER_P(IsFloatEq, v, fmt("The float is equal to \"%1%\"", v)) {
if (arg.type() != nFloat) {
return false;
}
return arg.fpoint == v;
}
MATCHER(IsTrue, "") {
if (arg.type() != nBool) {
return false;
}
return arg.boolean == true;
}
MATCHER(IsFalse, "") {
if (arg.type() != nBool) {
return false;
}
return arg.boolean == false;
}
MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) {
if (arg.type() != nPath) {
*result_listener << "Expected a path got " << arg.type();
return false;
} else {
auto path = arg.path();
if (path.path != CanonPath(p)) {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path;
return false;
}
}
return true;
}
MATCHER_P(IsListOfSize, n, fmt("Is a list of size [%1%]", n)) {
if (arg.type() != nList) {
*result_listener << "Expected list got " << arg.type();
return false;
} else if (arg.listSize() != (size_t)n) {
*result_listener << "Expected as list of size " << n << " got " << arg.listSize();
return false;
}
return true;
}
MATCHER_P(IsAttrsOfSize, n, fmt("Is a set of size [%1%]", n)) {
if (arg.type() != nAttrs) {
*result_listener << "Expected set got " << arg.type();
return false;
} else if (arg.attrs->size() != (size_t)n) {
*result_listener << "Expected a set with " << n << " attributes but got " << arg.attrs->size();
return false;
}
return true;
}
} /* namespace nix */

19
src/libexpr/tests/local.mk
View file

@ -1,19 +0,0 @@
check: libexpr-tests_RUN
programs += libexpr-tests
libexpr-tests_NAME := libnixexpr-tests
libexpr-tests_DIR := $(d)
libexpr-tests_INSTALL_DIR :=
libexpr-tests_SOURCES := \
$(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc)
libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock

832
src/libexpr/tests/primops.cc
View file

@ -1,832 +0,0 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "tests/libexpr.hh"
namespace nix {
class CaptureLogger : public Logger
{
std::ostringstream oss;
public:
CaptureLogger() {}
std::string get() const {
return oss.str();
}
void log(Verbosity lvl, std::string_view s) override {
oss << s << std::endl;
}
void logEI(const ErrorInfo & ei) override {
showErrorInfo(oss, ei, loggerSettings.showTrace.get());
}
};
class CaptureLogging {
Logger * oldLogger;
std::unique_ptr<CaptureLogger> tempLogger;
public:
CaptureLogging() : tempLogger(std::make_unique<CaptureLogger>()) {
oldLogger = logger;
logger = tempLogger.get();
}
~CaptureLogging() {
logger = oldLogger;
}
std::string get() const {
return tempLogger->get();
}
};
// Testing eval of PrimOp's
class PrimOpTest : public LibExprTest {};
TEST_F(PrimOpTest, throw) {
ASSERT_THROW(eval("throw \"foo\""), ThrownError);
}
TEST_F(PrimOpTest, abort) {
ASSERT_THROW(eval("abort \"abort\""), Abort);
}
TEST_F(PrimOpTest, ceil) {
auto v = eval("builtins.ceil 1.9");
ASSERT_THAT(v, IsIntEq(2));
}
TEST_F(PrimOpTest, floor) {
auto v = eval("builtins.floor 1.9");
ASSERT_THAT(v, IsIntEq(1));
}
TEST_F(PrimOpTest, tryEvalFailure) {
auto v = eval("builtins.tryEval (throw \"\")");
ASSERT_THAT(v, IsAttrsOfSize(2));
auto s = createSymbol("success");
auto p = v.attrs->get(s);
ASSERT_NE(p, nullptr);
ASSERT_THAT(*p->value, IsFalse());
}
TEST_F(PrimOpTest, tryEvalSuccess) {
auto v = eval("builtins.tryEval 123");
ASSERT_THAT(v, IsAttrs());
auto s = createSymbol("success");
auto p = v.attrs->get(s);
ASSERT_NE(p, nullptr);
ASSERT_THAT(*p->value, IsTrue());
s = createSymbol("value");
p = v.attrs->get(s);
ASSERT_NE(p, nullptr);
ASSERT_THAT(*p->value, IsIntEq(123));
}
TEST_F(PrimOpTest, getEnv) {
setenv("_NIX_UNIT_TEST_ENV_VALUE", "test value", 1);
auto v = eval("builtins.getEnv \"_NIX_UNIT_TEST_ENV_VALUE\"");
ASSERT_THAT(v, IsStringEq("test value"));
}
TEST_F(PrimOpTest, seq) {
ASSERT_THROW(eval("let x = throw \"test\"; in builtins.seq x { }"), ThrownError);
}
TEST_F(PrimOpTest, seqNotDeep) {
auto v = eval("let x = { z = throw \"test\"; }; in builtins.seq x { }");
ASSERT_THAT(v, IsAttrs());
}
TEST_F(PrimOpTest, deepSeq) {
ASSERT_THROW(eval("let x = { z = throw \"test\"; }; in builtins.deepSeq x { }"), ThrownError);
}
TEST_F(PrimOpTest, trace) {
CaptureLogging l;
auto v = eval("builtins.trace \"test string 123\" 123");
ASSERT_THAT(v, IsIntEq(123));
auto text = l.get();
ASSERT_NE(text.find("test string 123"), std::string::npos);
}
TEST_F(PrimOpTest, placeholder) {
auto v = eval("builtins.placeholder \"out\"");
ASSERT_THAT(v, IsStringEq("/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"));
}
TEST_F(PrimOpTest, baseNameOf) {
auto v = eval("builtins.baseNameOf /some/path");
ASSERT_THAT(v, IsStringEq("path"));
}
TEST_F(PrimOpTest, dirOf) {
auto v = eval("builtins.dirOf /some/path");
ASSERT_THAT(v, IsPathEq("/some"));
}
TEST_F(PrimOpTest, attrValues) {
auto v = eval("builtins.attrValues { x = \"foo\"; a = 1; }");
ASSERT_THAT(v, IsListOfSize(2));
ASSERT_THAT(*v.listElems()[0], IsIntEq(1));
ASSERT_THAT(*v.listElems()[1], IsStringEq("foo"));
}
TEST_F(PrimOpTest, getAttr) {
auto v = eval("builtins.getAttr \"x\" { x = \"foo\"; }");
ASSERT_THAT(v, IsStringEq("foo"));
}
TEST_F(PrimOpTest, getAttrNotFound) {
// FIXME: TypeError is really bad here, also the error wording is worse
// than on Nix <=2.3
ASSERT_THROW(eval("builtins.getAttr \"y\" { }"), TypeError);
}
TEST_F(PrimOpTest, unsafeGetAttrPos) {
// The `y` attribute is at position
const char* expr = "builtins.unsafeGetAttrPos \"y\" { y = \"x\"; }";
auto v = eval(expr);
ASSERT_THAT(v, IsNull());
}
TEST_F(PrimOpTest, hasAttr) {
auto v = eval("builtins.hasAttr \"x\" { x = 1; }");
ASSERT_THAT(v, IsTrue());
}
TEST_F(PrimOpTest, hasAttrNotFound) {
auto v = eval("builtins.hasAttr \"x\" { }");
ASSERT_THAT(v, IsFalse());
}
TEST_F(PrimOpTest, isAttrs) {
auto v = eval("builtins.isAttrs {}");
ASSERT_THAT(v, IsTrue());
}
TEST_F(PrimOpTest, isAttrsFalse) {
auto v = eval("builtins.isAttrs null");
ASSERT_THAT(v, IsFalse());
}
TEST_F(PrimOpTest, removeAttrs) {
auto v = eval("builtins.removeAttrs { x = 1; } [\"x\"]");
ASSERT_THAT(v, IsAttrsOfSize(0));
}
TEST_F(PrimOpTest, removeAttrsRetains) {
auto v = eval("builtins.removeAttrs { x = 1; y = 2; } [\"x\"]");
ASSERT_THAT(v, IsAttrsOfSize(1));
ASSERT_NE(v.attrs->find(createSymbol("y")), nullptr);
}
TEST_F(PrimOpTest, listToAttrsEmptyList) {
auto v = eval("builtins.listToAttrs []");
ASSERT_THAT(v, IsAttrsOfSize(0));
ASSERT_EQ(v.type(), nAttrs);
ASSERT_EQ(v.attrs->size(), 0);
}
TEST_F(PrimOpTest, listToAttrsNotFieldName) {
ASSERT_THROW(eval("builtins.listToAttrs [{}]"), Error);
}
TEST_F(PrimOpTest, listToAttrs) {
auto v = eval("builtins.listToAttrs [ { name = \"key\"; value = 123; } ]");
ASSERT_THAT(v, IsAttrsOfSize(1));
auto key = v.attrs->find(createSymbol("key"));
ASSERT_NE(key, nullptr);
ASSERT_THAT(*key->value, IsIntEq(123));
}
TEST_F(PrimOpTest, intersectAttrs) {
auto v = eval("builtins.intersectAttrs { a = 1; b = 2; } { b = 3; c = 4; }");
ASSERT_THAT(v, IsAttrsOfSize(1));
auto b = v.attrs->find(createSymbol("b"));
ASSERT_NE(b, nullptr);
ASSERT_THAT(*b->value, IsIntEq(3));
}
TEST_F(PrimOpTest, catAttrs) {
auto v = eval("builtins.catAttrs \"a\" [{a = 1;} {b = 0;} {a = 2;}]");
ASSERT_THAT(v, IsListOfSize(2));
ASSERT_THAT(*v.listElems()[0], IsIntEq(1));
ASSERT_THAT(*v.listElems()[1], IsIntEq(2));
}
TEST_F(PrimOpTest, functionArgs) {
auto v = eval("builtins.functionArgs ({ x, y ? 123}: 1)");
ASSERT_THAT(v, IsAttrsOfSize(2));
auto x = v.attrs->find(createSymbol("x"));
ASSERT_NE(x, nullptr);
ASSERT_THAT(*x->value, IsFalse());
auto y = v.attrs->find(createSymbol("y"));
ASSERT_NE(y, nullptr);
ASSERT_THAT(*y->value, IsTrue());
}
TEST_F(PrimOpTest, mapAttrs) {
auto v = eval("builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; }");
ASSERT_THAT(v, IsAttrsOfSize(2));
auto a = v.attrs->find(createSymbol("a"));
ASSERT_NE(a, nullptr);
ASSERT_THAT(*a->value, IsThunk());
state.forceValue(*a->value, noPos);
ASSERT_THAT(*a->value, IsIntEq(10));
auto b = v.attrs->find(createSymbol("b"));
ASSERT_NE(b, nullptr);
ASSERT_THAT(*b->value, IsThunk());
state.forceValue(*b->value, noPos);
ASSERT_THAT(*b->value, IsIntEq(20));
}
TEST_F(PrimOpTest, isList) {
auto v = eval("builtins.isList []");
ASSERT_THAT(v, IsTrue());
}
TEST_F(PrimOpTest, isListFalse) {
auto v = eval("builtins.isList null");
ASSERT_THAT(v, IsFalse());
}
TEST_F(PrimOpTest, elemtAt) {
auto v = eval("builtins.elemAt [0 1 2 3] 3");
ASSERT_THAT(v, IsIntEq(3));
}
TEST_F(PrimOpTest, elemtAtOutOfBounds) {
ASSERT_THROW(eval("builtins.elemAt [0 1 2 3] 5"), Error);
}
TEST_F(PrimOpTest, head) {
auto v = eval("builtins.head [ 3 2 1 0 ]");
ASSERT_THAT(v, IsIntEq(3));
}
TEST_F(PrimOpTest, headEmpty) {
ASSERT_THROW(eval("builtins.head [ ]"), Error);
}
TEST_F(PrimOpTest, headWrongType) {
ASSERT_THROW(eval("builtins.head { }"), Error);
}
TEST_F(PrimOpTest, tail) {
auto v = eval("builtins.tail [ 3 2 1 0 ]");
ASSERT_THAT(v, IsListOfSize(3));
for (const auto [n, elem] : enumerate(v.listItems()))
ASSERT_THAT(*elem, IsIntEq(2 - static_cast<int>(n)));
}
TEST_F(PrimOpTest, tailEmpty) {
ASSERT_THROW(eval("builtins.tail []"), Error);
}
TEST_F(PrimOpTest, map) {
auto v = eval("map (x: \"foo\" + x) [ \"bar\" \"bla\" \"abc\" ]");
ASSERT_THAT(v, IsListOfSize(3));
auto elem = v.listElems()[0];
ASSERT_THAT(*elem, IsThunk());
state.forceValue(*elem, noPos);
ASSERT_THAT(*elem, IsStringEq("foobar"));
elem = v.listElems()[1];
ASSERT_THAT(*elem, IsThunk());
state.forceValue(*elem, noPos);
ASSERT_THAT(*elem, IsStringEq("foobla"));
elem = v.listElems()[2];
ASSERT_THAT(*elem, IsThunk());
state.forceValue(*elem, noPos);
ASSERT_THAT(*elem, IsStringEq("fooabc"));
}
TEST_F(PrimOpTest, filter) {
auto v = eval("builtins.filter (x: x == 2) [ 3 2 3 2 3 2 ]");
ASSERT_THAT(v, IsListOfSize(3));
for (const auto elem : v.listItems())
ASSERT_THAT(*elem, IsIntEq(2));
}
TEST_F(PrimOpTest, elemTrue) {
auto v = eval("builtins.elem 3 [ 1 2 3 4 5 ]");
ASSERT_THAT(v, IsTrue());
}
TEST_F(PrimOpTest, elemFalse) {
auto v = eval("builtins.elem 6 [ 1 2 3 4 5 ]");
ASSERT_THAT(v, IsFalse());
}
TEST_F(PrimOpTest, concatLists) {
auto v = eval("builtins.concatLists [[1 2] [3 4]]");
ASSERT_THAT(v, IsListOfSize(4));
for (const auto [i, elem] : enumerate(v.listItems()))
ASSERT_THAT(*elem, IsIntEq(static_cast<int>(i)+1));
}
TEST_F(PrimOpTest, length) {
auto v = eval("builtins.length [ 1 2 3 ]");
ASSERT_THAT(v, IsIntEq(3));
}
TEST_F(PrimOpTest, foldStrict) {
auto v = eval("builtins.foldl' (a: b: a + b) 0 [1 2 3]");
ASSERT_THAT(v, IsIntEq(6));
}
TEST_F(PrimOpTest, anyTrue) {
auto v = eval("builtins.any (x: x == 2) [ 1 2 3 ]");
ASSERT_THAT(v, IsTrue());
}
TEST_F(PrimOpTest, anyFalse) {
auto v = eval("builtins.any (x: x == 5) [ 1 2 3 ]");
ASSERT_THAT(v, IsFalse());
}
TEST_F(PrimOpTest, allTrue) {
auto v = eval("builtins.all (x: x > 0) [ 1 2 3 ]");
ASSERT_THAT(v, IsTrue());
}
TEST_F(PrimOpTest, allFalse) {
auto v = eval("builtins.all (x: x <= 0) [ 1 2 3 ]");
ASSERT_THAT(v, IsFalse());
}
TEST_F(PrimOpTest, genList) {
auto v = eval("builtins.genList (x: x + 1) 3");
ASSERT_EQ(v.type(), nList);
ASSERT_EQ(v.listSize(), 3);
for (const auto [i, elem] : enumerate(v.listItems())) {
ASSERT_THAT(*elem, IsThunk());
state.forceValue(*elem, noPos);
ASSERT_THAT(*elem, IsIntEq(static_cast<int>(i)+1));
}
}
TEST_F(PrimOpTest, sortLessThan) {
auto v = eval("builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]");
ASSERT_EQ(v.type(), nList);
ASSERT_EQ(v.listSize(), 6);
const std::vector<int> numbers = { 42, 77, 147, 249, 483, 526 };
for (const auto [n, elem] : enumerate(v.listItems()))
ASSERT_THAT(*elem, IsIntEq(numbers[n]));
}
TEST_F(PrimOpTest, partition) {
auto v = eval("builtins.partition (x: x > 10) [1 23 9 3 42]");
ASSERT_THAT(v, IsAttrsOfSize(2));
auto right = v.attrs->get(createSymbol("right"));
ASSERT_NE(right, nullptr);
ASSERT_THAT(*right->value, IsListOfSize(2));
ASSERT_THAT(*right->value->listElems()[0], IsIntEq(23));
ASSERT_THAT(*right->value->listElems()[1], IsIntEq(42));
auto wrong = v.attrs->get(createSymbol("wrong"));
ASSERT_NE(wrong, nullptr);
ASSERT_EQ(wrong->value->type(), nList);
ASSERT_EQ(wrong->value->listSize(), 3);
ASSERT_THAT(*wrong->value, IsListOfSize(3));
ASSERT_THAT(*wrong->value->listElems()[0], IsIntEq(1));
ASSERT_THAT(*wrong->value->listElems()[1], IsIntEq(9));
ASSERT_THAT(*wrong->value->listElems()[2], IsIntEq(3));
}
TEST_F(PrimOpTest, concatMap) {
auto v = eval("builtins.concatMap (x: x ++ [0]) [ [1 2] [3 4] ]");
ASSERT_EQ(v.type(), nList);
ASSERT_EQ(v.listSize(), 6);
const std::vector<int> numbers = { 1, 2, 0, 3, 4, 0 };
for (const auto [n, elem] : enumerate(v.listItems()))
ASSERT_THAT(*elem, IsIntEq(numbers[n]));
}
TEST_F(PrimOpTest, addInt) {
auto v = eval("builtins.add 3 5");
ASSERT_THAT(v, IsIntEq(8));
}
TEST_F(PrimOpTest, addFloat) {
auto v = eval("builtins.add 3.0 5.0");
ASSERT_THAT(v, IsFloatEq(8.0));
}
TEST_F(PrimOpTest, addFloatToInt) {
auto v = eval("builtins.add 3.0 5");
ASSERT_THAT(v, IsFloatEq(8.0));
v = eval("builtins.add 3 5.0");
ASSERT_THAT(v, IsFloatEq(8.0));
}
TEST_F(PrimOpTest, subInt) {
auto v = eval("builtins.sub 5 2");
ASSERT_THAT(v, IsIntEq(3));
}
TEST_F(PrimOpTest, subFloat) {
auto v = eval("builtins.sub 5.0 2.0");
ASSERT_THAT(v, IsFloatEq(3.0));
}
TEST_F(PrimOpTest, subFloatFromInt) {
auto v = eval("builtins.sub 5.0 2");
ASSERT_THAT(v, IsFloatEq(3.0));
v = eval("builtins.sub 4 2.0");
ASSERT_THAT(v, IsFloatEq(2.0));
}
TEST_F(PrimOpTest, mulInt) {
auto v = eval("builtins.mul 3 5");
ASSERT_THAT(v, IsIntEq(15));
}
TEST_F(PrimOpTest, mulFloat) {
auto v = eval("builtins.mul 3.0 5.0");
ASSERT_THAT(v, IsFloatEq(15.0));
}
TEST_F(PrimOpTest, mulFloatMixed) {
auto v = eval("builtins.mul 3 5.0");
ASSERT_THAT(v, IsFloatEq(15.0));
v = eval("builtins.mul 2.0 5");
ASSERT_THAT(v, IsFloatEq(10.0));
}
TEST_F(PrimOpTest, divInt) {
auto v = eval("builtins.div 5 (-1)");
ASSERT_THAT(v, IsIntEq(-5));
}
TEST_F(PrimOpTest, divIntZero) {
ASSERT_THROW(eval("builtins.div 5 0"), EvalError);
}
TEST_F(PrimOpTest, divFloat) {
auto v = eval("builtins.div 5.0 (-1)");
ASSERT_THAT(v, IsFloatEq(-5.0));
}
TEST_F(PrimOpTest, divFloatZero) {
ASSERT_THROW(eval("builtins.div 5.0 0.0"), EvalError);
}
TEST_F(PrimOpTest, bitOr) {
auto v = eval("builtins.bitOr 1 2");
ASSERT_THAT(v, IsIntEq(3));
}
TEST_F(PrimOpTest, bitXor) {
auto v = eval("builtins.bitXor 3 2");
ASSERT_THAT(v, IsIntEq(1));
}
TEST_F(PrimOpTest, lessThanFalse) {
auto v = eval("builtins.lessThan 3 1");
ASSERT_THAT(v, IsFalse());
}
TEST_F(PrimOpTest, lessThanTrue) {
auto v = eval("builtins.lessThan 1 3");
ASSERT_THAT(v, IsTrue());
}
TEST_F(PrimOpTest, toStringAttrsThrows) {
ASSERT_THROW(eval("builtins.toString {}"), EvalError);
}
TEST_F(PrimOpTest, toStringLambdaThrows) {
ASSERT_THROW(eval("builtins.toString (x: x)"), EvalError);
}
class ToStringPrimOpTest :
public PrimOpTest,
public testing::WithParamInterface<std::tuple<std::string, std::string_view>>
{};
TEST_P(ToStringPrimOpTest, toString) {
const auto [input, output] = GetParam();
auto v = eval(input);
ASSERT_THAT(v, IsStringEq(output));
}
#define CASE(input, output) (std::make_tuple(std::string_view("builtins.toString " input), std::string_view(output)))
INSTANTIATE_TEST_SUITE_P(
toString,
ToStringPrimOpTest,
testing::Values(
CASE(R"("foo")", "foo"),
CASE(R"(1)", "1"),
CASE(R"([1 2 3])", "1 2 3"),
CASE(R"(.123)", "0.123000"),
CASE(R"(true)", "1"),
CASE(R"(false)", ""),
CASE(R"(null)", ""),
CASE(R"({ v = "bar"; __toString = self: self.v; })", "bar"),
CASE(R"({ v = "bar"; __toString = self: self.v; outPath = "foo"; })", "bar"),
CASE(R"({ outPath = "foo"; })", "foo"),
CASE(R"(./test)", "/test")
)
);
#undef CASE
TEST_F(PrimOpTest, substring){
auto v = eval("builtins.substring 0 3 \"nixos\"");
ASSERT_THAT(v, IsStringEq("nix"));
}
TEST_F(PrimOpTest, substringSmallerString){
auto v = eval("builtins.substring 0 3 \"n\"");
ASSERT_THAT(v, IsStringEq("n"));
}
TEST_F(PrimOpTest, substringEmptyString){
auto v = eval("builtins.substring 1 3 \"\"");
ASSERT_THAT(v, IsStringEq(""));
}
TEST_F(PrimOpTest, stringLength) {
auto v = eval("builtins.stringLength \"123\"");
ASSERT_THAT(v, IsIntEq(3));
}
TEST_F(PrimOpTest, hashStringMd5) {
auto v = eval("builtins.hashString \"md5\" \"asdf\"");
ASSERT_THAT(v, IsStringEq("912ec803b2ce49e4a541068d495ab570"));
}
TEST_F(PrimOpTest, hashStringSha1) {
auto v = eval("builtins.hashString \"sha1\" \"asdf\"");
ASSERT_THAT(v, IsStringEq("3da541559918a808c2402bba5012f6c60b27661c"));
}
TEST_F(PrimOpTest, hashStringSha256) {
auto v = eval("builtins.hashString \"sha256\" \"asdf\"");
ASSERT_THAT(v, IsStringEq("f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"));
}
TEST_F(PrimOpTest, hashStringSha512) {
auto v = eval("builtins.hashString \"sha512\" \"asdf\"");
ASSERT_THAT(v, IsStringEq("401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1"));
}
TEST_F(PrimOpTest, hashStringInvalidHashType) {
ASSERT_THROW(eval("builtins.hashString \"foobar\" \"asdf\""), Error);
}
TEST_F(PrimOpTest, nixPath) {
auto v = eval("builtins.nixPath");
ASSERT_EQ(v.type(), nList);
// We can't test much more as currently the EvalSettings are a global
// that we can't easily swap / replace
}
TEST_F(PrimOpTest, langVersion) {
auto v = eval("builtins.langVersion");
ASSERT_EQ(v.type(), nInt);
}
TEST_F(PrimOpTest, storeDir) {
auto v = eval("builtins.storeDir");
ASSERT_THAT(v, IsStringEq(settings.nixStore));
}
TEST_F(PrimOpTest, nixVersion) {
auto v = eval("builtins.nixVersion");
ASSERT_THAT(v, IsStringEq(nixVersion));
}
TEST_F(PrimOpTest, currentSystem) {
auto v = eval("builtins.currentSystem");
ASSERT_THAT(v, IsStringEq(settings.thisSystem.get()));
}
TEST_F(PrimOpTest, derivation) {
auto v = eval("derivation");
ASSERT_EQ(v.type(), nFunction);
ASSERT_TRUE(v.isLambda());
ASSERT_NE(v.lambda.fun, nullptr);
ASSERT_TRUE(v.lambda.fun->hasFormals());
}
TEST_F(PrimOpTest, currentTime) {
auto v = eval("builtins.currentTime");
ASSERT_EQ(v.type(), nInt);
ASSERT_TRUE(v.integer > 0);
}
TEST_F(PrimOpTest, splitVersion) {
auto v = eval("builtins.splitVersion \"1.2.3git\"");
ASSERT_THAT(v, IsListOfSize(4));
const std::vector<std::string_view> strings = { "1", "2", "3", "git" };
for (const auto [n, p] : enumerate(v.listItems()))
ASSERT_THAT(*p, IsStringEq(strings[n]));
}
class CompareVersionsPrimOpTest :
public PrimOpTest,
public testing::WithParamInterface<std::tuple<std::string, const int>>
{};
TEST_P(CompareVersionsPrimOpTest, compareVersions) {
auto [expression, expectation] = GetParam();
auto v = eval(expression);
ASSERT_THAT(v, IsIntEq(expectation));
}
#define CASE(a, b, expected) (std::make_tuple("builtins.compareVersions \"" #a "\" \"" #b "\"", expected))
INSTANTIATE_TEST_SUITE_P(
compareVersions,
CompareVersionsPrimOpTest,
testing::Values(
// The first two are weird cases. Intuition tells they should
// be the same but they aren't.
CASE(1.0, 1.0.0, -1),
CASE(1.0.0, 1.0, 1),
// the following are from the nix-env manual:
CASE(1.0, 2.3, -1),
CASE(2.1, 2.3, -1),
CASE(2.3, 2.3, 0),
CASE(2.5, 2.3, 1),
CASE(3.1, 2.3, 1),
CASE(2.3.1, 2.3, 1),
CASE(2.3.1, 2.3a, 1),
CASE(2.3pre1, 2.3, -1),
CASE(2.3pre3, 2.3pre12, -1),
CASE(2.3a, 2.3c, -1),
CASE(2.3pre1, 2.3c, -1),
CASE(2.3pre1, 2.3q, -1)
)
);
#undef CASE
class ParseDrvNamePrimOpTest :
public PrimOpTest,
public testing::WithParamInterface<std::tuple<std::string, std::string_view, std::string_view>>
{};
TEST_P(ParseDrvNamePrimOpTest, parseDrvName) {
auto [input, expectedName, expectedVersion] = GetParam();
const auto expr = fmt("builtins.parseDrvName \"%1%\"", input);
auto v = eval(expr);
ASSERT_THAT(v, IsAttrsOfSize(2));
auto name = v.attrs->find(createSymbol("name"));
ASSERT_TRUE(name);
ASSERT_THAT(*name->value, IsStringEq(expectedName));
auto version = v.attrs->find(createSymbol("version"));
ASSERT_TRUE(version);
ASSERT_THAT(*version->value, IsStringEq(expectedVersion));
}
INSTANTIATE_TEST_SUITE_P(
parseDrvName,
ParseDrvNamePrimOpTest,
testing::Values(
std::make_tuple("nix-0.12pre12876", "nix", "0.12pre12876"),
std::make_tuple("a-b-c-1234pre5+git", "a-b-c", "1234pre5+git")
)
);
TEST_F(PrimOpTest, replaceStrings) {
// FIXME: add a test that verifies the string context is as expected
auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\"");
ASSERT_EQ(v.type(), nString);
ASSERT_EQ(v.string_view(), "fabir");
}
TEST_F(PrimOpTest, concatStringsSep) {
// FIXME: add a test that verifies the string context is as expected
auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]");
ASSERT_EQ(v.type(), nString);
ASSERT_EQ(v.string_view(), "foo%bar%baz");
}
TEST_F(PrimOpTest, split1) {
// v = [ "" [ "a" ] "c" ]
auto v = eval("builtins.split \"(a)b\" \"abc\"");
ASSERT_THAT(v, IsListOfSize(3));
ASSERT_THAT(*v.listElems()[0], IsStringEq(""));
ASSERT_THAT(*v.listElems()[1], IsListOfSize(1));
ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a"));
ASSERT_THAT(*v.listElems()[2], IsStringEq("c"));
}
TEST_F(PrimOpTest, split2) {
// v is expected to be a list [ "" [ "a" ] "b" [ "c"] "" ]
auto v = eval("builtins.split \"([ac])\" \"abc\"");
ASSERT_THAT(v, IsListOfSize(5));
ASSERT_THAT(*v.listElems()[0], IsStringEq(""));
ASSERT_THAT(*v.listElems()[1], IsListOfSize(1));
ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a"));
ASSERT_THAT(*v.listElems()[2], IsStringEq("b"));
ASSERT_THAT(*v.listElems()[3], IsListOfSize(1));
ASSERT_THAT(*v.listElems()[3]->listElems()[0], IsStringEq("c"));
ASSERT_THAT(*v.listElems()[4], IsStringEq(""));
}
TEST_F(PrimOpTest, split3) {
auto v = eval("builtins.split \"(a)|(c)\" \"abc\"");
ASSERT_THAT(v, IsListOfSize(5));
// First list element
ASSERT_THAT(*v.listElems()[0], IsStringEq(""));
// 2nd list element is a list [ "" null ]
ASSERT_THAT(*v.listElems()[1], IsListOfSize(2));
ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a"));
ASSERT_THAT(*v.listElems()[1]->listElems()[1], IsNull());
// 3rd element
ASSERT_THAT(*v.listElems()[2], IsStringEq("b"));
// 4th element is a list: [ null "c" ]
ASSERT_THAT(*v.listElems()[3], IsListOfSize(2));
ASSERT_THAT(*v.listElems()[3]->listElems()[0], IsNull());
ASSERT_THAT(*v.listElems()[3]->listElems()[1], IsStringEq("c"));
// 5th element is the empty string
ASSERT_THAT(*v.listElems()[4], IsStringEq(""));
}
TEST_F(PrimOpTest, split4) {
auto v = eval("builtins.split \"([[:upper:]]+)\" \" FOO \"");
ASSERT_THAT(v, IsListOfSize(3));
auto first = v.listElems()[0];
auto second = v.listElems()[1];
auto third = v.listElems()[2];
ASSERT_THAT(*first, IsStringEq(" "));
ASSERT_THAT(*second, IsListOfSize(1));
ASSERT_THAT(*second->listElems()[0], IsStringEq("FOO"));
ASSERT_THAT(*third, IsStringEq(" "));
}
TEST_F(PrimOpTest, match1) {
auto v = eval("builtins.match \"ab\" \"abc\"");
ASSERT_THAT(v, IsNull());
}
TEST_F(PrimOpTest, match2) {
auto v = eval("builtins.match \"abc\" \"abc\"");
ASSERT_THAT(v, IsListOfSize(0));
}
TEST_F(PrimOpTest, match3) {
auto v = eval("builtins.match \"a(b)(c)\" \"abc\"");
ASSERT_THAT(v, IsListOfSize(2));
ASSERT_THAT(*v.listElems()[0], IsStringEq("b"));
ASSERT_THAT(*v.listElems()[1], IsStringEq("c"));
}
TEST_F(PrimOpTest, match4) {
auto v = eval("builtins.match \"[[:space:]]+([[:upper:]]+)[[:space:]]+\" \" FOO \"");
ASSERT_THAT(v, IsListOfSize(1));
ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO"));
}
TEST_F(PrimOpTest, attrNames) {
auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }");
ASSERT_THAT(v, IsListOfSize(4));
// ensure that the list is sorted
const std::vector<std::string_view> expected { "a", "x", "y", "z" };
for (const auto [n, elem] : enumerate(v.listItems()))
ASSERT_THAT(*elem, IsStringEq(expected[n]));
}
TEST_F(PrimOpTest, genericClosure_not_strict) {
// Operator should not be used when startSet is empty
auto v = eval("builtins.genericClosure { startSet = []; }");
ASSERT_THAT(v, IsListOfSize(0));
}
} /* namespace nix */

90
src/libexpr/tests/search-path.cc
View file

@ -1,90 +0,0 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "search-path.hh"
namespace nix {
TEST(SearchPathElem, parse_justPath) {
ASSERT_EQ(
SearchPath::Elem::parse("foo"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "" },
.path = SearchPath::Path { .s = "foo" },
}));
}
TEST(SearchPathElem, parse_emptyPrefix) {
ASSERT_EQ(
SearchPath::Elem::parse("=foo"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "" },
.path = SearchPath::Path { .s = "foo" },
}));
}
TEST(SearchPathElem, parse_oneEq) {
ASSERT_EQ(
SearchPath::Elem::parse("foo=bar"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "foo" },
.path = SearchPath::Path { .s = "bar" },
}));
}
TEST(SearchPathElem, parse_twoEqs) {
ASSERT_EQ(
SearchPath::Elem::parse("foo=bar=baz"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "foo" },
.path = SearchPath::Path { .s = "bar=baz" },
}));
}
TEST(SearchPathElem, suffixIfPotentialMatch_justPath) {
SearchPath::Prefix prefix { .s = "" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) {
SearchPath::Prefix prefix { .s = "fooX" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) {
SearchPath::Prefix prefix { .s = "foo/bar" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" });
}
}

196
src/libexpr/tests/trivial.cc
View file

@ -1,196 +0,0 @@
#include "tests/libexpr.hh"
namespace nix {
// Testing of trivial expressions
class TrivialExpressionTest : public LibExprTest {};
TEST_F(TrivialExpressionTest, true) {
auto v = eval("true");
ASSERT_THAT(v, IsTrue());
}
TEST_F(TrivialExpressionTest, false) {
auto v = eval("false");
ASSERT_THAT(v, IsFalse());
}
TEST_F(TrivialExpressionTest, null) {
auto v = eval("null");
ASSERT_THAT(v, IsNull());
}
TEST_F(TrivialExpressionTest, 1) {
auto v = eval("1");
ASSERT_THAT(v, IsIntEq(1));
}
TEST_F(TrivialExpressionTest, 1plus1) {
auto v = eval("1+1");
ASSERT_THAT(v, IsIntEq(2));
}
TEST_F(TrivialExpressionTest, minus1) {
auto v = eval("-1");
ASSERT_THAT(v, IsIntEq(-1));
}
TEST_F(TrivialExpressionTest, 1minus1) {
auto v = eval("1-1");
ASSERT_THAT(v, IsIntEq(0));
}
TEST_F(TrivialExpressionTest, lambdaAdd) {
auto v = eval("let add = a: b: a + b; in add 1 2");
ASSERT_THAT(v, IsIntEq(3));
}
TEST_F(TrivialExpressionTest, list) {
auto v = eval("[]");
ASSERT_THAT(v, IsListOfSize(0));
}
TEST_F(TrivialExpressionTest, attrs) {
auto v = eval("{}");
ASSERT_THAT(v, IsAttrsOfSize(0));
}
TEST_F(TrivialExpressionTest, float) {
auto v = eval("1.234");
ASSERT_THAT(v, IsFloatEq(1.234));
}
TEST_F(TrivialExpressionTest, updateAttrs) {
auto v = eval("{ a = 1; } // { b = 2; a = 3; }");
ASSERT_THAT(v, IsAttrsOfSize(2));
auto a = v.attrs->find(createSymbol("a"));
ASSERT_NE(a, nullptr);
ASSERT_THAT(*a->value, IsIntEq(3));
auto b = v.attrs->find(createSymbol("b"));
ASSERT_NE(b, nullptr);
ASSERT_THAT(*b->value, IsIntEq(2));
}
TEST_F(TrivialExpressionTest, hasAttrOpFalse) {
auto v = eval("{} ? a");
ASSERT_THAT(v, IsFalse());
}
TEST_F(TrivialExpressionTest, hasAttrOpTrue) {
auto v = eval("{ a = 123; } ? a");
ASSERT_THAT(v, IsTrue());
}
TEST_F(TrivialExpressionTest, withFound) {
auto v = eval("with { a = 23; }; a");
ASSERT_THAT(v, IsIntEq(23));
}
TEST_F(TrivialExpressionTest, withNotFound) {
ASSERT_THROW(eval("with {}; a"), Error);
}
TEST_F(TrivialExpressionTest, withOverride) {
auto v = eval("with { a = 23; }; with { a = 42; }; a");
ASSERT_THAT(v, IsIntEq(42));
}
TEST_F(TrivialExpressionTest, letOverWith) {
auto v = eval("let a = 23; in with { a = 1; }; a");
ASSERT_THAT(v, IsIntEq(23));
}
TEST_F(TrivialExpressionTest, multipleLet) {
auto v = eval("let a = 23; in let a = 42; in a");
ASSERT_THAT(v, IsIntEq(42));
}
TEST_F(TrivialExpressionTest, defaultFunctionArgs) {
auto v = eval("({ a ? 123 }: a) {}");
ASSERT_THAT(v, IsIntEq(123));
}
TEST_F(TrivialExpressionTest, defaultFunctionArgsOverride) {
auto v = eval("({ a ? 123 }: a) { a = 5; }");
ASSERT_THAT(v, IsIntEq(5));
}
TEST_F(TrivialExpressionTest, defaultFunctionArgsCaptureBack) {
auto v = eval("({ a ? 123 }@args: args) {}");
ASSERT_THAT(v, IsAttrsOfSize(0));
}
TEST_F(TrivialExpressionTest, defaultFunctionArgsCaptureFront) {
auto v = eval("(args@{ a ? 123 }: args) {}");
ASSERT_THAT(v, IsAttrsOfSize(0));
}
TEST_F(TrivialExpressionTest, assertThrows) {
ASSERT_THROW(eval("let x = arg: assert arg == 1; 123; in x 2"), Error);
}
TEST_F(TrivialExpressionTest, assertPassed) {
auto v = eval("let x = arg: assert arg == 1; 123; in x 1");
ASSERT_THAT(v, IsIntEq(123));
}
class AttrSetMergeTrvialExpressionTest :
public TrivialExpressionTest,
public testing::WithParamInterface<const char*>
{};
TEST_P(AttrSetMergeTrvialExpressionTest, attrsetMergeLazy) {
// Usually Nix rejects duplicate keys in an attrset but it does allow
// so if it is an attribute set that contains disjoint sets of keys.
// The below is equivalent to `{a.b = 1; a.c = 2; }`.
// The attribute set `a` will be a Thunk at first as the attribuets
// have to be merged (or otherwise computed) and that is done in a lazy
// manner.
auto expr = GetParam();
auto v = eval(expr);
ASSERT_THAT(v, IsAttrsOfSize(1));
auto a = v.attrs->find(createSymbol("a"));
ASSERT_NE(a, nullptr);
ASSERT_THAT(*a->value, IsThunk());
state.forceValue(*a->value, noPos);
ASSERT_THAT(*a->value, IsAttrsOfSize(2));
auto b = a->value->attrs->find(createSymbol("b"));
ASSERT_NE(b, nullptr);
ASSERT_THAT(*b->value, IsIntEq(1));
auto c = a->value->attrs->find(createSymbol("c"));
ASSERT_NE(c, nullptr);
ASSERT_THAT(*c->value, IsIntEq(2));
}
INSTANTIATE_TEST_SUITE_P(
attrsetMergeLazy,
AttrSetMergeTrvialExpressionTest,
testing::Values(
"{ a.b = 1; a.c = 2; }",
"{ a = { b = 1; }; a = { c = 2; }; }"
)
);
TEST_F(TrivialExpressionTest, functor) {
auto v = eval("{ __functor = self: arg: self.v + arg; v = 10; } 5");
ASSERT_THAT(v, IsIntEq(15));
}
TEST_F(TrivialExpressionTest, bindOr) {
auto v = eval("{ or = 1; }");
ASSERT_THAT(v, IsAttrsOfSize(1));
auto b = v.attrs->find(createSymbol("or"));
ASSERT_NE(b, nullptr);
ASSERT_THAT(*b->value, IsIntEq(1));
}
TEST_F(TrivialExpressionTest, orCantBeUsed) {
ASSERT_THROW(eval("let or = 1; in or"), Error);
}
} /* namespace nix */

162
src/libexpr/tests/value/context.cc
View file

@ -1,162 +0,0 @@
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
#include <rapidcheck/gtest.h>
#include "tests/path.hh"
#include "tests/libexpr.hh"
#include "tests/value/context.hh"
namespace nix {
// Test a few cases of invalid string context elements.
TEST(NixStringContextElemTest, empty_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(""),
BadNixStringContextElem);
}
TEST(NixStringContextElemTest, single_bang_invalid) {
EXPECT_THROW(
NixStringContextElem::parse("!"),
BadNixStringContextElem);
}
TEST(NixStringContextElemTest, double_bang_invalid) {
EXPECT_THROW(
NixStringContextElem::parse("!!/"),
BadStorePath);
}
TEST(NixStringContextElemTest, eq_slash_invalid) {
EXPECT_THROW(
NixStringContextElem::parse("=/"),
BadStorePath);
}
TEST(NixStringContextElemTest, slash_invalid) {
EXPECT_THROW(
NixStringContextElem::parse("/"),
BadStorePath);
}
/**
* Round trip (string <-> data structure) test for
* `NixStringContextElem::Opaque`.
*/
TEST(NixStringContextElemTest, opaque) {
std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
auto elem = NixStringContextElem::parse(opaque);
auto * p = std::get_if<NixStringContextElem::Opaque>(&elem.raw);
ASSERT_TRUE(p);
ASSERT_EQ(p->path, StorePath { opaque });
ASSERT_EQ(elem.to_string(), opaque);
}
/**
* Round trip (string <-> data structure) test for
* `NixStringContextElem::DrvDeep`.
*/
TEST(NixStringContextElemTest, drvDeep) {
std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(drvDeep);
auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem.raw);
ASSERT_TRUE(p);
ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) });
ASSERT_EQ(elem.to_string(), drvDeep);
}
/**
* Round trip (string <-> data structure) test for a simpler
* `NixStringContextElem::Built`.
*/
TEST(NixStringContextElemTest, built_opaque) {
std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(built);
auto * p = std::get_if<NixStringContextElem::Built>(&elem.raw);
ASSERT_TRUE(p);
ASSERT_EQ(p->output, "foo");
ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
.path = StorePath { built.substr(5) },
}));
ASSERT_EQ(elem.to_string(), built);
}
/**
* Round trip (string <-> data structure) test for a more complex,
* inductive `NixStringContextElem::Built`.
*/
TEST(NixStringContextElemTest, built_built) {
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(built, mockXpSettings);
auto * p = std::get_if<NixStringContextElem::Built>(&elem.raw);
ASSERT_TRUE(p);
ASSERT_EQ(p->output, "foo");
auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath);
ASSERT_TRUE(drvPath);
ASSERT_EQ(drvPath->output, "bar");
ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
.path = StorePath { built.substr(9) },
}));
ASSERT_EQ(elem.to_string(), built);
}
/**
* Without the right experimental features enabled, we cannot parse a
* complex inductive string context element.
*/
TEST(NixStringContextElemTest, built_built_xp) {
ASSERT_THROW(
NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature);
}
}
namespace rc {
using namespace nix;
Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary()
{
return gen::just(NixStringContextElem::DrvDeep {
.drvPath = *gen::arbitrary<StorePath>(),
});
}
Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
{
switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {
case 0:
return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>());
case 1:
return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>());
case 2:
return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>());
default:
assert(false);
}
}
}
namespace nix {
#ifndef COVERAGE
RC_GTEST_PROP(
NixStringContextElemTest,
prop_round_rip,
(const NixStringContextElem & o))
{
RC_ASSERT(o == NixStringContextElem::parse(o.to_string()));
}
#endif
}

31
src/libexpr/tests/value/context.hh
View file

@ -1,31 +0,0 @@
#pragma once
///@file
#include <rapidcheck/gen/Arbitrary.h>
#include <value/context.hh>
namespace rc {
using namespace nix;
template<>
struct Arbitrary<NixStringContextElem::Opaque> {
static Gen<NixStringContextElem::Opaque> arbitrary();
};
template<>
struct Arbitrary<NixStringContextElem::Built> {
static Gen<NixStringContextElem::Built> arbitrary();
};
template<>
struct Arbitrary<NixStringContextElem::DrvDeep> {
static Gen<NixStringContextElem::DrvDeep> arbitrary();
};
template<>
struct Arbitrary<NixStringContextElem> {
static Gen<NixStringContextElem> arbitrary();
};
}

236
src/libexpr/tests/value/print.cc
View file

@ -1,236 +0,0 @@
#include "tests/libexpr.hh"
#include "value.hh"
namespace nix {
using namespace testing;
struct ValuePrintingTests : LibExprTest
{
template<class... A>
void test(Value v, std::string_view expected, A... args)
{
std::stringstream out;
v.print(state.symbols, out, args...);
ASSERT_EQ(out.str(), expected);
}
};
TEST_F(ValuePrintingTests, tInt)
{
Value vInt;
vInt.mkInt(10);
test(vInt, "10");
}
TEST_F(ValuePrintingTests, tBool)
{
Value vBool;
vBool.mkBool(true);
test(vBool, "true");
}
TEST_F(ValuePrintingTests, tString)
{
Value vString;
vString.mkString("some-string");
test(vString, "\"some-string\"");
}
TEST_F(ValuePrintingTests, tPath)
{
Value vPath;
vPath.mkString("/foo");
test(vPath, "\"/foo\"");
}
TEST_F(ValuePrintingTests, tNull)
{
Value vNull;
vNull.mkNull();
test(vNull, "null");
}
TEST_F(ValuePrintingTests, tAttrs)
{
Value vOne;
vOne.mkInt(1);
Value vTwo;
vTwo.mkInt(2);
BindingsBuilder builder(state, state.allocBindings(10));
builder.insert(state.symbols.create("one"), &vOne);
builder.insert(state.symbols.create("two"), &vTwo);
Value vAttrs;
vAttrs.mkAttrs(builder.finish());
test(vAttrs, "{ one = 1; two = 2; }");
}
TEST_F(ValuePrintingTests, tList)
{
Value vOne;
vOne.mkInt(1);
Value vTwo;
vTwo.mkInt(2);
Value vList;
state.mkList(vList, 5);
vList.bigList.elems[0] = &vOne;
vList.bigList.elems[1] = &vTwo;
vList.bigList.size = 3;
test(vList, "[ 1 2 (nullptr) ]");
}
TEST_F(ValuePrintingTests, vThunk)
{
Value vThunk;
vThunk.mkThunk(nullptr, nullptr);
test(vThunk, "<CODE>");
}
TEST_F(ValuePrintingTests, vApp)
{
Value vApp;
vApp.mkApp(nullptr, nullptr);
test(vApp, "<CODE>");
}
TEST_F(ValuePrintingTests, vLambda)
{
Value vLambda;
vLambda.mkLambda(nullptr, nullptr);
test(vLambda, "<LAMBDA>");
}
TEST_F(ValuePrintingTests, vPrimOp)
{
Value vPrimOp;
vPrimOp.mkPrimOp(nullptr);
test(vPrimOp, "<PRIMOP>");
}
TEST_F(ValuePrintingTests, vPrimOpApp)
{
Value vPrimOpApp;
vPrimOpApp.mkPrimOpApp(nullptr, nullptr);
test(vPrimOpApp, "<PRIMOP-APP>");
}
TEST_F(ValuePrintingTests, vExternal)
{
struct MyExternal : ExternalValueBase
{
public:
std::string showType() const override
{
return "";
}
std::string typeOf() const override
{
return "";
}
virtual std::ostream & print(std::ostream & str) const override
{
str << "testing-external!";
return str;
}
} myExternal;
Value vExternal;
vExternal.mkExternal(&myExternal);
test(vExternal, "testing-external!");
}
TEST_F(ValuePrintingTests, vFloat)
{
Value vFloat;
vFloat.mkFloat(2.0);
test(vFloat, "2");
}
TEST_F(ValuePrintingTests, vBlackhole)
{
Value vBlackhole;
vBlackhole.mkBlackhole();
test(vBlackhole, "«potential infinite recursion»");
}
TEST_F(ValuePrintingTests, depthAttrs)
{
Value vOne;
vOne.mkInt(1);
Value vTwo;
vTwo.mkInt(2);
BindingsBuilder builder(state, state.allocBindings(10));
builder.insert(state.symbols.create("one"), &vOne);
builder.insert(state.symbols.create("two"), &vTwo);
Value vAttrs;
vAttrs.mkAttrs(builder.finish());
BindingsBuilder builder2(state, state.allocBindings(10));
builder2.insert(state.symbols.create("one"), &vOne);
builder2.insert(state.symbols.create("two"), &vTwo);
builder2.insert(state.symbols.create("nested"), &vAttrs);
Value vNested;
vNested.mkAttrs(builder2.finish());
test(vNested, "{ nested = «too deep»; one = «too deep»; two = «too deep»; }", false, 1);
test(vNested, "{ nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; }", false, 2);
test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 3);
test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 4);
}
TEST_F(ValuePrintingTests, depthList)
{
Value vOne;
vOne.mkInt(1);
Value vTwo;
vTwo.mkInt(2);
BindingsBuilder builder(state, state.allocBindings(10));
builder.insert(state.symbols.create("one"), &vOne);
builder.insert(state.symbols.create("two"), &vTwo);
Value vAttrs;
vAttrs.mkAttrs(builder.finish());
BindingsBuilder builder2(state, state.allocBindings(10));
builder2.insert(state.symbols.create("one"), &vOne);
builder2.insert(state.symbols.create("two"), &vTwo);
builder2.insert(state.symbols.create("nested"), &vAttrs);
Value vNested;
vNested.mkAttrs(builder2.finish());
Value vList;
state.mkList(vList, 5);
vList.bigList.elems[0] = &vOne;
vList.bigList.elems[1] = &vTwo;
vList.bigList.elems[2] = &vNested;
vList.bigList.size = 3;
test(vList, "[ «too deep» «too deep» «too deep» ]", false, 1);
test(vList, "[ 1 2 { nested = «too deep»; one = «too deep»; two = «too deep»; } ]", false, 2);
test(vList, "[ 1 2 { nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; } ]", false, 3);
test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 4);
test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 5);
}
} // namespace nix

176
src/libexpr/value.hh
View file

@ -3,10 +3,12 @@
#include <cassert>
#include <climits>
#include <span>
#include "symbol-table.hh"
#include "value/context.hh"
#include "input-accessor.hh"
#include "source-path.hh"
#if HAVE_BOEHMGC
#include <gc/gc_allocator.h>
@ -31,7 +33,6 @@ typedef enum {
tThunk,
tApp,
tLambda,
tBlackhole,
tPrimOp,
tPrimOpApp,
tExternal,
@ -61,12 +62,12 @@ class Bindings;
struct Env;
struct Expr;
struct ExprLambda;
struct ExprBlackHole;
struct PrimOp;
class Symbol;
class PosIdx;
struct Pos;
class StorePath;
class Store;
class EvalState;
class XMLWriter;
@ -151,49 +152,67 @@ public:
// type() == nThunk
inline bool isThunk() const { return internalType == tThunk; };
inline bool isApp() const { return internalType == tApp; };
inline bool isBlackhole() const { return internalType == tBlackhole; };
inline bool isBlackhole() const;
// type() == nFunction
inline bool isLambda() const { return internalType == tLambda; };
inline bool isPrimOp() const { return internalType == tPrimOp; };
inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
/**
* Strings in the evaluator carry a so-called `context` which
* is a list of strings representing store paths. This is to
* allow users to write things like
*
* "--with-freetype2-library=" + freetype + "/lib"
*
* where `freetype` is a derivation (or a source to be copied
* to the store). If we just concatenated the strings without
* keeping track of the referenced store paths, then if the
* string is used as a derivation attribute, the derivation
* will not have the correct dependencies in its inputDrvs and
* inputSrcs.
* The semantics of the context is as follows: when a string
* with context C is used as a derivation attribute, then the
* derivations in C will be added to the inputDrvs of the
* derivation, and the other store paths in C will be added to
* the inputSrcs of the derivations.
* For canonicity, the store paths should be in sorted order.
*/
struct StringWithContext {
const char * c_str;
const char * * context; // must be in sorted order
};
struct Path {
InputAccessor * accessor;
const char * path;
};
struct ClosureThunk {
Env * env;
Expr * expr;
};
struct FunctionApplicationThunk {
Value * left, * right;
};
struct Lambda {
Env * env;
ExprLambda * fun;
};
union
{
NixInt integer;
bool boolean;
/**
* Strings in the evaluator carry a so-called `context` which
* is a list of strings representing store paths. This is to
* allow users to write things like
StringWithContext string;
* "--with-freetype2-library=" + freetype + "/lib"
* where `freetype` is a derivation (or a source to be copied
* to the store). If we just concatenated the strings without
* keeping track of the referenced store paths, then if the
* string is used as a derivation attribute, the derivation
* will not have the correct dependencies in its inputDrvs and
* inputSrcs.
* The semantics of the context is as follows: when a string
* with context C is used as a derivation attribute, then the
* derivations in C will be added to the inputDrvs of the
* derivation, and the other store paths in C will be added to
* the inputSrcs of the derivations.
* For canonicity, the store paths should be in sorted order.
*/
struct {
const char * c_str;
const char * * context; // must be in sorted order
} string;
struct {
InputAccessor * accessor;
const char * path;
} _path;
Path _path;
Bindings * attrs;
struct {
@ -201,21 +220,11 @@ public:
Value * * elems;
} bigList;
Value * smallList[2];
struct {
Env * env;
Expr * expr;
} thunk;
struct {
Value * left, * right;
} app;
struct {
Env * env;
ExprLambda * fun;
} lambda;
ClosureThunk thunk;
FunctionApplicationThunk app;
Lambda lambda;
PrimOp * primOp;
struct {
Value * left, * right;
} primOpApp;
FunctionApplicationThunk primOpApp;
ExternalValueBase * external;
NixFloat fpoint;
};
@ -240,7 +249,7 @@ public:
case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
case tExternal: return nExternal;
case tFloat: return nFloat;
case tThunk: case tApp: case tBlackhole: return nThunk;
case tThunk: case tApp: return nThunk;
}
if (invalidIsThunk)
return nThunk;
@ -348,19 +357,9 @@ public:
lambda.fun = f;
}
inline void mkBlackhole()
{
internalType = tBlackhole;
// Value will be overridden anyways
}
inline void mkPrimOp(PrimOp * p)
{
clearValue();
internalType = tPrimOp;
primOp = p;
}
inline void mkBlackhole();
void mkPrimOp(PrimOp * p);
inline void mkPrimOpApp(Value * l, Value * r)
{
@ -393,7 +392,13 @@ public:
return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
}
const Value * const * listElems() const
std::span<Value * const> listItems() const
{
assert(isList());
return std::span<Value * const>(listElems(), listSize());
}
Value * const * listElems() const
{
return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
}
@ -412,41 +417,12 @@ public:
*/
bool isTrivial() const;
auto listItems()
{
struct ListIterable
{
typedef Value * const * iterator;
iterator _begin, _end;
iterator begin() const { return _begin; }
iterator end() const { return _end; }
};
assert(isList());
auto begin = listElems();
return ListIterable { begin, begin + listSize() };
}
auto listItems() const
{
struct ConstListIterable
{
typedef const Value * const * iterator;
iterator _begin, _end;
iterator begin() const { return _begin; }
iterator end() const { return _end; }
};
assert(isList());
auto begin = listElems();
return ConstListIterable { begin, begin + listSize() };
}
SourcePath path() const
{
assert(internalType == tPath);
return SourcePath {
.accessor = ref(_path.accessor->shared_from_this()),
.path = CanonPath(CanonPath::unchecked_t(), _path.path)
};
return SourcePath(
ref(_path.accessor->shared_from_this()),
CanonPath(CanonPath::unchecked_t(), _path.path));
}
std::string_view string_view() const
@ -468,6 +444,20 @@ public:
};
extern ExprBlackHole eBlackHole;
bool Value::isBlackhole() const
{
return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole;
}
void Value::mkBlackhole()
{
internalType = tThunk;
thunk.expr = (Expr*) &eBlackHole;
}
#if HAVE_BOEHMGC
typedef std::vector<Value *, traceable_allocator<Value *>> ValueVector;
typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *>>> ValueMap;

73
src/libfetchers/cache.cc
View file

@ -20,6 +20,9 @@ create table if not exists Cache (
);
)sql";
// FIXME: we should periodically purge/nuke this cache to prevent it
// from growing too big.
struct CacheImpl : Cache
{
struct State
@ -48,8 +51,62 @@ struct CacheImpl : Cache
"select info, path, immutable, timestamp from Cache where input = ?");
}
void upsert(
const Attrs & inAttrs,
const Attrs & infoAttrs) override
{
_state.lock()->add.use()
(attrsToJSON(inAttrs).dump())
(attrsToJSON(infoAttrs).dump())
("") // no path
(false)
(time(0)).exec();
}
std::optional<Attrs> lookup(const Attrs & inAttrs) override
{
if (auto res = lookupExpired(inAttrs))
return std::move(res->infoAttrs);
return {};
}
std::optional<Attrs> lookupWithTTL(const Attrs & inAttrs) override
{
if (auto res = lookupExpired(inAttrs)) {
if (!res->expired)
return std::move(res->infoAttrs);
debug("ignoring expired cache entry '%s'",
attrsToJSON(inAttrs).dump());
}
return {};
}
std::optional<Result2> lookupExpired(const Attrs & inAttrs) override
{
auto state(_state.lock());
auto inAttrsJSON = attrsToJSON(inAttrs).dump();
auto stmt(state->lookup.use()(inAttrsJSON));
if (!stmt.next()) {
debug("did not find cache entry for '%s'", inAttrsJSON);
return {};
}
auto infoJSON = stmt.getStr(0);
auto locked = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3);
debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON);
return Result2 {
.expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
.infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
};
}
void add(
ref<Store> store,
Store & store,
const Attrs & inAttrs,
const Attrs & infoAttrs,
const StorePath & storePath,
@ -58,13 +115,13 @@ struct CacheImpl : Cache
_state.lock()->add.use()
(attrsToJSON(inAttrs).dump())
(attrsToJSON(infoAttrs).dump())
(store->printStorePath(storePath))
(store.printStorePath(storePath))
(locked)
(time(0)).exec();
}
std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store,
Store & store,
const Attrs & inAttrs) override
{
if (auto res = lookupExpired(store, inAttrs)) {
@ -77,7 +134,7 @@ struct CacheImpl : Cache
}
std::optional<Result> lookupExpired(
ref<Store> store,
Store & store,
const Attrs & inAttrs) override
{
auto state(_state.lock());
@ -91,19 +148,19 @@ struct CacheImpl : Cache
}
auto infoJSON = stmt.getStr(0);
auto storePath = store->parseStorePath(stmt.getStr(1));
auto storePath = store.parseStorePath(stmt.getStr(1));
auto locked = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3);
store->addTempRoot(storePath);
if (!store->isValidPath(storePath)) {
store.addTempRoot(storePath);
if (!store.isValidPath(storePath)) {
// FIXME: we could try to substitute 'storePath'.
debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
return {};
}
debug("using cache entry '%s' -> '%s', '%s'",
inAttrsJSON, infoJSON, store->printStorePath(storePath));
inAttrsJSON, infoJSON, store.printStorePath(storePath));
return Result {
.expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),

45
src/libfetchers/cache.hh
View file

@ -6,19 +6,58 @@
namespace nix::fetchers {
/**
* A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp
* for expiration.
*/
struct Cache
{
virtual ~Cache() { }
/**
* Add a value to the cache. The cache is an arbitrary mapping of
* Attrs to Attrs.
*/
virtual void upsert(
const Attrs & inAttrs,
const Attrs & infoAttrs) = 0;
/**
* Look up a key with infinite TTL.
*/
virtual std::optional<Attrs> lookup(
const Attrs & inAttrs) = 0;
/**
* Look up a key. Return nothing if its TTL has exceeded
* `settings.tarballTTL`.
*/
virtual std::optional<Attrs> lookupWithTTL(
const Attrs & inAttrs) = 0;
struct Result2
{
bool expired = false;
Attrs infoAttrs;
};
/**
* Look up a key. Return a bool denoting whether its TTL has
* exceeded `settings.tarballTTL`.
*/
virtual std::optional<Result2> lookupExpired(
const Attrs & inAttrs) = 0;
/* Old cache for things that have a store path. */
virtual void add(
ref<Store> store,
Store & store,
const Attrs & inAttrs,
const Attrs & infoAttrs,
const StorePath & storePath,
bool locked) = 0;
virtual std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store,
Store & store,
const Attrs & inAttrs) = 0;
struct Result
@ -29,7 +68,7 @@ struct Cache
};
virtual std::optional<Result> lookupExpired(
ref<Store> store,
Store & store,
const Attrs & inAttrs) = 0;
};

68
src/libfetchers/fetch-to-store.cc Normal file
View file

@ -0,0 +1,68 @@
#include "fetch-to-store.hh"
#include "fetchers.hh"
#include "cache.hh"
namespace nix {
StorePath fetchToStore(
Store & store,
const SourcePath & path,
std::string_view name,
ContentAddressMethod method,
PathFilter * filter,
RepairFlag repair)
{
// FIXME: add an optimisation for the case where the accessor is
// an FSInputAccessor pointing to a store path.
std::optional<fetchers::Attrs> cacheKey;
if (!filter && path.accessor->fingerprint) {
cacheKey = fetchers::Attrs{
{"_what", "fetchToStore"},
{"store", store.storeDir},
{"name", std::string(name)},
{"fingerprint", *path.accessor->fingerprint},
{
"method",
std::visit(overloaded {
[](const TextIngestionMethod &) {
return "text";
},
[](const FileIngestionMethod & fim) {
switch (fim) {
case FileIngestionMethod::Flat: return "flat";
case FileIngestionMethod::Recursive: return "nar";
default: assert(false);
}
},
}, method.raw),
},
{"path", path.path.abs()}
};
if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
debug("store path cache hit for '%s'", path);
return res->second;
}
} else
debug("source path '%s' is uncacheable", path);
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path));
auto filter2 = filter ? *filter : defaultPathFilter;
auto storePath =
settings.readOnlyMode
? store.computeStorePath(
name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first
: store.addToStore(
name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair);
if (cacheKey)
fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
return storePath;
}
}

22
src/libfetchers/fetch-to-store.hh Normal file
View file

@ -0,0 +1,22 @@
#pragma once
#include "source-path.hh"
#include "store-api.hh"
#include "file-system.hh"
#include "repair-flag.hh"
#include "file-content-address.hh"
namespace nix {
/**
* Copy the `path` to the Nix store.
*/
StorePath fetchToStore(
Store & store,
const SourcePath & path,
std::string_view name = "source",
ContentAddressMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
}

36
src/libfetchers/fetchers.cc
View file

@ -1,5 +1,8 @@
#include "fetchers.hh"
#include "store-api.hh"
#include "input-accessor.hh"
#include "source-path.hh"
#include "fetch-to-store.hh"
#include <nlohmann/json.hpp>
@ -107,6 +110,11 @@ Input Input::fromAttrs(Attrs && attrs)
return std::move(*res);
}
std::optional<std::string> Input::getFingerprint(ref<Store> store) const
{
return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
}
ParsedURL Input::toURL() const
{
if (!scheme)
@ -219,6 +227,16 @@ std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
return {std::move(storePath), input};
}
std::pair<ref<InputAccessor>, Input> Input::getAccessor(ref<Store> store) const
{
try {
return scheme->getAccessor(store, *this);
} catch (Error & e) {
e.addTrace({}, "while fetching the input '%s'", to_string());
throw;
}
}
Input Input::applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const
@ -273,8 +291,8 @@ std::string Input::getType() const
std::optional<Hash> Input::getNarHash() const
{
if (auto s = maybeGetStrAttr(attrs, "narHash")) {
auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
if (hash.type != htSHA256)
auto hash = s->empty() ? Hash(HashAlgorithm::SHA256) : Hash::parseSRI(*s);
if (hash.algo != HashAlgorithm::SHA256)
throw UsageError("narHash must use SHA-256");
return hash;
}
@ -298,7 +316,7 @@ std::optional<Hash> Input::getRev() const
} catch (BadHash &e) {
// Default to sha1 for backwards compatibility with existing
// usages (e.g. `builtins.fetchTree` calls or flake inputs).
hash = Hash::parseAny(*s, htSHA1);
hash = Hash::parseAny(*s, HashAlgorithm::SHA1);
}
}
@ -355,6 +373,18 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
throw Error("do not know how to clone input '%s'", input.to_string());
}
std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
{
auto [accessor, input2] = getAccessor(store, input);
auto storePath = fetchToStore(*store, SourcePath(accessor), input2.getName());
return {storePath, input2};
}
std::pair<ref<InputAccessor>, Input> InputScheme::getAccessor(ref<Store> store, const Input & input) const
{
throw UnimplementedError("InputScheme must implement fetch() or getAccessor()");
}
std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
{
return {};

17
src/libfetchers/fetchers.hh
View file

@ -10,7 +10,7 @@
#include <memory>
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; class StorePath; }
namespace nix { class Store; class StorePath; struct InputAccessor; }
namespace nix::fetchers {
@ -83,6 +83,8 @@ public:
*/
std::pair<StorePath, Input> fetch(ref<Store> store) const;
std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store) const;
Input applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const;
@ -111,6 +113,12 @@ public:
std::optional<Hash> getRev() const;
std::optional<uint64_t> getRevCount() const;
std::optional<time_t> getLastModified() const;
/**
* For locked inputs, return a string that uniquely specifies the
* content of the input (typically a commit hash or content hash).
*/
std::optional<std::string> getFingerprint(ref<Store> store) const;
};
@ -167,7 +175,9 @@ struct InputScheme
std::string_view contents,
std::optional<std::string> commitMsg) const;
virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input);
virtual std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const;
/**
* Is this `InputScheme` part of an experimental feature?
@ -176,6 +186,9 @@ struct InputScheme
virtual bool isDirect(const Input & input) const
{ return true; }
virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
{ return std::nullopt; }
};
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

83
src/libfetchers/filtering-input-accessor.cc Normal file
View file

@ -0,0 +1,83 @@
#include "filtering-input-accessor.hh"
namespace nix {
std::string FilteringInputAccessor::readFile(const CanonPath & path)
{
checkAccess(path);
return next->readFile(prefix + path);
}
bool FilteringInputAccessor::pathExists(const CanonPath & path)
{
return isAllowed(path) && next->pathExists(prefix + path);
}
std::optional<InputAccessor::Stat> FilteringInputAccessor::maybeLstat(const CanonPath & path)
{
checkAccess(path);
return next->maybeLstat(prefix + path);
}
InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path)
{
checkAccess(path);
DirEntries entries;
for (auto & entry : next->readDirectory(prefix + path)) {
if (isAllowed(path + entry.first))
entries.insert(std::move(entry));
}
return entries;
}
std::string FilteringInputAccessor::readLink(const CanonPath & path)
{
checkAccess(path);
return next->readLink(prefix + path);
}
std::string FilteringInputAccessor::showPath(const CanonPath & path)
{
return next->showPath(prefix + path);
}
void FilteringInputAccessor::checkAccess(const CanonPath & path)
{
if (!isAllowed(path))
throw makeNotAllowedError
? makeNotAllowedError(path)
: RestrictedPathError("access to path '%s' is forbidden", showPath(path));
}
struct AllowListInputAccessorImpl : AllowListInputAccessor
{
std::set<CanonPath> allowedPaths;
AllowListInputAccessorImpl(
ref<InputAccessor> next,
std::set<CanonPath> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
: AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError))
, allowedPaths(std::move(allowedPaths))
{ }
bool isAllowed(const CanonPath & path) override
{
return path.isAllowed(allowedPaths);
}
void allowPath(CanonPath path) override
{
allowedPaths.insert(std::move(path));
}
};
ref<AllowListInputAccessor> AllowListInputAccessor::create(
ref<InputAccessor> next,
std::set<CanonPath> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
{
return make_ref<AllowListInputAccessorImpl>(next, std::move(allowedPaths), std::move(makeNotAllowedError));
}
}

74
src/libfetchers/filtering-input-accessor.hh Normal file
View file

@ -0,0 +1,74 @@
#pragma once
#include "input-accessor.hh"
#include "source-path.hh"
namespace nix {
/**
* A function that should throw an exception of type
* `RestrictedPathError` explaining that access to `path` is
* forbidden.
*/
typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
/**
* An abstract wrapping `InputAccessor` that performs access
* control. Subclasses should override `isAllowed()` to implement an
* access control policy. The error message is customized at construction.
*/
struct FilteringInputAccessor : InputAccessor
{
ref<InputAccessor> next;
CanonPath prefix;
MakeNotAllowedError makeNotAllowedError;
FilteringInputAccessor(const SourcePath & src, MakeNotAllowedError && makeNotAllowedError)
: next(src.accessor)
, prefix(src.path)
, makeNotAllowedError(std::move(makeNotAllowedError))
{ }
std::string readFile(const CanonPath & path) override;
bool pathExists(const CanonPath & path) override;
std::optional<Stat> maybeLstat(const CanonPath & path) override;
DirEntries readDirectory(const CanonPath & path) override;
std::string readLink(const CanonPath & path) override;
std::string showPath(const CanonPath & path) override;
/**
* Call `makeNotAllowedError` to throw a `RestrictedPathError`
* exception if `isAllowed()` returns `false` for `path`.
*/
void checkAccess(const CanonPath & path);
/**
* Return `true` iff access to path is allowed.
*/
virtual bool isAllowed(const CanonPath & path) = 0;
};
/**
* A wrapping `InputAccessor` that checks paths against an allow-list.
*/
struct AllowListInputAccessor : public FilteringInputAccessor
{
/**
* Grant access to the specified path.
*/
virtual void allowPath(CanonPath path) = 0;
static ref<AllowListInputAccessor> create(
ref<InputAccessor> next,
std::set<CanonPath> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError);
using FilteringInputAccessor::FilteringInputAccessor;
};
}

78
src/libfetchers/fs-input-accessor.cc
View file

@ -4,20 +4,14 @@
namespace nix {
struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
struct FSInputAccessor : InputAccessor, PosixSourceAccessor
{
CanonPath root;
std::optional<std::set<CanonPath>> allowedPaths;
MakeNotAllowedError makeNotAllowedError;
FSInputAccessorImpl(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
FSInputAccessor(const CanonPath & root)
: root(root)
, allowedPaths(std::move(allowedPaths))
, makeNotAllowedError(std::move(makeNotAllowedError))
{
displayPrefix = root.isRoot() ? "" : root.abs();
}
void readFile(
@ -26,39 +20,30 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
std::function<void(uint64_t)> sizeCallback) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
}
bool pathExists(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
return PosixSourceAccessor::pathExists(makeAbsPath(path));
}
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::maybeLstat(absPath);
return PosixSourceAccessor::maybeLstat(makeAbsPath(path));
}
DirEntries readDirectory(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
DirEntries res;
for (auto & entry : PosixSourceAccessor::readDirectory(absPath))
if (isAllowed(absPath + entry.first))
res.emplace(entry);
for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path)))
res.emplace(entry);
return res;
}
std::string readLink(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::readLink(absPath);
return PosixSourceAccessor::readLink(makeAbsPath(path));
}
CanonPath makeAbsPath(const CanonPath & path)
@ -66,59 +51,22 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
return root + path;
}
void checkAllowed(const CanonPath & absPath) override
{
if (!isAllowed(absPath))
throw makeNotAllowedError
? makeNotAllowedError(absPath)
: RestrictedPathError("access to path '%s' is forbidden", absPath);
}
bool isAllowed(const CanonPath & absPath)
{
if (!absPath.isWithin(root))
return false;
if (allowedPaths) {
auto p = absPath.removePrefix(root);
if (!p.isAllowed(*allowedPaths))
return false;
}
return true;
}
void allowPath(CanonPath path) override
{
if (allowedPaths)
allowedPaths->insert(std::move(path));
}
bool hasAccessControl() override
{
return (bool) allowedPaths;
}
std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override
{
return makeAbsPath(path);
}
};
ref<FSInputAccessor> makeFSInputAccessor(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
ref<InputAccessor> makeFSInputAccessor(const CanonPath & root)
{
return make_ref<FSInputAccessorImpl>(root, std::move(allowedPaths), std::move(makeNotAllowedError));
return make_ref<FSInputAccessor>(root);
}
ref<FSInputAccessor> makeStorePathAccessor(
ref<InputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath,
MakeNotAllowedError && makeNotAllowedError)
const StorePath & storePath)
{
return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError));
return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)));
}
SourcePath getUnfilteredRootPath(CanonPath path)

23
src/libfetchers/fs-input-accessor.hh
View file

@ -1,32 +1,19 @@
#pragma once
#include "input-accessor.hh"
#include "source-path.hh"
namespace nix {
class StorePath;
class Store;
struct FSInputAccessor : InputAccessor
{
virtual void checkAllowed(const CanonPath & absPath) = 0;
ref<InputAccessor> makeFSInputAccessor(
const CanonPath & root);
virtual void allowPath(CanonPath path) = 0;
virtual bool hasAccessControl() = 0;
};
typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
ref<FSInputAccessor> makeFSInputAccessor(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths = {},
MakeNotAllowedError && makeNotAllowedError = {});
ref<FSInputAccessor> makeStorePathAccessor(
ref<InputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath,
MakeNotAllowedError && makeNotAllowedError = {});
const StorePath & storePath);
SourcePath getUnfilteredRootPath(CanonPath path);

677
src/libfetchers/git-utils.cc Normal file
View file

@ -0,0 +1,677 @@
#include "git-utils.hh"
#include "input-accessor.hh"
#include "cache.hh"
#include "finally.hh"
#include "processes.hh"
#include "signals.hh"
#include <boost/core/span.hpp>
#include <git2/blob.h>
#include <git2/commit.h>
#include <git2/config.h>
#include <git2/describe.h>
#include <git2/errors.h>
#include <git2/global.h>
#include <git2/object.h>
#include <git2/refs.h>
#include <git2/remote.h>
#include <git2/repository.h>
#include <git2/status.h>
#include <git2/submodule.h>
#include <git2/tree.h>
#include <unordered_set>
#include <queue>
#include <regex>
namespace std {
template<> struct hash<git_oid>
{
size_t operator()(const git_oid & oid) const
{
return * (size_t *) oid.id;
}
};
}
std::ostream & operator << (std::ostream & str, const git_oid & oid)
{
str << git_oid_tostr_s(&oid);
return str;
}
bool operator == (const git_oid & oid1, const git_oid & oid2)
{
return git_oid_equal(&oid1, &oid2);
}
namespace nix {
// Some wrapper types that ensure that the git_*_free functions get called.
template<auto del>
struct Deleter
{
template <typename T>
void operator()(T * p) const { del(p); };
};
typedef std::unique_ptr<git_repository, Deleter<git_repository_free>> Repository;
typedef std::unique_ptr<git_tree_entry, Deleter<git_tree_entry_free>> TreeEntry;
typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
typedef std::unique_ptr<git_treebuilder, Deleter<git_treebuilder_free>> TreeBuilder;
typedef std::unique_ptr<git_blob, Deleter<git_blob_free>> Blob;
typedef std::unique_ptr<git_object, Deleter<git_object_free>> Object;
typedef std::unique_ptr<git_commit, Deleter<git_commit_free>> Commit;
typedef std::unique_ptr<git_reference, Deleter<git_reference_free>> Reference;
typedef std::unique_ptr<git_describe_result, Deleter<git_describe_result_free>> DescribeResult;
typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusList;
typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
typedef std::unique_ptr<git_config_iterator, Deleter<git_config_iterator_free>> ConfigIterator;
// A helper to ensure that we don't leak objects returned by libgit2.
template<typename T>
struct Setter
{
T & t;
typename T::pointer p = nullptr;
Setter(T & t) : t(t) { }
~Setter() { if (p) t = T(p); }
operator typename T::pointer * () { return &p; }
};
Hash toHash(const git_oid & oid)
{
#ifdef GIT_EXPERIMENTAL_SHA256
assert(oid.type == GIT_OID_SHA1);
#endif
Hash hash(HashAlgorithm::SHA1);
memcpy(hash.hash, oid.id, hash.hashSize);
return hash;
}
static void initLibGit2()
{
if (git_libgit2_init() < 0)
throw Error("initialising libgit2: %s", git_error_last()->message);
}
git_oid hashToOID(const Hash & hash)
{
git_oid oid;
if (git_oid_fromstr(&oid, hash.gitRev().c_str()))
throw Error("cannot convert '%s' to a Git OID", hash.gitRev());
return oid;
}
Object lookupObject(git_repository * repo, const git_oid & oid)
{
Object obj;
if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) {
auto err = git_error_last();
throw Error("getting Git object '%s': %s", oid, err->message);
}
return obj;
}
template<typename T>
T peelObject(git_repository * repo, git_object * obj, git_object_t type)
{
T obj2;
if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) {
auto err = git_error_last();
throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message);
}
return obj2;
}
struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
{
CanonPath path;
Repository repo;
GitRepoImpl(CanonPath _path, bool create, bool bare)
: path(std::move(_path))
{
initLibGit2();
if (pathExists(path.abs())) {
if (git_repository_open(Setter(repo), path.c_str()))
throw Error("opening Git repository '%s': %s", path, git_error_last()->message);
} else {
if (git_repository_init(Setter(repo), path.c_str(), bare))
throw Error("creating Git repository '%s': %s", path, git_error_last()->message);
}
}
operator git_repository * ()
{
return repo.get();
}
uint64_t getRevCount(const Hash & rev) override
{
std::unordered_set<git_oid> done;
std::queue<Commit> todo;
todo.push(peelObject<Commit>(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT));
while (auto commit = pop(todo)) {
if (!done.insert(*git_commit_id(commit->get())).second) continue;
for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) {
git_commit * parent;
if (git_commit_parent(&parent, commit->get(), n))
throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message);
todo.push(Commit(parent));
}
}
return done.size();
}
uint64_t getLastModified(const Hash & rev) override
{
auto commit = peelObject<Commit>(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT);
return git_commit_time(commit.get());
}
bool isShallow() override
{
return git_repository_is_shallow(*this);
}
Hash resolveRef(std::string ref) override
{
// Handle revisions used as refs.
{
git_oid oid;
if (git_oid_fromstr(&oid, ref.c_str()) == 0)
return toHash(oid);
}
// Resolve short names like 'master'.
Reference ref2;
if (!git_reference_dwim(Setter(ref2), *this, ref.c_str()))
ref = git_reference_name(ref2.get());
// Resolve full references like 'refs/heads/master'.
Reference ref3;
if (git_reference_lookup(Setter(ref3), *this, ref.c_str()))
throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message);
auto oid = git_reference_target(ref3.get());
if (!oid)
throw Error("cannot get OID for Git reference '%s'", git_reference_name(ref3.get()));
return toHash(*oid);
}
std::vector<Submodule> parseSubmodules(const CanonPath & configFile)
{
GitConfig config;
if (git_config_open_ondisk(Setter(config), configFile.abs().c_str()))
throw Error("parsing .gitmodules file: %s", git_error_last()->message);
ConfigIterator it;
if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
throw Error("iterating over .gitmodules: %s", git_error_last()->message);
std::map<std::string, std::string> entries;
while (true) {
git_config_entry * entry = nullptr;
if (auto err = git_config_next(&entry, it.get())) {
if (err == GIT_ITEROVER) break;
throw Error("iterating over .gitmodules: %s", git_error_last()->message);
}
entries.emplace(entry->name + 10, entry->value);
}
std::vector<Submodule> result;
for (auto & [key, value] : entries) {
if (!hasSuffix(key, ".path")) continue;
std::string key2(key, 0, key.size() - 5);
auto path = CanonPath(value);
result.push_back(Submodule {
.path = path,
.url = entries[key2 + ".url"],
.branch = entries[key2 + ".branch"],
});
}
return result;
}
// Helper for statusCallback below.
static int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
{
return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
}
WorkdirInfo getWorkdirInfo() override
{
WorkdirInfo info;
/* Get the head revision, if any. */
git_oid headRev;
if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) {
if (err != GIT_ENOTFOUND)
throw Error("resolving HEAD: %s", git_error_last()->message);
} else
info.headRev = toHash(headRev);
/* Get all tracked files and determine whether the working
directory is dirty. */
std::function<int(const char * path, unsigned int statusFlags)> statusCallback = [&](const char * path, unsigned int statusFlags)
{
if (!(statusFlags & GIT_STATUS_INDEX_DELETED) &&
!(statusFlags & GIT_STATUS_WT_DELETED))
info.files.insert(CanonPath(path));
if (statusFlags != GIT_STATUS_CURRENT)
info.isDirty = true;
return 0;
};
git_status_options options = GIT_STATUS_OPTIONS_INIT;
options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
throw Error("getting working directory status: %s", git_error_last()->message);
/* Get submodule info. */
auto modulesFile = path + ".gitmodules";
if (pathExists(modulesFile.abs()))
info.submodules = parseSubmodules(modulesFile);
return info;
}
std::optional<std::string> getWorkdirRef() override
{
Reference ref;
if (git_reference_lookup(Setter(ref), *this, "HEAD"))
throw Error("looking up HEAD: %s", git_error_last()->message);
if (auto target = git_reference_symbolic_target(ref.get()))
return target;
return std::nullopt;
}
std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;
std::string resolveSubmoduleUrl(
const std::string & url,
const std::string & base) override
{
git_buf buf = GIT_BUF_INIT;
if (git_submodule_resolve_url(&buf, *this, url.c_str()))
throw Error("resolving Git submodule URL '%s'", url);
Finally cleanup = [&]() { git_buf_dispose(&buf); };
std::string res(buf.ptr);
if (!hasPrefix(res, "/") && res.find("://") == res.npos)
res = parseURL(base + "/" + res).canonicalise().to_string();
return res;
}
bool hasObject(const Hash & oid_) override
{
auto oid = hashToOID(oid_);
Object obj;
if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) {
if (errCode == GIT_ENOTFOUND) return false;
auto err = git_error_last();
throw Error("getting Git object '%s': %s", oid, err->message);
}
return true;
}
ref<InputAccessor> getAccessor(const Hash & rev) override;
static int sidebandProgressCallback(const char * str, int len, void * payload)
{
auto act = (Activity *) payload;
act->result(resFetchStatus, trim(std::string_view(str, len)));
return _isInterrupted ? -1 : 0;
}
static int transferProgressCallback(const git_indexer_progress * stats, void * payload)
{
auto act = (Activity *) payload;
act->result(resFetchStatus,
fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB",
stats->received_objects,
stats->total_objects,
stats->indexed_deltas,
stats->total_deltas,
stats->received_bytes / (1024.0 * 1024.0)));
return _isInterrupted ? -1 : 0;
}
void fetch(
const std::string & url,
const std::string & refspec,
bool shallow) override
{
Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url));
Remote remote;
if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message);
char * refspecs[] = {(char *) refspec.c_str()};
git_strarray refspecs2 {
.strings = refspecs,
.count = 1
};
git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
// FIXME: for some reason, shallow fetching over ssh barfs
// with "could not read from remote repository".
opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL;
opts.callbacks.payload = &act;
opts.callbacks.sideband_progress = sidebandProgressCallback;
opts.callbacks.transfer_progress = transferProgressCallback;
if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr))
throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
}
void verifyCommit(
const Hash & rev,
const std::vector<fetchers::PublicKey> & publicKeys) override
{
// Create ad-hoc allowedSignersFile and populate it with publicKeys
auto allowedSignersFile = createTempFile().second;
std::string allowedSigners;
for (const fetchers::PublicKey & k : publicKeys) {
if (k.type != "ssh-dsa"
&& k.type != "ssh-ecdsa"
&& k.type != "ssh-ecdsa-sk"
&& k.type != "ssh-ed25519"
&& k.type != "ssh-ed25519-sk"
&& k.type != "ssh-rsa")
throw Error("Unknown key type '%s'.\n"
"Please use one of\n"
"- ssh-dsa\n"
" ssh-ecdsa\n"
" ssh-ecdsa-sk\n"
" ssh-ed25519\n"
" ssh-ed25519-sk\n"
" ssh-rsa", k.type);
allowedSigners += "* " + k.type + " " + k.key + "\n";
}
writeFile(allowedSignersFile, allowedSigners);
// Run verification command
auto [status, output] = runProgram(RunOptions {
.program = "git",
.args = {
"-c",
"gpg.ssh.allowedSignersFile=" + allowedSignersFile,
"-C", path.abs(),
"verify-commit",
rev.gitRev()
},
.mergeStderrToStdout = true,
});
/* Evaluate result through status code and checking if public
key fingerprints appear on stderr. This is neccessary
because the git command might also succeed due to the
commit being signed by gpg keys that are present in the
users key agent. */
std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
for (const fetchers::PublicKey & k : publicKeys){
// Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
re += "(" + escaped_fingerprint + ")";
}
re += "]";
if (status == 0 && std::regex_search(output, std::regex(re)))
printTalkative("Signature verification on commit %s succeeded.", rev.gitRev());
else
throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
}
};
ref<GitRepo> GitRepo::openRepo(const CanonPath & path, bool create, bool bare)
{
return make_ref<GitRepoImpl>(path, create, bare);
}
struct GitInputAccessor : InputAccessor
{
ref<GitRepoImpl> repo;
Tree root;
GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
: repo(repo_)
, root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
{
}
std::string readBlob(const CanonPath & path, bool symlink)
{
auto blob = getBlob(path, symlink);
auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
return std::string(data);
}
std::string readFile(const CanonPath & path) override
{
return readBlob(path, false);
}
bool pathExists(const CanonPath & path) override
{
return path.isRoot() ? true : (bool) lookup(path);
}
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
if (path.isRoot())
return Stat { .type = tDirectory };
auto entry = lookup(path);
if (!entry)
return std::nullopt;
auto mode = git_tree_entry_filemode(entry);
if (mode == GIT_FILEMODE_TREE)
return Stat { .type = tDirectory };
else if (mode == GIT_FILEMODE_BLOB)
return Stat { .type = tRegular };
else if (mode == GIT_FILEMODE_BLOB_EXECUTABLE)
return Stat { .type = tRegular, .isExecutable = true };
else if (mode == GIT_FILEMODE_LINK)
return Stat { .type = tSymlink };
else if (mode == GIT_FILEMODE_COMMIT)
// Treat submodules as an empty directory.
return Stat { .type = tDirectory };
else
throw Error("file '%s' has an unsupported Git file type");
}
DirEntries readDirectory(const CanonPath & path) override
{
return std::visit(overloaded {
[&](Tree tree) {
DirEntries res;
auto count = git_tree_entrycount(tree.get());
for (size_t n = 0; n < count; ++n) {
auto entry = git_tree_entry_byindex(tree.get(), n);
// FIXME: add to cache
res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{});
}
return res;
},
[&](Submodule) {
return DirEntries();
}
}, getTree(path));
}
std::string readLink(const CanonPath & path) override
{
return readBlob(path, true);
}
Hash getSubmoduleRev(const CanonPath & path)
{
auto entry = need(path);
if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
throw Error("'%s' is not a submodule", showPath(path));
return toHash(*git_tree_entry_id(entry));
}
std::unordered_map<CanonPath, TreeEntry> lookupCache;
/* Recursively look up 'path' relative to the root. */
git_tree_entry * lookup(const CanonPath & path)
{
if (path.isRoot()) return nullptr;
auto i = lookupCache.find(path);
if (i == lookupCache.end()) {
TreeEntry entry;
if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) {
if (err != GIT_ENOTFOUND)
throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
}
i = lookupCache.emplace(path, std::move(entry)).first;
}
return &*i->second;
}
git_tree_entry * need(const CanonPath & path)
{
auto entry = lookup(path);
if (!entry)
throw Error("'%s' does not exist", showPath(path));
return entry;
}
struct Submodule { };
std::variant<Tree, Submodule> getTree(const CanonPath & path)
{
if (path.isRoot()) {
Tree tree;
if (git_tree_dup(Setter(tree), root.get()))
throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message);
return tree;
}
auto entry = need(path);
if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT)
return Submodule();
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
throw Error("'%s' is not a directory", showPath(path));
Tree tree;
if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry))
throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
return tree;
}
Blob getBlob(const CanonPath & path, bool expectSymlink)
{
auto notExpected = [&]()
{
throw Error(
expectSymlink
? "'%s' is not a symlink"
: "'%s' is not a regular file",
showPath(path));
};
if (path.isRoot()) notExpected();
auto entry = need(path);
if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB)
notExpected();
auto mode = git_tree_entry_filemode(entry);
if (expectSymlink) {
if (mode != GIT_FILEMODE_LINK)
notExpected();
} else {
if (mode != GIT_FILEMODE_BLOB && mode != GIT_FILEMODE_BLOB_EXECUTABLE)
notExpected();
}
Blob blob;
if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry))
throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message);
return blob;
}
};
ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
{
return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
}
std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev)
{
/* Read the .gitmodules files from this revision. */
CanonPath modulesFile(".gitmodules");
auto accessor = getAccessor(rev);
if (!accessor->pathExists(modulesFile)) return {};
/* Parse it and get the revision of each submodule. */
auto configS = accessor->readFile(modulesFile);
auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
writeFull(fdTemp.get(), configS);
std::vector<std::tuple<Submodule, Hash>> result;
for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) {
auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(submodule.path);
result.push_back({std::move(submodule), rev});
}
return result;
}
}

90
src/libfetchers/git-utils.hh Normal file
View file

@ -0,0 +1,90 @@
#pragma once
#include "input-accessor.hh"
namespace nix {
namespace fetchers { struct PublicKey; }
struct GitRepo
{
virtual ~GitRepo()
{ }
static ref<GitRepo> openRepo(const CanonPath & path, bool create = false, bool bare = false);
virtual uint64_t getRevCount(const Hash & rev) = 0;
virtual uint64_t getLastModified(const Hash & rev) = 0;
virtual bool isShallow() = 0;
/* Return the commit hash to which a ref points. */
virtual Hash resolveRef(std::string ref) = 0;
/**
* Info about a submodule.
*/
struct Submodule
{
CanonPath path;
std::string url;
std::string branch;
};
struct WorkdirInfo
{
bool isDirty = false;
/* The checked out commit, or nullopt if there are no commits
in the repo yet. */
std::optional<Hash> headRev;
/* All files in the working directory that are unchanged,
modified or added, but excluding deleted files. */
std::set<CanonPath> files;
/* The submodules listed in .gitmodules of this workdir. */
std::vector<Submodule> submodules;
};
virtual WorkdirInfo getWorkdirInfo() = 0;
/* Get the ref that HEAD points to. */
virtual std::optional<std::string> getWorkdirRef() = 0;
/**
* Return the submodules of this repo at the indicated revision,
* along with the revision of each submodule.
*/
virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;
virtual std::string resolveSubmoduleUrl(
const std::string & url,
const std::string & base) = 0;
struct TarballInfo
{
Hash treeHash;
time_t lastModified;
};
virtual bool hasObject(const Hash & oid) = 0;
virtual ref<InputAccessor> getAccessor(const Hash & rev) = 0;
virtual void fetch(
const std::string & url,
const std::string & refspec,
bool shallow) = 0;
/**
* Verify that commit `rev` is signed by one of the keys in
* `publicKeys`. Throw an error if it isn't.
*/
virtual void verifyCommit(
const Hash & rev,
const std::vector<fetchers::PublicKey> & publicKeys) = 0;
};
}

831
src/libfetchers/git.cc
View file

@ -8,6 +8,12 @@
#include "pathlocks.hh"
#include "processes.hh"
#include "git.hh"
#include "fs-input-accessor.hh"
#include "filtering-input-accessor.hh"
#include "mounted-input-accessor.hh"
#include "git-utils.hh"
#include "logging.hh"
#include "finally.hh"
#include "fetch-settings.hh"
@ -47,7 +53,7 @@ bool touchCacheFile(const Path & path, time_t touch_time)
Path getCachePath(std::string_view key)
{
return getCacheDir() + "/nix/gitv3/" +
hashString(htSHA256, key).to_string(HashFormat::Base32, false);
hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false);
}
// Returns the name of the HEAD branch.
@ -133,189 +139,19 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
return std::nullopt;
}
bool isNotDotGitDirectory(const Path & path)
std::vector<PublicKey> getPublicKeys(const Attrs & attrs)
{
return baseNameOf(path) != ".git";
}
struct WorkdirInfo
{
bool clean = false;
bool hasHead = false;
};
std::vector<PublicKey> getPublicKeys(const Attrs & attrs) {
std::vector<PublicKey> publicKeys;
if (attrs.contains("publicKeys")) {
nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys"));
ensureType(publicKeysJson, nlohmann::json::value_t::array);
publicKeys = publicKeysJson.get<std::vector<PublicKey>>();
}
else {
publicKeys = {};
}
if (attrs.contains("publicKey"))
publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")});
return publicKeys;
}
void doCommitVerification(const Path repoDir, const Path gitDir, const std::string rev, const std::vector<PublicKey>& publicKeys) {
// Create ad-hoc allowedSignersFile and populate it with publicKeys
auto allowedSignersFile = createTempFile().second;
std::string allowedSigners;
for (const PublicKey& k : publicKeys) {
if (k.type != "ssh-dsa"
&& k.type != "ssh-ecdsa"
&& k.type != "ssh-ecdsa-sk"
&& k.type != "ssh-ed25519"
&& k.type != "ssh-ed25519-sk"
&& k.type != "ssh-rsa")
warn("Unknown keytype: %s\n"
"Please use one of\n"
"- ssh-dsa\n"
" ssh-ecdsa\n"
" ssh-ecdsa-sk\n"
" ssh-ed25519\n"
" ssh-ed25519-sk\n"
" ssh-rsa", k.type);
allowedSigners += "* " + k.type + " " + k.key + "\n";
}
writeFile(allowedSignersFile, allowedSigners);
// Run verification command
auto [status, output] = runProgram(RunOptions {
.program = "git",
.args = {"-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, "-C", repoDir,
"--git-dir", gitDir, "verify-commit", rev},
.mergeStderrToStdout = true,
});
/* Evaluate result through status code and checking if public key fingerprints appear on stderr
* This is neccessary because the git command might also succeed due to the commit being signed by gpg keys
* that are present in the users key agent. */
std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
for (const PublicKey& k : publicKeys){
// Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
re += "(" + escaped_fingerprint + ")";
}
re += "]";
if (status == 0 && std::regex_search(output, std::regex(re)))
printTalkative("Signature verification on commit %s succeeded", rev);
else
throw Error("Commit signature verification on commit %s failed: \n%s", rev, output);
}
// Returns whether a git workdir is clean and has commits.
WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
{
const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
std::string gitDir(".git");
auto env = getEnv();
// Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
// that way unknown errors can lead to a failure instead of continuing through the wrong code path
env["LC_ALL"] = "C";
/* Check whether HEAD points to something that looks like a commit,
since that is the refrence we want to use later on. */
auto result = runProgram(RunOptions {
.program = "git",
.args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
.environment = env,
.mergeStderrToStdout = true
});
auto exitCode = WEXITSTATUS(result.first);
auto errorMessage = result.second;
if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
throw Error("'%s' is not a Git repository", workdir);
} else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
// indicates that the repo does not have any commits
// we want to proceed and will consider it dirty later
} else if (exitCode != 0) {
// any other errors should lead to a failure
throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage);
}
bool clean = false;
bool hasHead = exitCode == 0;
try {
if (hasHead) {
// Using git diff is preferrable over lower-level operations here,
// because its conceptually simpler and we only need the exit code anyways.
auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
if (!submodules) {
// Changes in submodules should only make the tree dirty
// when those submodules will be copied as well.
gitDiffOpts.emplace_back("--ignore-submodules");
}
gitDiffOpts.emplace_back("--");
runProgram("git", true, gitDiffOpts);
clean = true;
}
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
}
return WorkdirInfo { .clean = clean, .hasHead = hasHead };
}
std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo)
{
const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
auto gitDir = ".git";
if (!fetchSettings.allowDirty)
throw Error("Git tree '%s' is dirty", workdir);
if (fetchSettings.warnDirty)
warn("Git tree '%s' is dirty", workdir);
auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" });
if (submodules)
gitOpts.emplace_back("--recurse-submodules");
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, gitOpts), "\0"s);
Path actualPath(absPath(workdir));
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualPath));
std::string file(p, actualPath.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
if (workdirInfo.hasHead) {
input.attrs.insert_or_assign("dirtyRev", chomp(
runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty");
input.attrs.insert_or_assign("dirtyShortRev", chomp(
runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty");
}
return {std::move(storePath), input};
}
} // end namespace
struct GitInputScheme : InputScheme
@ -386,9 +222,6 @@ struct GitInputScheme : InputScheme
|| name == "publicKeys")
experimentalFeatureSettings.require(Xp::VerifiedFetches);
maybeGetBoolAttr(attrs, "shallow");
maybeGetBoolAttr(attrs, "submodules");
maybeGetBoolAttr(attrs, "allRefs");
maybeGetBoolAttr(attrs, "verifyCommit");
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
@ -401,6 +234,9 @@ struct GitInputScheme : InputScheme
auto url = fixGitURL(getStrAttr(attrs, "url"));
parseURL(url);
input.attrs["url"] = url;
getShallowAttr(input);
getSubmodulesAttr(input);
getAllRefsAttr(input);
return input;
}
@ -410,8 +246,10 @@ struct GitInputScheme : InputScheme
if (url.scheme != "git") url.scheme = "git+" + url.scheme;
if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
if (getShallowAttr(input))
url.query.insert_or_assign("shallow", "1");
if (getSubmodulesAttr(input))
url.query.insert_or_assign("submodules", "1");
if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
url.query.insert_or_assign("verifyCommit", "1");
auto publicKeys = getPublicKeys(input.attrs);
@ -439,11 +277,11 @@ struct GitInputScheme : InputScheme
void clone(const Input & input, const Path & destDir) const override
{
auto [isLocal, actualUrl] = getActualUrl(input);
auto repoInfo = getRepoInfo(input);
Strings args = {"clone"};
args.push_back(actualUrl);
args.push_back(repoInfo.url);
if (auto ref = input.getRef()) {
args.push_back("--branch");
@ -459,10 +297,9 @@ struct GitInputScheme : InputScheme
std::optional<Path> getSourcePath(const Input & input) const override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
return url.path;
return {};
auto repoInfo = getRepoInfo(input);
if (repoInfo.isLocal) return repoInfo.url;
return std::nullopt;
}
void putFile(
@ -471,24 +308,88 @@ struct GitInputScheme : InputScheme
std::string_view contents,
std::optional<std::string> commitMsg) const override
{
auto root = getSourcePath(input);
if (!root)
auto repoInfo = getRepoInfo(input);
if (!repoInfo.isLocal)
throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
writeFile((CanonPath(*root) + path).abs(), contents);
writeFile((CanonPath(repoInfo.url) + path).abs(), contents);
auto gitDir = ".git";
auto result = runProgram(RunOptions {
.program = "git",
.args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
});
auto exitCode = WEXITSTATUS(result.first);
runProgram("git", true,
{ "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
if (commitMsg)
if (exitCode != 0) {
// The path is not `.gitignore`d, we can add the file.
runProgram("git", true,
{ "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
{ "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
if (commitMsg) {
// Pause the logger to allow for user input (such as a gpg passphrase) in `git commit`
logger->pause();
Finally restoreLogger([]() { logger->resume(); });
runProgram("git", true,
{ "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
}
}
}
std::pair<bool, std::string> getActualUrl(const Input & input) const
struct RepoInfo
{
/* Whether this is a local, non-bare repository. */
bool isLocal = false;
/* Working directory info: the complete list of files, and
whether the working directory is dirty compared to HEAD. */
GitRepo::WorkdirInfo workdirInfo;
/* URL of the repo, or its path if isLocal. Never a `file` URL. */
std::string url;
void warnDirty() const
{
if (workdirInfo.isDirty) {
if (!fetchSettings.allowDirty)
throw Error("Git tree '%s' is dirty", url);
if (fetchSettings.warnDirty)
warn("Git tree '%s' is dirty", url);
}
}
std::string gitDir = ".git";
};
bool getShallowAttr(const Input & input) const
{
return maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
}
bool getSubmodulesAttr(const Input & input) const
{
return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
}
bool getAllRefsAttr(const Input & input) const
{
return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
}
RepoInfo getRepoInfo(const Input & input) const
{
auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
{
if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256))
throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
};
if (auto rev = input.getRev())
checkHashAlgorithm(rev);
RepoInfo repoInfo;
// file:// URIs are normally not cloned (but otherwise treated the
// same as remote URIs, i.e. we don't use the working tree or
// HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
@ -496,162 +397,132 @@ struct GitInputScheme : InputScheme
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
return {isLocal, isLocal ? url.path : url.base};
repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
repoInfo.url = repoInfo.isLocal ? url.path : url.base;
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo();
return repoInfo;
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
{
Input input(_input);
auto gitDir = ".git";
Attrs key{{"_what", "gitLastModified"}, {"rev", rev.gitRev()}};
auto cache = getCache();
if (auto res = cache->lookup(key))
return getIntAttr(*res, "lastModified");
auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev);
cache->upsert(key, Attrs{{"lastModified", lastModified}});
return lastModified;
}
uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
{
Attrs key{{"_what", "gitRevCount"}, {"rev", rev.gitRev()}};
auto cache = getCache();
if (auto revCountAttrs = cache->lookup(key))
return getIntAttr(*revCountAttrs, "revCount");
Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url));
auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev);
cache->upsert(key, Attrs{{"revCount", revCount}});
return revCount;
}
std::string getDefaultRef(const RepoInfo & repoInfo) const
{
auto head = repoInfo.isLocal
? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef()
: readHeadCached(repoInfo.url);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url);
return "master";
}
return *head;
}
static MakeNotAllowedError makeNotAllowedError(std::string url)
{
return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError
{
if (nix::pathExists(path.abs()))
return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url);
else
return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url);
};
}
void verifyCommit(const Input & input, std::shared_ptr<GitRepo> repo) const
{
auto publicKeys = getPublicKeys(input.attrs);
auto verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty());
if (verifyCommit) {
if (input.getRev() && repo)
repo->verifyCommit(*input.getRev(), publicKeys);
else
throw Error("commit verification is required for Git repository '%s', but it's dirty", input.to_string());
}
}
std::pair<ref<InputAccessor>, Input> getAccessorFromCommit(
ref<Store> store,
RepoInfo & repoInfo,
Input && input) const
{
assert(!repoInfo.workdirInfo.isDirty);
auto origRev = input.getRev();
std::string name = input.getName();
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
std::vector<PublicKey> publicKeys = getPublicKeys(input.attrs);
bool verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty());
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
if (submodules) cacheType += "-submodules";
if (allRefs) cacheType += "-all-refs";
auto checkHashType = [&](const std::optional<Hash> & hash)
{
if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
};
auto getLockedAttrs = [&]()
{
checkHashType(input.getRev());
return Attrs({
{"type", cacheType},
{"name", name},
{"rev", input.getRev()->gitRev()},
{"verifyCommit", verifyCommit},
{"publicKeys", publicKeys_to_string(publicKeys)},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<StorePath, Input>
{
assert(input.getRev());
assert(!_input.getRev() || _input.getRev() == input.getRev());
if (!shallow)
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
return {std::move(storePath), input};
};
if (input.getRev()) {
if (auto res = getCache()->lookup(store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second));
}
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
/* If this is a local directory, no ref or revision is given and no signature verification is needed,
allow fetching directly from a dirty workdir. */
if (!input.getRef() && !input.getRev() && isLocal) {
auto workdirInfo = getWorkdirInfo(input, actualUrl);
if (!workdirInfo.clean) {
if (verifyCommit)
throw Error("Can't fetch from a dirty workdir with commit signature verification enabled.");
else
return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
}
}
Attrs unlockedAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
{"verifyCommit", verifyCommit},
{"publicKeys", publicKeys_to_string(publicKeys)},
});
auto originalRef = input.getRef();
auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
input.attrs.insert_or_assign("ref", ref);
Path repoDir;
if (isLocal) {
if (!input.getRef()) {
auto head = readHead(actualUrl);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
head = "master";
}
input.attrs.insert_or_assign("ref", *head);
unlockedAttrs.insert_or_assign("ref", *head);
}
if (repoInfo.isLocal) {
repoDir = repoInfo.url;
if (!input.getRev())
input.attrs.insert_or_assign("rev",
Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
} else {
const bool useHeadRef = !input.getRef();
if (useHeadRef) {
auto head = readHeadCached(actualUrl);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
head = "master";
}
input.attrs.insert_or_assign("ref", *head);
unlockedAttrs.insert_or_assign("ref", *head);
} else {
if (!input.getRev()) {
unlockedAttrs.insert_or_assign("ref", input.getRef().value());
}
}
if (auto res = getCache()->lookup(store, unlockedAttrs)) {
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
Path cacheDir = getCachePath(actualUrl);
Path cacheDir = getCachePath(repoInfo.url);
repoDir = cacheDir;
gitDir = ".";
repoInfo.gitDir = ".";
createDirs(dirOf(cacheDir));
PathLocks cacheDirLock({cacheDir + ".lock"});
PathLocks cacheDirLock({cacheDir});
if (!pathExists(cacheDir)) {
runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
}
auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true);
Path localRefFile =
input.getRef()->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input.getRef()
: cacheDir + "/refs/heads/" + *input.getRef();
ref.compare(0, 5, "refs/") == 0
? cacheDir + "/" + ref
: cacheDir + "/refs/heads/" + ref;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (input.getRev()) {
try {
runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
if (auto rev = input.getRev()) {
doFetch = !repo->hasObject(*rev);
} else {
if (allRefs) {
if (getAllRefsAttr(input)) {
doFetch = true;
} else {
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
@ -663,163 +534,213 @@ struct GitInputScheme : InputScheme
}
if (doFetch) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
try {
auto ref = input.getRef();
auto fetchRef = allRefs
auto fetchRef =
getAllRefsAttr(input)
? "refs/*"
: ref->compare(0, 5, "refs/") == 0
? *ref
: ref == "HEAD"
? *ref
: "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
: input.getRev()
? input.getRev()->gitRev()
: ref.compare(0, 5, "refs/") == 0
? ref
: ref == "HEAD"
? ref
: "refs/heads/" + ref;
repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input));
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
logError(e.info());
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url);
}
if (!touchCacheFile(localRefFile, now))
warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno));
if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef()))
warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl);
if (!originalRef && !storeCachedHead(repoInfo.url, ref))
warn("could not update cached head '%s' for '%s'", ref, repoInfo.url);
}
if (!input.getRev())
input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
if (auto rev = input.getRev()) {
if (!repo->hasObject(*rev))
throw Error(
"Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
"Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
"allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
rev->gitRev(),
ref,
repoInfo.url
);
} else
input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), HashAlgorithm::SHA1).gitRev());
// cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true";
auto repo = GitRepo::openRepo(CanonPath(repoDir));
if (isShallow && !shallow)
throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl);
auto isShallow = repo->isShallow();
// FIXME: check whether rev is an ancestor of ref.
if (isShallow && !getShallowAttr(input))
throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url);
printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
// FIXME: check whether rev is an ancestor of ref?
/* Now that we know the ref, check again whether we have it in
the store. */
if (auto res = getCache()->lookup(store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;
auto result = runProgram(RunOptions {
.program = "git",
.args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
.mergeStderrToStdout = true
});
if (WEXITSTATUS(result.first) == 128
&& result.second.find("bad file") != std::string::npos)
{
throw Error(
"Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
"Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
"allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
input.getRev()->gitRev(),
*input.getRef(),
actualUrl
);
}
if (verifyCommit)
doCommitVerification(repoDir, gitDir, input.getRev()->gitRev(), publicKeys);
if (submodules) {
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);
runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
{
// TODO: repoDir might lack the ref (it only checks if rev
// exists, see FIXME above) so use a big hammer and fetch
// everything to ensure we get the rev.
Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
}
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
/* Ensure that we use the correct origin for fetching
submodules. This matters for submodules with relative
URLs. */
if (isLocal) {
writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config"));
/* Restore the config.bare setting we may have just
copied erroneously from the user's repo. */
runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" });
} else
runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl });
/* As an optimisation, copy the modules directory of the
source repo if it exists. */
auto modulesPath = repoDir + "/" + gitDir + "/modules";
if (pathExists(modulesPath)) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl));
runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" });
}
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
}
filter = isNotDotGitDirectory;
} else {
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
runProgram2({
.program = "git",
.args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
.standardOut = &sink
});
});
unpackTarfile(*source, tmpDir);
}
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
auto rev = *input.getRev();
Attrs infoAttrs({
{"rev", input.getRev()->gitRev()},
{"lastModified", lastModified},
{"rev", rev.gitRev()},
{"lastModified", getLastModified(repoInfo, repoDir, rev)},
});
if (!shallow)
if (!getShallowAttr(input))
infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() })));
getRevCount(repoInfo, repoDir, rev));
if (!_input.getRev())
getCache()->add(
store,
unlockedAttrs,
infoAttrs,
storePath,
false);
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url);
getCache()->add(
store,
getLockedAttrs(),
infoAttrs,
storePath,
true);
verifyCommit(input, repo);
return makeResult(infoAttrs, std::move(storePath));
auto accessor = repo->getAccessor(rev);
accessor->setPathDisplay("«" + input.to_string() + "»");
/* If the repo has submodules, fetch them and return a mounted
input accessor consisting of the accessor for the top-level
repo and the accessors for the submodules. */
if (getSubmodulesAttr(input)) {
std::map<CanonPath, nix::ref<InputAccessor>> mounts;
for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url);
debug("Git submodule %s: %s %s %s -> %s",
submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", resolved);
if (submodule.branch != "")
attrs.insert_or_assign("ref", submodule.branch);
attrs.insert_or_assign("rev", submoduleRev.gitRev());
auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
auto [submoduleAccessor, submoduleInput2] =
submoduleInput.getAccessor(store);
mounts.insert_or_assign(submodule.path, submoduleAccessor);
}
if (!mounts.empty()) {
mounts.insert_or_assign(CanonPath::root, accessor);
accessor = makeMountedInputAccessor(std::move(mounts));
}
}
assert(!origRev || origRev == rev);
if (!getShallowAttr(input))
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
return {accessor, std::move(input)};
}
std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
ref<Store> store,
RepoInfo & repoInfo,
Input && input) const
{
if (getSubmodulesAttr(input))
/* Create mountpoints for the submodules. */
for (auto & submodule : repoInfo.workdirInfo.submodules)
repoInfo.workdirInfo.files.insert(submodule.path);
ref<InputAccessor> accessor =
AllowListInputAccessor::create(
makeFSInputAccessor(CanonPath(repoInfo.url)),
std::move(repoInfo.workdirInfo.files),
makeNotAllowedError(repoInfo.url));
/* If the repo has submodules, return a mounted input accessor
consisting of the accessor for the top-level repo and the
accessors for the submodule workdirs. */
if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) {
std::map<CanonPath, nix::ref<InputAccessor>> mounts;
for (auto & submodule : repoInfo.workdirInfo.submodules) {
auto submodulePath = CanonPath(repoInfo.url) + submodule.path;
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", submodulePath.abs());
auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
auto [submoduleAccessor, submoduleInput2] =
submoduleInput.getAccessor(store);
/* If the submodule is dirty, mark this repo dirty as
well. */
if (!submoduleInput2.getRev())
repoInfo.workdirInfo.isDirty = true;
mounts.insert_or_assign(submodule.path, submoduleAccessor);
}
mounts.insert_or_assign(CanonPath::root, accessor);
accessor = makeMountedInputAccessor(std::move(mounts));
}
if (!repoInfo.workdirInfo.isDirty) {
auto repo = GitRepo::openRepo(CanonPath(repoInfo.url));
if (auto ref = repo->getWorkdirRef())
input.attrs.insert_or_assign("ref", *ref);
auto rev = repoInfo.workdirInfo.headRev.value();
input.attrs.insert_or_assign("rev", rev.gitRev());
input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev));
verifyCommit(input, repo);
} else {
repoInfo.warnDirty();
if (repoInfo.workdirInfo.headRev) {
input.attrs.insert_or_assign("dirtyRev",
repoInfo.workdirInfo.headRev->gitRev() + "-dirty");
input.attrs.insert_or_assign("dirtyShortRev",
repoInfo.workdirInfo.headRev->gitShortRev() + "-dirty");
}
verifyCommit(input, nullptr);
}
input.attrs.insert_or_assign(
"lastModified",
repoInfo.workdirInfo.headRev
? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev)
: 0);
input.locked = true; // FIXME
return {accessor, std::move(input)};
}
std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
{
Input input(_input);
auto repoInfo = getRepoInfo(input);
auto [accessor, final] =
input.getRef() || input.getRev() || !repoInfo.isLocal
? getAccessorFromCommit(store, repoInfo, std::move(input))
: getAccessorFromWorkdir(store, repoInfo, std::move(input));
accessor->fingerprint = final.getFingerprint(store);
return {accessor, std::move(final)};
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
else
return std::nullopt;
}
};

22
src/libfetchers/github.cc
View file

@ -42,7 +42,7 @@ struct GitArchiveInputScheme : InputScheme
auto size = path.size();
if (size == 3) {
if (std::regex_match(path[2], revRegex))
rev = Hash::parseAny(path[2], htSHA1);
rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
@ -68,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme
if (name == "rev") {
if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
rev = Hash::parseAny(value, htSHA1);
rev = Hash::parseAny(value, HashAlgorithm::SHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
@ -201,7 +201,7 @@ struct GitArchiveInputScheme : InputScheme
{"rev", rev->gitRev()},
});
if (auto res = getCache()->lookup(store, lockedAttrs)) {
if (auto res = getCache()->lookup(*store, lockedAttrs)) {
input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
return {std::move(res->second), input};
}
@ -213,7 +213,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
getCache()->add(
store,
*store,
lockedAttrs,
{
{"rev", rev->gitRev()},
@ -229,6 +229,14 @@ struct GitArchiveInputScheme : InputScheme
{
return Xp::Flakes;
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev();
else
return std::nullopt;
}
};
struct GitHubInputScheme : GitArchiveInputScheme
@ -276,7 +284,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
auto rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
@ -348,7 +356,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
auto rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
@ -440,7 +448,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
if(!id)
throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
auto rev = Hash::parseAny(*id, htSHA1);
auto rev = Hash::parseAny(*id, HashAlgorithm::SHA1);
debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
return rev;
}

4
src/libfetchers/indirect.cc
View file

@ -20,7 +20,7 @@ struct IndirectInputScheme : InputScheme
if (path.size() == 1) {
} else if (path.size() == 2) {
if (std::regex_match(path[1], revRegex))
rev = Hash::parseAny(path[1], htSHA1);
rev = Hash::parseAny(path[1], HashAlgorithm::SHA1);
else if (std::regex_match(path[1], refRegex))
ref = path[1];
else
@ -31,7 +31,7 @@ struct IndirectInputScheme : InputScheme
ref = path[1];
if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
rev = Hash::parseAny(path[2], htSHA1);
rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);

98
src/libfetchers/input-accessor.cc
View file

@ -1,98 +0,0 @@
#include "input-accessor.hh"
#include "store-api.hh"
namespace nix {
StorePath InputAccessor::fetchToStore(
ref<Store> store,
const CanonPath & path,
std::string_view name,
FileIngestionMethod method,
PathFilter * filter,
RepairFlag repair)
{
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive)
dumpPath(path, sink, filter ? *filter : defaultPathFilter);
else
readFile(path, sink);
});
auto storePath =
settings.readOnlyMode
? store->computeStorePathFromDump(*source, name, method, htSHA256).first
: store->addToStoreFromDump(*source, name, method, htSHA256, repair);
return storePath;
}
SourcePath InputAccessor::root()
{
return {ref(shared_from_this()), CanonPath::root};
}
std::ostream & operator << (std::ostream & str, const SourcePath & path)
{
str << path.to_string();
return str;
}
StorePath SourcePath::fetchToStore(
ref<Store> store,
std::string_view name,
FileIngestionMethod method,
PathFilter * filter,
RepairFlag repair) const
{
return accessor->fetchToStore(store, path, name, method, filter, repair);
}
std::string_view SourcePath::baseName() const
{
return path.baseName().value_or("source");
}
SourcePath SourcePath::parent() const
{
auto p = path.parent();
assert(p);
return {accessor, std::move(*p)};
}
SourcePath SourcePath::resolveSymlinks() const
{
auto res = accessor->root();
int linksAllowed = 1024;
std::list<std::string> todo;
for (auto & c : path)
todo.push_back(std::string(c));
while (!todo.empty()) {
auto c = *todo.begin();
todo.pop_front();
if (c == "" || c == ".")
;
else if (c == "..")
res.path.pop();
else {
res.path.push(c);
if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) {
if (!linksAllowed--)
throw Error("infinite symlink recursion in path '%s'", path);
auto target = res.readLink();
res.path.pop();
if (hasPrefix(target, "/"))
res.path = CanonPath::root;
todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
}
}
}
return res;
}
}

169
src/libfetchers/input-accessor.hh
View file

@ -1,169 +0,0 @@
#pragma once
///@file
#include "source-accessor.hh"
#include "ref.hh"
#include "types.hh"
#include "file-system.hh"
#include "repair-flag.hh"
#include "content-address.hh"
namespace nix {
MakeError(RestrictedPathError, Error);
struct SourcePath;
class StorePath;
class Store;
struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
{
/**
* Return the maximum last-modified time of the files in this
* tree, if available.
*/
virtual std::optional<time_t> getLastModified()
{
return std::nullopt;
}
StorePath fetchToStore(
ref<Store> store,
const CanonPath & path,
std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
SourcePath root();
};
/**
* An abstraction for accessing source files during
* evaluation. Currently, it's just a wrapper around `CanonPath` that
* accesses files in the regular filesystem, but in the future it will
* support fetching files in other ways.
*/
struct SourcePath
{
ref<InputAccessor> accessor;
CanonPath path;
std::string_view baseName() const;
/**
* Construct the parent of this `SourcePath`. Aborts if `this`
* denotes the root.
*/
SourcePath parent() const;
/**
* If this `SourcePath` denotes a regular file (not a symlink),
* return its contents; otherwise throw an error.
*/
std::string readFile() const
{ return accessor->readFile(path); }
/**
* Return whether this `SourcePath` denotes a file (of any type)
* that exists
*/
bool pathExists() const
{ return accessor->pathExists(path); }
/**
* Return stats about this `SourcePath`, or throw an exception if
* it doesn't exist.
*/
InputAccessor::Stat lstat() const
{ return accessor->lstat(path); }
/**
* Return stats about this `SourcePath`, or std::nullopt if it
* doesn't exist.
*/
std::optional<InputAccessor::Stat> maybeLstat() const
{ return accessor->maybeLstat(path); }
/**
* If this `SourcePath` denotes a directory (not a symlink),
* return its directory entries; otherwise throw an error.
*/
InputAccessor::DirEntries readDirectory() const
{ return accessor->readDirectory(path); }
/**
* If this `SourcePath` denotes a symlink, return its target;
* otherwise throw an error.
*/
std::string readLink() const
{ return accessor->readLink(path); }
/**
* Dump this `SourcePath` to `sink` as a NAR archive.
*/
void dumpPath(
Sink & sink,
PathFilter & filter = defaultPathFilter) const
{ return accessor->dumpPath(path, sink, filter); }
/**
* Copy this `SourcePath` to the Nix store.
*/
StorePath fetchToStore(
ref<Store> store,
std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair) const;
/**
* Return the location of this path in the "real" filesystem, if
* it has a physical location.
*/
std::optional<CanonPath> getPhysicalPath() const
{ return accessor->getPhysicalPath(path); }
std::string to_string() const
{ return path.abs(); }
/**
* Append a `CanonPath` to this path.
*/
SourcePath operator + (const CanonPath & x) const
{ return {accessor, path + x}; }
/**
* Append a single component `c` to this path. `c` must not
* contain a slash. A slash is implicitly added between this path
* and `c`.
*/
SourcePath operator + (std::string_view c) const
{ return {accessor, path + c}; }
bool operator == (const SourcePath & x) const
{
return std::tie(accessor, path) == std::tie(x.accessor, x.path);
}
bool operator != (const SourcePath & x) const
{
return std::tie(accessor, path) != std::tie(x.accessor, x.path);
}
bool operator < (const SourcePath & x) const
{
return std::tie(accessor, path) < std::tie(x.accessor, x.path);
}
/**
* Resolve any symlinks in this `SourcePath` (including its
* parents). The result is a `SourcePath` in which no element is a
* symlink.
*/
SourcePath resolveSymlinks() const;
};
std::ostream & operator << (std::ostream & str, const SourcePath & path);
}

2
src/libfetchers/local.mk
View file

@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
libfetchers_LDFLAGS += -pthread
libfetchers_LDFLAGS += $(THREAD_LDFLAGS) $(LIBGIT2_LIBS) -larchive
libfetchers_LIBS = libutil libstore

1
src/libfetchers/memory-input-accessor.cc
View file

@ -1,5 +1,6 @@
#include "memory-input-accessor.hh"
#include "memory-source-accessor.hh"
#include "source-path.hh"
namespace nix {

1
src/libfetchers/memory-input-accessor.hh
View file

@ -1,4 +1,5 @@
#include "input-accessor.hh"
#include "source-path.hh"
namespace nix {

41
src/libfetchers/mercurial.cc
View file

@ -6,6 +6,7 @@
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include "posix-source-accessor.hh"
#include "fetch-settings.hh"
@ -210,7 +211,12 @@ struct MercurialInputScheme : InputScheme
return files.count(file);
};
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
PosixSourceAccessor accessor;
auto storePath = store->addToStore(
input.getName(),
accessor, CanonPath { actualPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {},
filter);
return {std::move(storePath), input};
}
@ -218,16 +224,16 @@ struct MercurialInputScheme : InputScheme
if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
auto checkHashType = [&](const std::optional<Hash> & hash)
auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
{
if (hash.has_value() && hash->type != htSHA1)
if (hash.has_value() && hash->algo != HashAlgorithm::SHA1)
throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
};
auto getLockedAttrs = [&]()
{
checkHashType(input.getRev());
checkHashAlgorithm(input.getRev());
return Attrs({
{"type", "hg"},
@ -246,7 +252,7 @@ struct MercurialInputScheme : InputScheme
};
if (input.getRev()) {
if (auto res = getCache()->lookup(store, getLockedAttrs()))
if (auto res = getCache()->lookup(*store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second));
}
@ -259,15 +265,15 @@ struct MercurialInputScheme : InputScheme
{"ref", *input.getRef()},
});
if (auto res = getCache()->lookup(store, unlockedAttrs)) {
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (auto res = getCache()->lookup(*store, unlockedAttrs)) {
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false));
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false));
/* If this is a commit hash that we already have, we don't
have to pull again. */
@ -301,11 +307,11 @@ struct MercurialInputScheme : InputScheme
runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashAlgorithm::SHA1).gitRev());
auto revCount = std::stoull(tokens[1]);
input.attrs.insert_or_assign("ref", tokens[2]);
if (auto res = getCache()->lookup(store, getLockedAttrs()))
if (auto res = getCache()->lookup(*store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir();
@ -315,7 +321,8 @@ struct MercurialInputScheme : InputScheme
deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store->addToStore(name, tmpDir);
PosixSourceAccessor accessor;
auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir });
Attrs infoAttrs({
{"rev", input.getRev()->gitRev()},
@ -324,14 +331,14 @@ struct MercurialInputScheme : InputScheme
if (!_input.getRev())
getCache()->add(
store,
*store,
unlockedAttrs,
infoAttrs,
storePath,
false);
getCache()->add(
store,
*store,
getLockedAttrs(),
infoAttrs,
storePath,
@ -339,6 +346,14 @@ struct MercurialInputScheme : InputScheme
return makeResult(infoAttrs, std::move(storePath));
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev();
else
return std::nullopt;
}
};
static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });

77
src/libfetchers/mounted-input-accessor.cc Normal file
View file

@ -0,0 +1,77 @@
#include "mounted-input-accessor.hh"
namespace nix {
struct MountedInputAccessor : InputAccessor
{
std::map<CanonPath, ref<InputAccessor>> mounts;
MountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
: mounts(std::move(_mounts))
{
// Currently we require a root filesystem. This could be relaxed.
assert(mounts.contains(CanonPath::root));
// FIXME: return dummy parent directories automatically?
}
std::string readFile(const CanonPath & path) override
{
auto [accessor, subpath] = resolve(path);
return accessor->readFile(subpath);
}
bool pathExists(const CanonPath & path) override
{
auto [accessor, subpath] = resolve(path);
return accessor->pathExists(subpath);
}
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
auto [accessor, subpath] = resolve(path);
return accessor->maybeLstat(subpath);
}
DirEntries readDirectory(const CanonPath & path) override
{
auto [accessor, subpath] = resolve(path);
return accessor->readDirectory(subpath);
}
std::string readLink(const CanonPath & path) override
{
auto [accessor, subpath] = resolve(path);
return accessor->readLink(subpath);
}
std::string showPath(const CanonPath & path) override
{
auto [accessor, subpath] = resolve(path);
return accessor->showPath(subpath);
}
std::pair<ref<InputAccessor>, CanonPath> resolve(CanonPath path)
{
// Find the nearest parent of `path` that is a mount point.
std::vector<std::string> subpath;
while (true) {
auto i = mounts.find(path);
if (i != mounts.end()) {
std::reverse(subpath.begin(), subpath.end());
return {i->second, CanonPath(subpath)};
}
assert(!path.isRoot());
subpath.push_back(std::string(*path.baseName()));
path.pop();
}
}
};
ref<InputAccessor> makeMountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
{
return make_ref<MountedInputAccessor>(std::move(mounts));
}
}

9
src/libfetchers/mounted-input-accessor.hh Normal file
View file

@ -0,0 +1,9 @@
#pragma once
#include "input-accessor.hh"
namespace nix {
ref<InputAccessor> makeMountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
}

18
src/libfetchers/tarball.cc
View file

@ -8,6 +8,7 @@
#include "tarfile.hh"
#include "types.hh"
#include "split.hh"
#include "posix-source-accessor.hh"
namespace nix::fetchers {
@ -26,7 +27,7 @@ DownloadFileResult downloadFile(
{"name", name},
});
auto cached = getCache()->lookupExpired(store, inAttrs);
auto cached = getCache()->lookupExpired(*store, inAttrs);
auto useCached = [&]() -> DownloadFileResult
{
@ -73,7 +74,7 @@ DownloadFileResult downloadFile(
} else {
StringSink sink;
dumpString(res.data, sink);
auto hash = hashString(htSHA256, res.data);
auto hash = hashString(HashAlgorithm::SHA256, res.data);
ValidPathInfo info {
*store,
name,
@ -82,7 +83,7 @@ DownloadFileResult downloadFile(
.hash = hash,
.references = {},
},
hashString(htSHA256, sink.s),
hashString(HashAlgorithm::SHA256, sink.s),
};
info.narSize = sink.s.size();
auto source = StringSource { sink.s };
@ -91,7 +92,7 @@ DownloadFileResult downloadFile(
}
getCache()->add(
store,
*store,
inAttrs,
infoAttrs,
*storePath,
@ -99,7 +100,7 @@ DownloadFileResult downloadFile(
if (url != res.effectiveUri)
getCache()->add(
store,
*store,
{
{"type", "file"},
{"url", res.effectiveUri},
@ -130,7 +131,7 @@ DownloadTarballResult downloadTarball(
{"name", name},
});
auto cached = getCache()->lookupExpired(store, inAttrs);
auto cached = getCache()->lookupExpired(*store, inAttrs);
if (cached && !cached->expired)
return {
@ -156,7 +157,8 @@ DownloadTarballResult downloadTarball(
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
lastModified = lstat(topDir).st_mtime;
unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
PosixSourceAccessor accessor;
unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, defaultPathFilter, NoRepair);
}
Attrs infoAttrs({
@ -168,7 +170,7 @@ DownloadTarballResult downloadTarball(
infoAttrs.emplace("immutableUrl", *res.immutableUrl);
getCache()->add(
store,
*store,
inAttrs,
infoAttrs,
*unpackedStorePath,

2
src/libmain/local.mk
View file

@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil
libmain_ALLOW_UNDEFINED = 1
$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))
$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))

8
src/libmain/progress-bar.cc
View file

@ -340,6 +340,14 @@ public:
state->activitiesByType[type].expected += j;
update(*state);
}
else if (type == resFetchStatus) {
auto i = state->its.find(act);
assert(i != state->its.end());
ActInfo & actInfo = *i->second;
actInfo.lastLine = getS(fields, 0);
update(*state);
}
}
void update(State & state)

134
src/libstore/binary-cache-store.cc
View file

@ -12,6 +12,7 @@
#include "thread-pool.hh"
#include "callback.hh"
#include "signals.hh"
#include "archive.hh"
#include <chrono>
#include <future>
@ -27,7 +28,8 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
, Store(params)
{
if (secretKeyFile != "")
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
signer = std::make_unique<LocalSigner>(
SecretKey { readFile(secretKeyFile) });
StringSink sink;
sink << narVersionMagic1;
@ -143,9 +145,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink { htSHA256 };
HashSink fileHashSink { HashAlgorithm::SHA256 };
std::shared_ptr<SourceAccessor> narAccessor;
HashSink narHashSink { htSHA256 };
HashSink narHashSink { HashAlgorithm::SHA256 };
{
FdSink fileSink(fdTemp.get());
TeeSink teeSinkCompressed { fileSink, fileHashSink };
@ -165,8 +167,8 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "zstd" ? ".zst" :
compression == "lzip" ? ".lzip" :
@ -273,7 +275,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/
if (secretKey) narInfo->sign(*this, *secretKey);
if (signer) narInfo->sign(*this, *signer);
writeNarInfo(narInfo);
@ -300,24 +302,60 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}});
}
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
StorePath BinaryCacheStore::addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair)
{
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
std::optional<Hash> caHash;
std::string nar;
if (auto * dump2p = dynamic_cast<StringSource *>(&dump)) {
auto & dump2 = *dump2p;
// Hack, this gives us a "replayable" source so we can compute
// multiple hashes more easily.
caHash = hashString(HashAlgorithm::SHA256, dump2.s);
switch (method.getFileIngestionMethod()) {
case FileIngestionMethod::Recursive:
// The dump is already NAR in this case, just use it.
nar = dump2.s;
break;
case FileIngestionMethod::Flat:
// The dump is Flat, so we need to convert it to NAR with a
// single file.
StringSink s;
dumpString(dump2.s, s);
nar = std::move(s.s);
break;
}
} else {
// Otherwise, we have to do th same hashing as NAR so our single
// hash will suffice for both purposes.
if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
unsupported("addToStoreFromDump");
}
StringSource narDump { nar };
// Use `narDump` if we wrote to `nar`.
Source & narDump2 = nar.size() > 0
? static_cast<Source &>(narDump)
: dump;
return addToStoreCommon(narDump2, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
*this,
name,
FixedOutputInfo {
.method = method,
.hash = nar.first,
.references = {
ContentAddressWithReferences::fromParts(
method,
caHash ? *caHash : nar.first,
{
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false,
},
},
}),
nar.first,
};
info.narSize = nar.second;
@ -400,71 +438,35 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
StorePath BinaryCacheStore::addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
SourceAccessor & accessor,
const CanonPath & path,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references)
RepairFlag repair)
{
/* FIXME: Make BinaryCacheStore::addToStoreCommon support
non-recursive+sha256 so we can just use the default
implementation of this method in terms of addToStoreFromDump. */
HashSink sink { hashAlgo };
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
} else {
readFile(srcPath, sink);
}
auto h = sink.finish().first;
auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
auto source = sinkToSource([&](Sink & sink) {
dumpPath(srcPath, sink, filter);
accessor.dumpPath(path, sink, filter);
});
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
*this,
name,
FixedOutputInfo {
.method = method,
.hash = h,
.references = {
ContentAddressWithReferences::fromParts(
method,
h,
{
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false,
},
},
nar.first,
};
info.narSize = nar.second;
return info;
})->path;
}
StorePath BinaryCacheStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair)
{
auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, TextInfo { { textHash }, references });
if (!repair && isValidPath(path))
return path;
StringSink sink;
dumpString(s, sink);
StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
*this,
std::string { name },
TextInfo {
.hash = textHash,
.references = references,
},
}),
nar.first,
};
info.narSize = nar.second;

29
src/libstore/binary-cache-store.hh
View file

@ -1,7 +1,7 @@
#pragma once
///@file
#include "crypto.hh"
#include "signature/local-keys.hh"
#include "store-api.hh"
#include "log-store.hh"
@ -57,8 +57,7 @@ class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
{
private:
std::unique_ptr<SecretKey> secretKey;
std::unique_ptr<Signer> signer;
protected:
@ -123,22 +122,22 @@ public:
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair) override;
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,
SourceAccessor & accessor,
const CanonPath & srcPath,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
PathFilter & filter,
RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;

77
src/libstore/build/derivation-goal.cc
View file

@ -196,10 +196,19 @@ void DerivationGoal::loadDerivation()
things being garbage collected while we're busy. */
worker.evalStore.addTempRoot(drvPath);
assert(worker.evalStore.isValidPath(drvPath));
/* Get the derivation. It is probably in the eval store, but it might be inthe main store:
/* Get the derivation. */
drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
- Resolved derivation are resolved against main store realisations, and so must be stored there.
- Dynamic derivations are built, and so are found in the main store.
*/
for (auto * drvStore : { &worker.evalStore, &worker.store }) {
if (drvStore->isValidPath(drvPath)) {
drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
break;
}
}
assert(drv);
haveDerivation();
}
@ -401,11 +410,15 @@ void DerivationGoal::gaveUpOnSubstitution()
}
/* Copy the input sources from the eval store to the build
store. */
store.
Note that some inputs might not be in the eval store because they
are (resolved) derivation outputs in a resolved derivation. */
if (&worker.evalStore != &worker.store) {
RealisedPath::Set inputSrcs;
for (auto & i : drv->inputSrcs)
inputSrcs.insert(i);
if (worker.evalStore.isValidPath(i))
inputSrcs.insert(i);
copyClosure(worker.evalStore, worker.store, inputSrcs);
}
@ -453,7 +466,7 @@ void DerivationGoal::repairClosure()
std::map<StorePath, StorePath> outputsToDrv;
for (auto & i : inputClosure)
if (i.isDerivation()) {
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
for (auto & j : depOutputs)
if (j.second)
outputsToDrv.insert_or_assign(*j.second, i);
@ -558,7 +571,7 @@ void DerivationGoal::inputsRealised()
inputDrvOutputs statefully, sometimes it gets out of sync with
the real source of truth (store). So we query the store
directly if there's a problem. */
attempt = fullDrv.tryResolve(worker.store);
attempt = fullDrv.tryResolve(worker.store, &worker.evalStore);
}
assert(attempt);
Derivation drvResolved { std::move(*attempt) };
@ -604,7 +617,13 @@ void DerivationGoal::inputsRealised()
return *outPath;
}
else {
auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
auto outMap = [&]{
for (auto * drvStore : { &worker.evalStore, &worker.store })
if (drvStore->isValidPath(depDrvPath))
return worker.store.queryDerivationOutputMap(depDrvPath, drvStore);
assert(false);
}();
auto outMapPath = outMap.find(outputName);
if (outMapPath == outMap.end()) {
throw Error(
@ -1085,8 +1104,12 @@ void DerivationGoal::resolvedFinished()
auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
newRealisation.signatures.clear();
if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
if (!drv->type().isFixed()) {
auto & drvStore = worker.evalStore.isValidPath(drvPath)
? worker.evalStore
: worker.store;
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
}
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
}
@ -1317,9 +1340,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
// ensure that logs from a builder using `ssh-ng://` as protocol
// are also available to `nix log`.
if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) {
auto f = (*json)["fields"];
(*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n");
if (s && !isWrittenToLog && logSink) {
const auto type = (*json)["type"];
const auto fields = (*json)["fields"];
if (type == resBuildLogLine) {
(*logSink)((fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n");
} else if (type == resSetPhase && ! fields.is_null()) {
const auto phase = fields[0];
if (! phase.is_null()) {
// nixpkgs' stdenv produces lines in the log to signal
// phase changes.
// We want to get the same lines in case of remote builds.
// The format is:
// @nix { "action": "setPhase", "phase": "$curPhase" }
const auto logLine = nlohmann::json::object({
{"action", "setPhase"},
{"phase", phase}
});
(*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n");
}
}
}
}
currentHookLine.clear();
@ -1362,7 +1402,10 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
res.insert_or_assign(name, output.path(worker.store, drv->name, name));
return res;
} else {
return worker.store.queryPartialDerivationOutputMap(drvPath);
for (auto * drvStore : { &worker.evalStore, &worker.store })
if (drvStore->isValidPath(drvPath))
return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore);
assert(false);
}
}
@ -1375,7 +1418,10 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
res.insert_or_assign(name, *output.second);
return res;
} else {
return worker.store.queryDerivationOutputMap(drvPath);
for (auto * drvStore : { &worker.evalStore, &worker.store })
if (drvStore->isValidPath(drvPath))
return worker.store.queryDerivationOutputMap(drvPath, drvStore);
assert(false);
}
}
@ -1474,6 +1520,7 @@ void DerivationGoal::done(
SingleDrvOutputs builtOutputs,
std::optional<Error> ex)
{
outputLocks.unlock();
buildResult.status = status;
if (ex)
buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));

8
src/libstore/build/entry-points.cc
View file

@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
worker.run(goals);
StorePathSet failed;
StringSet failed;
std::optional<Error> ex;
for (auto & i : goals) {
if (i->ex) {
@ -26,9 +26,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
}
if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
failed.insert(i2->drvPath);
failed.insert(std::string { i2->drvPath.to_string() });
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
failed.insert(i2->storePath);
failed.insert(std::string { i2->storePath.to_string()});
}
}
@ -37,7 +37,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
throw std::move(*ex);
} else if (!failed.empty()) {
if (ex) logError(ex->info());
throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
}
}

106
src/libstore/build/local-derivation-goal.cc
View file

@ -19,6 +19,8 @@
#include "namespaces.hh"
#include "child.hh"
#include "unix-domain-socket.hh"
#include "posix-fs-canonicalise.hh"
#include "posix-source-accessor.hh"
#include <regex>
#include <queue>
@ -1065,8 +1067,8 @@ void LocalDerivationGoal::initTmpDir() {
if (passAsFile.find(i.first) == passAsFile.end()) {
env[i.first] = i.second;
} else {
auto hash = hashString(htSHA256, i.first);
std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false);
auto hash = hashString(HashAlgorithm::SHA256, i.first);
std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false);
Path p = tmpDir + "/" + fn;
writeFile(p, rewriteStrings(i.second, inputRewrites));
chownToBuilder(p);
@ -1290,12 +1292,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
SourceAccessor & accessor,
const CanonPath & srcPath,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override
RepairFlag repair) override
{ throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
@ -1305,26 +1308,15 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
goal.addDependency(info.path);
}
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair = NoRepair) override
{
auto path = next->addTextToStore(name, s, references, repair);
goal.addDependency(path);
return path;
}
StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
FileIngestionMethod method,
HashType hashAlgo,
RepairFlag repair,
const StorePathSet & references) override
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair) override
{
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair);
goal.addDependency(path);
return path;
}
@ -2452,8 +2444,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
throw BuildError(
"output path %1% without valid stats info",
actualPath);
if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat)
{
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
@ -2465,38 +2456,23 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
rewriteOutput(outputRewrites);
/* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath->hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart };
std::visit(overloaded {
[&](const TextIngestionMethod &) {
readFile(actualPath, caSink);
},
[&](const FileIngestionMethod & m2) {
switch (m2) {
case FileIngestionMethod::Recursive:
dumpPath(actualPath, caSink);
break;
case FileIngestionMethod::Flat:
readFile(actualPath, caSink);
break;
}
},
}, outputHash.method.raw);
auto got = caSink.finish().first;
auto got = ({
HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
PosixSourceAccessor accessor;
dumpPath(
accessor, CanonPath { actualPath },
caSink,
outputHash.method.getFileIngestionMethod());
caSink.finish().first;
});
auto optCA = ContentAddressWithReferences::fromPartsOpt(
outputHash.method,
std::move(got),
rewriteRefs());
if (!optCA) {
// TODO track distinct failure modes separately (at the time of
// writing there is just one but `nullopt` is unclear) so this
// message can't get out of sync.
throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
}
ValidPathInfo newInfo0 {
worker.store,
outputPathName(drv->name, outputName),
std::move(*optCA),
ContentAddressWithReferences::fromParts(
outputHash.method,
std::move(got),
rewriteRefs()),
Hash::dummy,
};
if (*scratchPath != newInfo0.path) {
@ -2510,9 +2486,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
std::string(newInfo0.path.hashPart())}});
}
HashResult narHashAndSize = hashPath(htSHA256, actualPath);
newInfo0.narHash = narHashAndSize.first;
newInfo0.narSize = narHashAndSize.second;
{
PosixSourceAccessor accessor;
HashResult narHashAndSize = hashPath(
accessor, CanonPath { actualPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
newInfo0.narHash = narHashAndSize.first;
newInfo0.narSize = narHashAndSize.second;
}
assert(newInfo0.ca);
return newInfo0;
@ -2530,7 +2511,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
std::string { scratchPath->hashPart() },
std::string { requiredFinalPath.hashPart() });
rewriteOutput(outputRewrites);
auto narHashAndSize = hashPath(htSHA256, actualPath);
PosixSourceAccessor accessor;
HashResult narHashAndSize = hashPath(
accessor, CanonPath { actualPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
newInfo0.narSize = narHashAndSize.second;
auto refs = rewriteRefs();
@ -2545,7 +2529,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
.method = dof.ca.method,
.hashType = wanted.type,
.hashAlgo = wanted.algo,
});
/* Check wanted hash */
@ -2582,7 +2566,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
[&](const DerivationOutput::Impure & doi) {
return newInfoFromCA(DerivationOutput::CAFloating {
.method = doi.method,
.hashType = doi.hashType,
.hashAlgo = doi.hashAlgo,
});
},
@ -2944,7 +2928,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName)
{
return worker.store.makeStorePath(
"rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
Hash(htSHA256), outputPathName(drv->name, outputName));
Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName));
}
@ -2952,7 +2936,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path)
{
return worker.store.makeStorePath(
"rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
Hash(htSHA256), path.name());
Hash(HashAlgorithm::SHA256), path.name());
}

1
src/libstore/build/sandbox-defaults.sb
View file

@ -68,6 +68,7 @@ R""(
(allow file*
(literal "/dev/null")
(literal "/dev/random")
(literal "/dev/stderr")
(literal "/dev/stdin")
(literal "/dev/stdout")
(literal "/dev/tty")

3
src/libstore/build/substitution-goal.cc
View file

@ -2,6 +2,7 @@
#include "substitution-goal.hh"
#include "nar-info.hh"
#include "finally.hh"
#include "signals.hh"
namespace nix {
@ -217,6 +218,8 @@ void PathSubstitutionGoal::tryToRun()
thr = std::thread([this]() {
try {
ReceiveInterrupts receiveInterrupts;
/* Wake up the worker loop when we're done. */
Finally updateStats([this]() { outPipe.writeSide.close(); });

27
src/libstore/build/worker.cc
View file

@ -199,8 +199,16 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
child.respectTimeouts = respectTimeouts;
children.emplace_back(child);
if (inBuildSlot) {
if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
else nrLocalBuilds++;
switch (goal->jobCategory()) {
case JobCategory::Substitution:
nrSubstitutions++;
break;
case JobCategory::Build:
nrLocalBuilds++;
break;
default:
abort();
}
}
}
@ -212,12 +220,17 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
if (i == children.end()) return;
if (i->inBuildSlot) {
if (goal->jobCategory() == JobCategory::Substitution) {
switch (goal->jobCategory()) {
case JobCategory::Substitution:
assert(nrSubstitutions > 0);
nrSubstitutions--;
} else {
break;
case JobCategory::Build:
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
break;
default:
abort();
}
}
@ -506,8 +519,10 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
Hash nullHash(htSHA256);
HashResult current = hashPath(
*store.getFSAccessor(), CanonPath { store.printStorePath(path) },
FileIngestionMethod::Recursive, info->narHash.algo);
Hash nullHash(HashAlgorithm::SHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
pathContentsGoodCache.insert_or_assign(path, res);

1
src/libstore/builtins/buildenv.cc
View file

@ -1,4 +1,5 @@
#include "buildenv.hh"
#include "derivations.hh"
#include <sys/stat.h>
#include <sys/types.h>

1
src/libstore/builtins/buildenv.hh
View file

@ -1,7 +1,6 @@
#pragma once
///@file
#include "derivations.hh"
#include "store-api.hh"
namespace nix {

4
src/libstore/builtins/fetchurl.cc
View file

@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
std::optional<HashAlgorithm> ht = parseHashAlgoOpt(getAttr("outputHashAlgo"));
Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false));
fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false));
return;
} catch (Error & e) {
debug(e.what());

4
src/libstore/common-protocol-impl.hh
View file

@ -16,11 +16,11 @@ namespace nix {
/* protocol-agnostic templates */
#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \
TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \
{ \
return LengthPrefixedProtoHelper<CommonProto, T >::read(store, conn); \
} \
TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \
TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \
{ \
LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
}

28
src/libstore/common-protocol.cc
View file

@ -13,40 +13,40 @@ namespace nix {
/* protocol-agnostic definitions */
std::string CommonProto::Serialise<std::string>::read(const Store & store, CommonProto::ReadConn conn)
std::string CommonProto::Serialise<std::string>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
return readString(conn.from);
}
void CommonProto::Serialise<std::string>::write(const Store & store, CommonProto::WriteConn conn, const std::string & str)
void CommonProto::Serialise<std::string>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str)
{
conn.to << str;
}
StorePath CommonProto::Serialise<StorePath>::read(const Store & store, CommonProto::ReadConn conn)
StorePath CommonProto::Serialise<StorePath>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
return store.parseStorePath(readString(conn.from));
}
void CommonProto::Serialise<StorePath>::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath)
void CommonProto::Serialise<StorePath>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath)
{
conn.to << store.printStorePath(storePath);
}
ContentAddress CommonProto::Serialise<ContentAddress>::read(const Store & store, CommonProto::ReadConn conn)
ContentAddress CommonProto::Serialise<ContentAddress>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
return ContentAddress::parse(readString(conn.from));
}
void CommonProto::Serialise<ContentAddress>::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca)
void CommonProto::Serialise<ContentAddress>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca)
{
conn.to << renderContentAddress(ca);
}
Realisation CommonProto::Serialise<Realisation>::read(const Store & store, CommonProto::ReadConn conn)
Realisation CommonProto::Serialise<Realisation>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
std::string rawInput = readString(conn.from);
return Realisation::fromJSON(
@ -55,41 +55,41 @@ Realisation CommonProto::Serialise<Realisation>::read(const Store & store, Commo
);
}
void CommonProto::Serialise<Realisation>::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation)
void CommonProto::Serialise<Realisation>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation)
{
conn.to << realisation.toJSON().dump();
}
DrvOutput CommonProto::Serialise<DrvOutput>::read(const Store & store, CommonProto::ReadConn conn)
DrvOutput CommonProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
return DrvOutput::parse(readString(conn.from));
}
void CommonProto::Serialise<DrvOutput>::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
void CommonProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
{
conn.to << drvOutput.to_string();
}
std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const Store & store, CommonProto::ReadConn conn)
std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
auto s = readString(conn.from);
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
}
void CommonProto::Serialise<std::optional<StorePath>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
void CommonProto::Serialise<std::optional<StorePath>>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
{
conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
}
std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, CommonProto::ReadConn conn)
std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
return ContentAddress::parseOpt(readString(conn.from));
}
void CommonProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
void CommonProto::Serialise<std::optional<ContentAddress>>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
{
conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
}

8
src/libstore/common-protocol.hh
View file

@ -5,7 +5,7 @@
namespace nix {
class Store;
struct StoreDirConfig;
struct Source;
// items being serialized
@ -48,7 +48,7 @@ struct CommonProto
* infer the type instead of having to write it down explicitly.
*/
template<typename T>
static void write(const Store & store, WriteConn conn, const T & t)
static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
{
CommonProto::Serialise<T>::write(store, conn, t);
}
@ -57,8 +57,8 @@ struct CommonProto
#define DECLARE_COMMON_SERIALISER(T) \
struct CommonProto::Serialise< T > \
{ \
static T read(const Store & store, CommonProto::ReadConn conn); \
static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \
static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \
static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \
}
template<>

54
src/libstore/content-address.cc
View file

@ -38,14 +38,26 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
return FileIngestionMethod::Flat;
}
std::string ContentAddressMethod::render(HashType ht) const
std::string ContentAddressMethod::render(HashAlgorithm ha) const
{
return std::visit(overloaded {
[&](const TextIngestionMethod & th) {
return std::string{"text:"} + printHashType(ht);
return std::string{"text:"} + printHashAlgo(ha);
},
[&](const FileIngestionMethod & fim) {
return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha);
}
}, raw);
}
FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const
{
return std::visit(overloaded {
[&](const TextIngestionMethod & th) {
return FileIngestionMethod::Flat;
},
[&](const FileIngestionMethod & fim) {
return fim;
}
}, raw);
}
@ -61,13 +73,13 @@ std::string ContentAddress::render() const
+ makeFileIngestionPrefix(method);
},
}, method.raw)
+ this->hash.to_string(HashFormat::Base32, true);
+ this->hash.to_string(HashFormat::Nix32, true);
}
/**
* Parses content address strings up to the hash.
*/
static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodPrefix(std::string_view & rest)
{
std::string_view wholeInput { rest };
@ -79,31 +91,31 @@ static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix
prefix = *optPrefix;
}
auto parseHashType_ = [&](){
auto parseHashAlgorithm_ = [&](){
auto hashTypeRaw = splitPrefixTo(rest, ':');
if (!hashTypeRaw)
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
HashType hashType = parseHashType(*hashTypeRaw);
return hashType;
HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw);
return hashAlgo;
};
// Switch on prefix
if (prefix == "text") {
// No parsing of the ingestion method, "text" only support flat.
HashType hashType = parseHashType_();
HashAlgorithm hashAlgo = parseHashAlgorithm_();
return {
TextIngestionMethod {},
std::move(hashType),
std::move(hashAlgo),
};
} else if (prefix == "fixed") {
// Parse method
auto method = FileIngestionMethod::Flat;
if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive;
HashType hashType = parseHashType_();
HashAlgorithm hashAlgo = parseHashAlgorithm_();
return {
std::move(method),
std::move(hashType),
std::move(hashAlgo),
};
} else
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
@ -113,15 +125,15 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
{
auto rest = rawCa;
auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest);
auto [caMethod, hashAlgo] = parseContentAddressMethodPrefix(rest);
return ContentAddress {
.method = std::move(caMethod),
.hash = Hash::parseNonSRIUnprefixed(rest, hashType),
.hash = Hash::parseNonSRIUnprefixed(rest, hashAlgo),
};
}
std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parse(std::string_view caMethod)
{
std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference
@ -144,7 +156,7 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
std::string ContentAddress::printMethodAlgo() const
{
return method.renderPrefix()
+ printHashType(hash.type);
+ printHashAlgo(hash.algo);
}
bool StoreReferences::empty() const
@ -176,13 +188,13 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
}, ca.method.raw);
}
std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept
ContentAddressWithReferences ContentAddressWithReferences::fromParts(
ContentAddressMethod method, Hash hash, StoreReferences refs)
{
return std::visit(overloaded {
[&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> {
[&](TextIngestionMethod _) -> ContentAddressWithReferences {
if (refs.self)
return std::nullopt;
throw Error("self-reference not allowed with text hashing");
return ContentAddressWithReferences {
TextInfo {
.hash = std::move(hash),
@ -190,7 +202,7 @@ std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPa
}
};
},
[&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
[&](FileIngestionMethod m2) -> ContentAddressWithReferences {
return ContentAddressWithReferences {
FixedOutputInfo {
.method = m2,

48
src/libstore/content-address.hh
View file

@ -4,6 +4,7 @@
#include <variant>
#include "hash.hh"
#include "path.hh"
#include "file-content-address.hh"
#include "comparator.hh"
#include "variant-wrapper.hh"
@ -31,22 +32,6 @@ namespace nix {
*/
struct TextIngestionMethod : std::monostate { };
/**
* An enumeration of the main ways we can serialize file system
* objects.
*/
enum struct FileIngestionMethod : uint8_t {
/**
* Flat-file hashing. Directly ingest the contents of a single file
*/
Flat = false,
/**
* Recursive (or NAR) hashing. Serializes the file-system object in Nix
* Archive format and ingest that
*/
Recursive = true
};
/**
* Compute the prefix to the hash algorithm which indicates how the
* files were ingested.
@ -54,7 +39,7 @@ enum struct FileIngestionMethod : uint8_t {
std::string makeFileIngestionPrefix(FileIngestionMethod m);
/**
* An enumeration of all the ways we can serialize file system objects.
* An enumeration of all the ways we can content-address store objects.
*
* Just the type of a content address. Combine with the hash itself, and
* we have a `ContentAddress` as defined below. Combine that, in turn,
@ -94,7 +79,7 @@ struct ContentAddressMethod
/**
* Parse a content addressing method and hash type.
*/
static std::pair<ContentAddressMethod, HashType> parse(std::string_view rawCaMethod);
static std::pair<ContentAddressMethod, HashAlgorithm> parse(std::string_view rawCaMethod);
/**
* Render a content addressing method and hash type in a
@ -102,7 +87,15 @@ struct ContentAddressMethod
*
* The rough inverse of `parse()`.
*/
std::string render(HashType ht) const;
std::string render(HashAlgorithm ht) const;
/**
* Get the underlying way to content-address file system objects.
*
* Different ways of hashing store objects may use the same method
* for hashing file systeme objects.
*/
FileIngestionMethod getFileIngestionMethod() const;
};
@ -116,11 +109,11 @@ struct ContentAddressMethod
* serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the
* following forms:
*
* - ‘text:sha256:<sha256 hash of file contents>’: For paths
* computed by Store::makeTextPath() / Store::addTextToStore().
* - `TextIngestionMethod`:
* ‘text:sha256:<sha256 hash of file contents>’
*
* - ‘fixed:<r?>:<ht>:<h>’: For paths computed by
* Store::makeFixedOutputPath() / Store::addToStore().
* - `FixedIngestionMethod`:
* ‘fixed:<r?>:<hash type>:<hash of file contents>’
*/
struct ContentAddress
{
@ -266,11 +259,12 @@ struct ContentAddressWithReferences
*
* @param refs References to other store objects or oneself.
*
* Do note that not all combinations are supported; `nullopt` is
* returns for invalid combinations.
* @note note that all combinations are supported. This is a
* *partial function* and exceptions will be thrown for invalid
* combinations.
*/
static std::optional<ContentAddressWithReferences> fromPartsOpt(
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept;
static ContentAddressWithReferences fromParts(
ContentAddressMethod method, Hash hash, StoreReferences refs);
ContentAddressMethod getMethod() const;

58
src/libstore/daemon.cc
View file

@ -400,31 +400,18 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork();
auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr);
auto hashType = hashType_; // work around clang bug
auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
auto hashAlgo = hashAlgo_; // work around clang bug
FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded {
[&](const TextIngestionMethod &) {
if (hashType != htSHA256)
throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
name, printHashType(hashType));
// We could stream this by changing Store
std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path);
},
[&](const FileIngestionMethod & fim) {
auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
return store->queryPathInfo(path);
},
}, contentAddressMethod.raw);
// TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair);
return store->queryPathInfo(path);
}();
logger->stopWork();
WorkerProto::Serialise<ValidPathInfo>::write(*store, wconn, *pathInfo);
} else {
HashType hashAlgo;
HashAlgorithm hashAlgo;
std::string baseName;
FileIngestionMethod method;
{
@ -440,7 +427,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
hashAlgoRaw = "sha256";
method = FileIngestionMethod::Recursive;
}
hashAlgo = parseHashType(hashAlgoRaw);
hashAlgo = parseHashAlgo(hashAlgoRaw);
}
auto dumpSource = sinkToSource([&](Sink & saved) {
@ -496,7 +483,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
std::string s = readString(from);
auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
auto path = ({
StringSource source { s };
store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
});
logger->stopWork();
to << store->printStorePath(path);
break;
@ -574,6 +564,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case WorkerProto::Op::BuildDerivation: {
auto drvPath = store->parseStorePath(readString(from));
BasicDerivation drv;
/*
* Note: unlike wopEnsurePath, this operation reads a
* derivation-to-be-realized from the client with
* readDerivation(Source,Store) rather than reading it from
* the local store with Store::readDerivation(). Since the
* derivation-to-be-realized is not registered in the store
* it cannot be trusted that its outPath was calculated
* correctly.
*/
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
@ -657,6 +656,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
case WorkerProto::Op::AddPermRoot: {
if (!trusted)
throw Error(
"you are not privileged to create perm roots\n\n"
"hint: you can just do this client-side without special privileges, and probably want to do that instead.");
auto storePath = WorkerProto::Serialise<StorePath>::read(*store, rconn);
Path gcRoot = absPath(readString(from));
logger->startWork();
auto & localFSStore = require<LocalFSStore>(*store);
localFSStore.addPermRoot(storePath, gcRoot);
logger->stopWork();
to << gcRoot;
break;
}
case WorkerProto::Op::AddIndirectRoot: {
Path path = absPath(readString(from));
@ -868,7 +882,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
bool repair, dontCheckSigs;
auto path = store->parseStorePath(readString(from));
auto deriver = readString(from);
auto narHash = Hash::parseAny(readString(from), htSHA256);
auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256);
ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = store->parseStorePath(deriver);

231
src/libstore/derivations.cc
View file

@ -2,6 +2,7 @@
#include "downstream-placeholder.hh"
#include "store-api.hh"
#include "globals.hh"
#include "types.hh"
#include "util.hh"
#include "split.hh"
#include "common-protocol.hh"
@ -11,7 +12,7 @@
namespace nix {
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
std::optional<StorePath> DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
{
return std::visit(overloaded {
[](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
@ -35,7 +36,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
}
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
{
return store.makeFixedOutputPathFromCA(
outputPathName(drvName, outputName),
@ -143,37 +144,89 @@ StorePath writeDerivation(Store & store,
auto suffix = std::string(drv.name) + drvExtension;
auto contents = drv.unparse(store, false);
return readOnly || settings.readOnlyMode
? store.computeStorePathForText(suffix, contents, references)
: store.addTextToStore(suffix, contents, references, repair);
? store.makeFixedOutputPathFromCA(suffix, TextInfo {
.hash = hashString(HashAlgorithm::SHA256, contents),
.references = std::move(references),
})
: ({
StringSource s { contents };
store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
});
}
namespace {
/**
* This mimics std::istream to some extent. We use this much smaller implementation
* instead of plain istreams because the sentry object overhead is too high.
*/
struct StringViewStream {
std::string_view remaining;
int peek() const {
return remaining.empty() ? EOF : remaining[0];
}
int get() {
if (remaining.empty()) return EOF;
char c = remaining[0];
remaining.remove_prefix(1);
return c;
}
};
constexpr struct Escapes {
char map[256];
constexpr Escapes() {
for (int i = 0; i < 256; i++) map[i] = (char) (unsigned char) i;
map[(int) (unsigned char) 'n'] = '\n';
map[(int) (unsigned char) 'r'] = '\r';
map[(int) (unsigned char) 't'] = '\t';
}
char operator[](char c) const { return map[(unsigned char) c]; }
} escapes;
}
/* Read string `s' from stream `str'. */
static void expect(std::istream & str, std::string_view s)
static void expect(StringViewStream & str, std::string_view s)
{
char s2[s.size()];
str.read(s2, s.size());
std::string_view s2View { s2, s.size() };
if (s2View != s)
throw FormatError("expected string '%s', got '%s'", s, s2View);
if (!str.remaining.starts_with(s))
throw FormatError("expected string '%1%'", s);
str.remaining.remove_prefix(s.size());
}
/* Read a C-style string from stream `str'. */
static std::string parseString(std::istream & str)
static BackedStringView parseString(StringViewStream & str)
{
std::string res;
expect(str, "\"");
int c;
while ((c = str.get()) != '"')
if (c == '\\') {
c = str.get();
if (c == 'n') res += '\n';
else if (c == 'r') res += '\r';
else if (c == 't') res += '\t';
else res += c;
auto c = str.remaining.begin(), end = str.remaining.end();
bool escaped = false;
for (; c != end && *c != '"'; c++) {
if (*c == '\\') {
c++;
if (c == end)
throw FormatError("unterminated string in derivation");
escaped = true;
}
else res += c;
}
const auto contentLen = c - str.remaining.begin();
const auto content = str.remaining.substr(0, contentLen);
str.remaining.remove_prefix(contentLen + 1);
if (!escaped)
return content;
std::string res;
res.reserve(content.size());
for (c = content.begin(), end = content.end(); c != end; c++)
if (*c == '\\') {
c++;
res += escapes[*c];
}
else res += *c;
return res;
}
@ -182,15 +235,15 @@ static void validatePath(std::string_view s) {
throw FormatError("bad path '%1%' in derivation", s);
}
static Path parsePath(std::istream & str)
static BackedStringView parsePath(StringViewStream & str)
{
auto s = parseString(str);
validatePath(s);
validatePath(*s);
return s;
}
static bool endOfList(std::istream & str)
static bool endOfList(StringViewStream & str)
{
if (str.peek() == ',') {
str.get();
@ -204,37 +257,37 @@ static bool endOfList(std::istream & str)
}
static StringSet parseStrings(std::istream & str, bool arePaths)
static StringSet parseStrings(StringViewStream & str, bool arePaths)
{
StringSet res;
expect(str, "[");
while (!endOfList(str))
res.insert(arePaths ? parsePath(str) : parseString(str));
res.insert((arePaths ? parsePath(str) : parseString(str)).toOwned());
return res;
}
static DerivationOutput parseDerivationOutput(
const Store & store,
std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
const StoreDirConfig & store,
std::string_view pathS, std::string_view hashAlgoStr, std::string_view hashS,
const ExperimentalFeatureSettings & xpSettings)
{
if (hashAlgo != "") {
ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
if (hashAlgoStr != "") {
ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr);
if (method == TextIngestionMethod {})
xpSettings.require(Xp::DynamicDerivations);
const auto hashType = parseHashType(hashAlgo);
const auto hashAlgo = parseHashAlgo(hashAlgoStr);
if (hashS == "impure") {
xpSettings.require(Xp::ImpureDerivations);
if (pathS != "")
throw FormatError("impure derivation output should not specify output path");
return DerivationOutput::Impure {
.method = std::move(method),
.hashType = std::move(hashType),
.hashAlgo = std::move(hashAlgo),
};
} else if (hashS != "") {
validatePath(pathS);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashAlgo);
return DerivationOutput::CAFixed {
.ca = ContentAddress {
.method = std::move(method),
@ -247,7 +300,7 @@ static DerivationOutput parseDerivationOutput(
throw FormatError("content-addressed derivation output should not specify output path");
return DerivationOutput::CAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
.hashAlgo = std::move(hashAlgo),
};
}
} else {
@ -262,7 +315,7 @@ static DerivationOutput parseDerivationOutput(
}
static DerivationOutput parseDerivationOutput(
const Store & store, std::istringstream & str,
const StoreDirConfig & store, StringViewStream & str,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
{
expect(str, ","); const auto pathS = parseString(str);
@ -270,7 +323,7 @@ static DerivationOutput parseDerivationOutput(
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings);
return parseDerivationOutput(store, *pathS, *hashAlgo, *hash, xpSettings);
}
/**
@ -291,8 +344,8 @@ enum struct DerivationATermVersion {
};
static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
const Store & store,
std::istringstream & str,
const StoreDirConfig & store,
StringViewStream & str,
DerivationATermVersion version)
{
DerivedPathMap<StringSet>::ChildNode node;
@ -318,7 +371,7 @@ static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "(");
auto outputName = parseString(str);
auto outputName = parseString(str).toOwned();
expect(str, ",");
node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version));
expect(str, ")");
@ -338,13 +391,13 @@ static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
Derivation parseDerivation(
const Store & store, std::string && s, std::string_view name,
const StoreDirConfig & store, std::string && s, std::string_view name,
const ExperimentalFeatureSettings & xpSettings)
{
Derivation drv;
drv.name = name;
std::istringstream str(std::move(s));
StringViewStream str{s};
expect(str, "D");
DerivationATermVersion version;
switch (str.peek()) {
@ -355,12 +408,12 @@ Derivation parseDerivation(
case 'r': {
expect(str, "rvWithVersion(");
auto versionS = parseString(str);
if (versionS == "xp-dyn-drv") {
if (*versionS == "xp-dyn-drv") {
// Only verison we have so far
version = DerivationATermVersion::DynamicDerivations;
xpSettings.require(Xp::DynamicDerivations);
} else {
throw FormatError("Unknown derivation ATerm format version '%s'", versionS);
throw FormatError("Unknown derivation ATerm format version '%s'", *versionS);
}
expect(str, ",");
break;
@ -372,7 +425,7 @@ Derivation parseDerivation(
/* Parse the list of outputs. */
expect(str, "[");
while (!endOfList(str)) {
expect(str, "("); std::string id = parseString(str);
expect(str, "("); std::string id = parseString(str).toOwned();
auto output = parseDerivationOutput(store, str, xpSettings);
drv.outputs.emplace(std::move(id), std::move(output));
}
@ -381,28 +434,28 @@ Derivation parseDerivation(
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "(");
Path drvPath = parsePath(str);
auto drvPath = parsePath(str);
expect(str, ",");
drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version));
drv.inputDrvs.map.insert_or_assign(store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version));
expect(str, ")");
}
expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
expect(str, ","); drv.platform = parseString(str);
expect(str, ","); drv.builder = parseString(str);
expect(str, ","); drv.platform = parseString(str).toOwned();
expect(str, ","); drv.builder = parseString(str).toOwned();
/* Parse the builder arguments. */
expect(str, ",[");
while (!endOfList(str))
drv.args.push_back(parseString(str));
drv.args.push_back(parseString(str).toOwned());
/* Parse the environment variables. */
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "("); auto name = parseString(str);
expect(str, ","); auto value = parseString(str);
expect(str, "("); auto name = parseString(str).toOwned();
expect(str, ","); auto value = parseString(str).toOwned();
expect(str, ")");
drv.env[name] = value;
drv.env.insert_or_assign(std::move(name), std::move(value));
}
expect(str, ")");
@ -471,7 +524,7 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt
}
static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
{
s += ',';
if (node.childMap.empty()) {
@ -512,7 +565,7 @@ static bool hasDynamicDrvDep(const Derivation & drv)
}
std::string Derivation::unparse(const Store & store, bool maskOutputs,
std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const
{
std::string s;
@ -548,7 +601,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
},
[&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType));
s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo));
s += ','; printUnquotedString(s, "");
},
[&](const DerivationOutput::Deferred &) {
@ -559,7 +612,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
[&](const DerivationOutput::Impure & doi) {
// FIXME
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo));
s += ','; printUnquotedString(s, "impure");
}
}, i.second.raw);
@ -632,7 +685,7 @@ DerivationType BasicDerivation::type() const
floatingCAOutputs,
deferredIAOutputs,
impureOutputs;
std::optional<HashType> floatingHashType;
std::optional<HashAlgorithm> floatingHashAlgo;
for (auto & i : outputs) {
std::visit(overloaded {
@ -644,10 +697,10 @@ DerivationType BasicDerivation::type() const
},
[&](const DerivationOutput::CAFloating & dof) {
floatingCAOutputs.insert(i.first);
if (!floatingHashType) {
floatingHashType = dof.hashType;
if (!floatingHashAlgo) {
floatingHashAlgo = dof.hashAlgo;
} else {
if (*floatingHashType != dof.hashType)
if (*floatingHashAlgo != dof.hashAlgo)
throw Error("all floating outputs must use the same hash type");
}
},
@ -775,7 +828,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
auto hash = hashString(htSHA256, "fixed:out:"
auto hash = hashString(HashAlgorithm::SHA256, "fixed:out:"
+ dof.ca.printMethodAlgo() + ":"
+ dof.ca.hash.to_string(HashFormat::Base16, false) + ":"
+ store.printStorePath(dof.path(store, drv.name, i.first)));
@ -826,7 +879,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
}
}
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs) {
@ -846,7 +899,7 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store)
{
const auto pathS = readString(in);
const auto hashAlgo = readString(in);
@ -863,7 +916,7 @@ StringSet BasicDerivation::outputNames() const
return names;
}
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const
{
DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto & [outputName, output] : outputs)
@ -885,7 +938,7 @@ std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
}
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name)
{
drv.name = name;
@ -913,7 +966,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
}
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
for (auto & i : drv.outputs) {
@ -931,7 +984,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
},
[&](const DerivationOutput::CAFloating & dof) {
out << ""
<< (dof.method.renderPrefix() + printHashType(dof.hashType))
<< (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo))
<< "";
},
[&](const DerivationOutput::Deferred &) {
@ -941,7 +994,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
},
[&](const DerivationOutput::Impure & doi) {
out << ""
<< (doi.method.renderPrefix() + printHashType(doi.hashType))
<< (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo))
<< "impure";
},
}, i.second.raw);
@ -959,7 +1012,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
std::string hashPlaceholder(const OutputNameView outputName)
{
// FIXME: memoize?
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false);
}
@ -1003,13 +1056,13 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const
{
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
accum = [&](auto & inputDrv, auto & node) {
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) {
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) {
if (outputPath) {
inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
if (auto p = get(node.childMap, outputName))
@ -1151,10 +1204,10 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
}
const Hash impureOutputHash = hashString(htSHA256, "impure");
const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure");
nlohmann::json DerivationOutput::toJSON(
const Store & store, std::string_view drvName, OutputNameView outputName) const
const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
{
nlohmann::json res = nlohmann::json::object();
std::visit(overloaded {
@ -1168,11 +1221,11 @@ nlohmann::json DerivationOutput::toJSON(
// FIXME print refs?
},
[&](const DerivationOutput::CAFloating & dof) {
res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType);
res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo);
},
[&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) {
res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo);
res["impure"] = true;
},
}, raw);
@ -1181,7 +1234,7 @@ nlohmann::json DerivationOutput::toJSON(
DerivationOutput DerivationOutput::fromJSON(
const Store & store, std::string_view drvName, OutputNameView outputName,
const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName,
const nlohmann::json & _json,
const ExperimentalFeatureSettings & xpSettings)
{
@ -1192,15 +1245,15 @@ DerivationOutput DerivationOutput::fromJSON(
for (const auto & [key, _] : json)
keys.insert(key);
auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> {
std::string hashAlgo = json["hashAlgo"];
auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashAlgorithm> {
std::string hashAlgoStr = json["hashAlgo"];
// remaining to parse, will be mutated by parsers
std::string_view s = hashAlgo;
std::string_view s = hashAlgoStr;
ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
if (method == TextIngestionMethod {})
xpSettings.require(Xp::DynamicDerivations);
auto hashType = parseHashType(s);
return { std::move(method), std::move(hashType) };
auto hashAlgo = parseHashAlgo(s);
return { std::move(method), std::move(hashAlgo) };
};
if (keys == (std::set<std::string_view> { "path" })) {
@ -1210,11 +1263,11 @@ DerivationOutput DerivationOutput::fromJSON(
}
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
auto [method, hashType] = methodAlgo();
auto [method, hashAlgo] = methodAlgo();
auto dof = DerivationOutput::CAFixed {
.ca = ContentAddress {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashAlgo),
},
};
if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
@ -1224,10 +1277,10 @@ DerivationOutput DerivationOutput::fromJSON(
else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
xpSettings.require(Xp::CaDerivations);
auto [method, hashType] = methodAlgo();
auto [method, hashAlgo] = methodAlgo();
return DerivationOutput::CAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
.hashAlgo = std::move(hashAlgo),
};
}
@ -1237,10 +1290,10 @@ DerivationOutput DerivationOutput::fromJSON(
else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
xpSettings.require(Xp::ImpureDerivations);
auto [method, hashType] = methodAlgo();
auto [method, hashAlgo] = methodAlgo();
return DerivationOutput::Impure {
.method = std::move(method),
.hashType = hashType,
.hashAlgo = hashAlgo,
};
}
@ -1250,7 +1303,7 @@ DerivationOutput DerivationOutput::fromJSON(
}
nlohmann::json Derivation::toJSON(const Store & store) const
nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
{
nlohmann::json res = nlohmann::json::object();
@ -1303,7 +1356,7 @@ nlohmann::json Derivation::toJSON(const Store & store) const
Derivation Derivation::fromJSON(
const Store & store,
const StoreDirConfig & store,
const nlohmann::json & json,
const ExperimentalFeatureSettings & xpSettings)
{

36
src/libstore/derivations.hh
View file

@ -17,7 +17,7 @@
namespace nix {
class Store;
struct StoreDirConfig;
/* Abstract syntax of derivations. */
@ -55,7 +55,7 @@ struct DerivationOutput
* @param drvName The name of the derivation this is an output of, without the `.drv`.
* @param outputName The name of this output.
*/
StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
GENERATE_CMP(CAFixed, me->ca);
};
@ -75,9 +75,9 @@ struct DerivationOutput
/**
* How the serialization will be hashed
*/
HashType hashType;
HashAlgorithm hashAlgo;
GENERATE_CMP(CAFloating, me->method, me->hashType);
GENERATE_CMP(CAFloating, me->method, me->hashAlgo);
};
/**
@ -102,9 +102,9 @@ struct DerivationOutput
/**
* How the serialization will be hashed
*/
HashType hashType;
HashAlgorithm hashAlgo;
GENERATE_CMP(Impure, me->method, me->hashType);
GENERATE_CMP(Impure, me->method, me->hashAlgo);
};
typedef std::variant<
@ -132,17 +132,17 @@ struct DerivationOutput
* the safer interface provided by
* BasicDerivation::outputsAndOptPaths
*/
std::optional<StorePath> path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
std::optional<StorePath> path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
nlohmann::json toJSON(
const Store & store,
const StoreDirConfig & store,
std::string_view drvName,
OutputNameView outputName) const;
/**
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivationOutput fromJSON(
const Store & store,
const StoreDirConfig & store,
std::string_view drvName,
OutputNameView outputName,
const nlohmann::json & json,
@ -304,7 +304,7 @@ struct BasicDerivation
* augmented with knowledge of the Store paths they would be written
* into.
*/
DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const;
static std::string_view nameFromPath(const StorePath & storePath);
@ -318,6 +318,8 @@ struct BasicDerivation
me->name);
};
class Store;
struct Derivation : BasicDerivation
{
/**
@ -328,7 +330,7 @@ struct Derivation : BasicDerivation
/**
* Print a derivation.
*/
std::string unparse(const Store & store, bool maskOutputs,
std::string unparse(const StoreDirConfig & store, bool maskOutputs,
DerivedPathMap<StringSet>::ChildNode::Map * actualInputs = nullptr) const;
/**
@ -340,7 +342,7 @@ struct Derivation : BasicDerivation
* 2. Input placeholders are replaced with realized input store
* paths.
*/
std::optional<BasicDerivation> tryResolve(Store & store) const;
std::optional<BasicDerivation> tryResolve(Store & store, Store * evalStore = nullptr) const;
/**
* Like the above, but instead of querying the Nix database for
@ -365,9 +367,9 @@ struct Derivation : BasicDerivation
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
nlohmann::json toJSON(const Store & store) const;
nlohmann::json toJSON(const StoreDirConfig & store) const;
static Derivation fromJSON(
const Store & store,
const StoreDirConfig & store,
const nlohmann::json & json,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
@ -391,7 +393,7 @@ StorePath writeDerivation(Store & store,
* Read a derivation from a file.
*/
Derivation parseDerivation(
const Store & store,
const StoreDirConfig & store,
std::string && s,
std::string_view name,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
@ -493,8 +495,8 @@ extern Sync<DrvHashes> drvHashes;
struct Source;
struct Sink;
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv);
/**
* This creates an opaque and almost certainly unique string

47
src/libstore/derived-path.cc
View file

@ -1,4 +1,5 @@
#include "derived-path.hh"
#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
@ -11,9 +12,9 @@ namespace nix {
bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
{ \
const MY_TYPE* me = this; \
auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
auto fields1 = std::tie(*me->drvPath, me->FIELD); \
me = &other; \
auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
auto fields2 = std::tie(*me->drvPath, me->FIELD); \
return fields1 COMPARATOR fields2; \
}
#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
@ -21,18 +22,14 @@ namespace nix {
CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
#define FIELD_TYPE std::string
CMP(SingleDerivedPath, SingleDerivedPathBuilt, output)
#undef FIELD_TYPE
#define FIELD_TYPE OutputsSpec
CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
#undef FIELD_TYPE
#undef CMP
#undef CMP_ONE
nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const
nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const
{
return store.printStorePath(path);
}
@ -86,50 +83,50 @@ nlohmann::json DerivedPath::toJSON(Store & store) const
}, raw());
}
std::string DerivedPath::Opaque::to_string(const Store & store) const
std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const
{
return store.printStorePath(path);
}
std::string SingleDerivedPath::Built::to_string(const Store & store) const
std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const
{
return drvPath->to_string(store) + "^" + output;
}
std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const
std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const
{
return drvPath->to_string(store) + "!" + output;
}
std::string DerivedPath::Built::to_string(const Store & store) const
std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const
{
return drvPath->to_string(store)
+ '^'
+ outputs.to_string();
}
std::string DerivedPath::Built::to_string_legacy(const Store & store) const
std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const
{
return drvPath->to_string_legacy(store)
+ "!"
+ outputs.to_string();
}
std::string SingleDerivedPath::to_string(const Store & store) const
std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
raw());
}
std::string DerivedPath::to_string(const Store & store) const
std::string DerivedPath::to_string(const StoreDirConfig & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
raw());
}
std::string SingleDerivedPath::to_string_legacy(const Store & store) const
std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const
{
return std::visit(overloaded {
[&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); },
@ -137,7 +134,7 @@ std::string SingleDerivedPath::to_string_legacy(const Store & store) const
}, this->raw());
}
std::string DerivedPath::to_string_legacy(const Store & store) const
std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const
{
return std::visit(overloaded {
[&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
@ -146,7 +143,7 @@ std::string DerivedPath::to_string_legacy(const Store & store) const
}
DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s)
DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s)
{
return {store.parseStorePath(s)};
}
@ -166,7 +163,7 @@ void drvRequireExperiment(
}
SingleDerivedPath::Built SingleDerivedPath::Built::parse(
const Store & store, ref<SingleDerivedPath> drv,
const StoreDirConfig & store, ref<SingleDerivedPath> drv,
OutputNameView output,
const ExperimentalFeatureSettings & xpSettings)
{
@ -178,7 +175,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
}
DerivedPath::Built DerivedPath::Built::parse(
const Store & store, ref<SingleDerivedPath> drv,
const StoreDirConfig & store, ref<SingleDerivedPath> drv,
OutputNameView outputsS,
const ExperimentalFeatureSettings & xpSettings)
{
@ -190,7 +187,7 @@ DerivedPath::Built DerivedPath::Built::parse(
}
static SingleDerivedPath parseWithSingle(
const Store & store, std::string_view s, std::string_view separator,
const StoreDirConfig & store, std::string_view s, std::string_view separator,
const ExperimentalFeatureSettings & xpSettings)
{
size_t n = s.rfind(separator);
@ -207,7 +204,7 @@ static SingleDerivedPath parseWithSingle(
}
SingleDerivedPath SingleDerivedPath::parse(
const Store & store,
const StoreDirConfig & store,
std::string_view s,
const ExperimentalFeatureSettings & xpSettings)
{
@ -215,7 +212,7 @@ SingleDerivedPath SingleDerivedPath::parse(
}
SingleDerivedPath SingleDerivedPath::parseLegacy(
const Store & store,
const StoreDirConfig & store,
std::string_view s,
const ExperimentalFeatureSettings & xpSettings)
{
@ -223,7 +220,7 @@ SingleDerivedPath SingleDerivedPath::parseLegacy(
}
static DerivedPath parseWith(
const Store & store, std::string_view s, std::string_view separator,
const StoreDirConfig & store, std::string_view s, std::string_view separator,
const ExperimentalFeatureSettings & xpSettings)
{
size_t n = s.rfind(separator);
@ -240,7 +237,7 @@ static DerivedPath parseWith(
}
DerivedPath DerivedPath::parse(
const Store & store,
const StoreDirConfig & store,
std::string_view s,
const ExperimentalFeatureSettings & xpSettings)
{
@ -248,7 +245,7 @@ DerivedPath DerivedPath::parse(
}
DerivedPath DerivedPath::parseLegacy(
const Store & store,
const StoreDirConfig & store,
std::string_view s,
const ExperimentalFeatureSettings & xpSettings)
{

37
src/libstore/derived-path.hh
View file

@ -12,6 +12,9 @@
namespace nix {
struct StoreDirConfig;
// TODO stop needing this, `toJSON` below should be pure
class Store;
/**
@ -24,9 +27,9 @@ class Store;
struct DerivedPathOpaque {
StorePath path;
std::string to_string(const Store & store) const;
static DerivedPathOpaque parse(const Store & store, std::string_view);
nlohmann::json toJSON(const Store & store) const;
std::string to_string(const StoreDirConfig & store) const;
static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view);
nlohmann::json toJSON(const StoreDirConfig & store) const;
GENERATE_CMP(DerivedPathOpaque, me->path);
};
@ -59,18 +62,18 @@ struct SingleDerivedPathBuilt {
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
std::string to_string(const StoreDirConfig & store) const;
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
std::string to_string_legacy(const StoreDirConfig & store) const;
/**
* The caller splits on the separator, so it works for both variants.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static SingleDerivedPathBuilt parse(
const Store & store, ref<SingleDerivedPath> drvPath,
const StoreDirConfig & store, ref<SingleDerivedPath> drvPath,
OutputNameView outputs,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;
@ -120,18 +123,18 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
std::string to_string(const StoreDirConfig & store) const;
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
std::string to_string_legacy(const StoreDirConfig & store) const;
/**
* Uses `^` as the separator
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static SingleDerivedPath parse(
const Store & store,
const StoreDirConfig & store,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
@ -140,7 +143,7 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static SingleDerivedPath parseLegacy(
const Store & store,
const StoreDirConfig & store,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;
@ -182,18 +185,18 @@ struct DerivedPathBuilt {
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
std::string to_string(const StoreDirConfig & store) const;
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
std::string to_string_legacy(const StoreDirConfig & store) const;
/**
* The caller splits on the separator, so it works for both variants.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivedPathBuilt parse(
const Store & store, ref<SingleDerivedPath>,
const StoreDirConfig & store, ref<SingleDerivedPath>,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;
@ -242,18 +245,18 @@ struct DerivedPath : _DerivedPathRaw {
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
std::string to_string(const StoreDirConfig & store) const;
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
std::string to_string_legacy(const StoreDirConfig & store) const;
/**
* Uses `^` as the separator
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivedPath parse(
const Store & store,
const StoreDirConfig & store,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
@ -262,7 +265,7 @@ struct DerivedPath : _DerivedPathRaw {
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivedPath parseLegacy(
const Store & store,
const StoreDirConfig & store,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);

8
src/libstore/downstream-placeholder.cc
View file

@ -5,7 +5,7 @@ namespace nix {
std::string DownstreamPlaceholder::render() const
{
return "/" + hash.to_string(HashFormat::Base32, false);
return "/" + hash.to_string(HashFormat::Nix32, false);
}
@ -19,7 +19,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
return DownstreamPlaceholder {
hashString(htSHA256, clearText)
hashString(HashAlgorithm::SHA256, clearText)
};
}
@ -31,10 +31,10 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
xpSettings.require(Xp::DynamicDerivations);
auto compressed = compressHash(placeholder.hash, 20);
auto clearText = "nix-computed-output:"
+ compressed.to_string(HashFormat::Base32, false)
+ compressed.to_string(HashFormat::Nix32, false)
+ ":" + std::string { outputName };
return DownstreamPlaceholder {
hashString(htSHA256, clearText)
hashString(HashAlgorithm::SHA256, clearText)
};
}

7
src/libstore/dummy-store.cc
View file

@ -58,13 +58,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
RepairFlag repair, CheckSigsFlag checkSigs) override
{ unsupported("addToStore"); }
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override
{ unsupported("addTextToStore"); }
void narFromPath(const StorePath & path, Sink & sink) override
{ unsupported("narFromPath"); }

8
src/libstore/export-import.cc
View file

@ -30,7 +30,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
{
auto info = queryPathInfo(path);
HashSink hashSink(htSHA256);
HashSink hashSink(HashAlgorithm::SHA256);
TeeSink teeSink(sink, hashSink);
narFromPath(path, teeSink);
@ -39,9 +39,9 @@ void Store::exportPath(const StorePath & path, Sink & sink)
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashSink.currentHash().first;
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
if (hash != info->narHash && info->narHash != Hash(info->narHash.algo))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true));
printStorePath(path), info->narHash.to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true));
teeSink
<< exportMagic
@ -79,7 +79,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
auto references = CommonProto::Serialise<StorePathSet>::read(*this,
CommonProto::ReadConn { .from = source });
auto deriver = readString(source);
auto narHash = hashString(htSHA256, saved.s);
auto narHash = hashString(HashAlgorithm::SHA256, saved.s);
ValidPathInfo info { path, narHash };
if (deriver != "")

11
src/libstore/gc.cc
View file

@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target)
void LocalStore::addIndirectRoot(const Path & path)
{
std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false);
std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false);
Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
makeSymlink(realRoot, path);
}
@ -330,9 +330,7 @@ typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots
static void readProcLink(const std::string & file, UncheckedRoots & roots)
{
/* 64 is the starting buffer size gnu readlink uses... */
auto bufsiz = ssize_t{64};
try_again:
constexpr auto bufsiz = PATH_MAX;
char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) {
@ -341,10 +339,7 @@ try_again:
throw SysError("reading symlink");
}
if (res == bufsiz) {
if (SSIZE_MAX / 2 < bufsiz)
throw Error("stupidly long symlink");
bufsiz *= 2;
goto try_again;
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
}
if (res > 0 && buf[0] == '/')
roots[std::string(static_cast<char *>(buf), res)]

Some files were not shown because too many files have changed in this diff Show more