1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-27 08:31:16 +02:00

Merge pull request #9177 from edolstra/input-accessors

Backport FSInputAccessor and MemoryInputAccessor from lazy-trees
This commit is contained in:
Eelco Dolstra 2023-10-23 11:42:04 +02:00 committed by GitHub
commit 955bbe53c5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 861 additions and 312 deletions

View file

@ -132,7 +132,7 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
if (colon == std::string::npos) fail();
std::string filename(fn, 0, colon);
auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
return {CanonPath(fn.substr(0, colon)), lineno};
return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
} catch (std::invalid_argument & e) {
fail();
abort();

View file

@ -12,6 +12,8 @@
#include "function-trace.hh"
#include "profiles.hh"
#include "print.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
#include <algorithm>
#include <chrono>
@ -503,7 +505,17 @@ EvalState::EvalState(
, sOutputSpecified(symbols.create("outputSpecified"))
, repair(NoRepair)
, emptyBindings(0)
, derivationInternal(rootPath(CanonPath("/builtin/derivation.nix")))
, rootFS(makeFSInputAccessor(CanonPath::root))
, corepkgsFS(makeMemoryInputAccessor())
, internalFS(makeMemoryInputAccessor())
, derivationInternal{corepkgsFS->addFile(
CanonPath("derivation-internal.nix"),
#include "primops/derivation.nix.gen.hh"
)}
, callFlakeInternal{internalFS->addFile(
CanonPath("call-flake.nix"),
#include "flake/call-flake.nix.gen.hh"
)}
, store(store)
, buildStore(buildStore ? buildStore : store)
, debugRepl(nullptr)
@ -555,6 +567,11 @@ EvalState::EvalState(
}
}
corepkgsFS->addFile(
CanonPath("fetchurl.nix"),
#include "fetchurl.nix.gen.hh"
);
createBaseEnv();
}
@ -585,6 +602,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
SourcePath EvalState::checkSourcePath(const SourcePath & path_)
{
// Don't check non-rootFS accessors, they're in a different namespace.
if (path_.accessor != ref<InputAccessor>(rootFS)) return path_;
if (!allowedPaths) return path_;
auto i = resolvedPaths.find(path_.path.abs());
@ -599,8 +619,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
*/
Path abspath = canonPath(path_.path.abs());
if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath);
for (auto & i : *allowedPaths) {
if (isDirOrInDir(abspath, i)) {
found = true;
@ -617,7 +635,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
/* Resolve symlinks. */
debug("checking access to '%s'", abspath);
SourcePath path = CanonPath(canonPath(abspath, true));
SourcePath path = rootPath(CanonPath(canonPath(abspath, true)));
for (auto & i : *allowedPaths) {
if (isDirOrInDir(path.path.abs(), i)) {
@ -649,12 +667,12 @@ void EvalState::checkURI(const std::string & uri)
/* If the URI is a path, then check it against allowedPaths as
well. */
if (hasPrefix(uri, "/")) {
checkSourcePath(CanonPath(uri));
checkSourcePath(rootPath(CanonPath(uri)));
return;
}
if (hasPrefix(uri, "file://")) {
checkSourcePath(CanonPath(std::string(uri, 7)));
checkSourcePath(rootPath(CanonPath(std::string(uri, 7))));
return;
}
@ -950,7 +968,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context)
void Value::mkPath(const SourcePath & path)
{
mkPath(makeImmutableString(path.path.abs()));
mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
}
@ -1165,24 +1183,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
if (!e)
e = parseExprFromFile(checkSourcePath(resolvedPath));
cacheFile(path, resolvedPath, e, v, mustBeTrivial);
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial)
{
fileParseCache[resolvedPath] = e;
try {
@ -1211,6 +1211,13 @@ void EvalState::cacheFile(
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::eval(Expr * e, Value & v)
{
e->eval(*this, baseEnv, v);
@ -2037,7 +2044,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
else if (firstType == nPath) {
if (!context.empty())
state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>();
v.mkPath(CanonPath(canonPath(str())));
v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
} else
v.mkStringMove(c_str(), context);
}
@ -2236,7 +2243,7 @@ BackedStringView EvalState::coerceToString(
!canonicalizePath && !copyToStore
? // FIXME: hack to preserve path literals that end in a
// slash, as in /foo/${x}.
v._path
v._path.path
: copyToStore
? store->printStorePath(copyPathToStore(context, v.path()))
: std::string(v.path().path.abs());
@ -2310,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
auto dstPath = i != srcToStore.end()
? i->second
: [&]() {
auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair);
auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
allowPath(dstPath);
srcToStore.insert_or_assign(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@ -2326,10 +2333,34 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
{
try {
forceValue(v, pos);
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
/* Handle path values directly, without coercing to a string. */
if (v.type() == nPath)
return v.path();
/* Similarly, handle __toString where the result may be a path
value. */
if (v.type() == nAttrs) {
auto i = v.attrs->find(sToString);
if (i != v.attrs->end()) {
Value v1;
callFunction(*i->value, v, v1, pos);
return coerceToPath(pos, v1, context, errorCtx);
}
}
/* Any other value should be coercable to a string, interpreted
relative to the root filesystem. */
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (path == "" || path[0] != '/')
error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
return CanonPath(path);
return rootPath(CanonPath(path));
}
@ -2429,7 +2460,10 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
return v1.string_view().compare(v2.string_view()) == 0;
case nPath:
return strcmp(v1._path, v2._path) == 0;
return
// FIXME: compare accessors by their fingerprint.
v1._path.accessor == v2._path.accessor
&& strcmp(v1._path.path, v2._path.path) == 0;
case nNull:
return true;

View file

@ -24,6 +24,8 @@ class EvalState;
class StorePath;
struct SingleDerivedPath;
enum RepairFlag : bool;
struct FSInputAccessor;
struct MemoryInputAccessor;
/**
@ -211,8 +213,26 @@ public:
Bindings emptyBindings;
/**
* The accessor for the root filesystem.
*/
const ref<FSInputAccessor> rootFS;
/**
* The in-memory filesystem for <nix/...> paths.
*/
const ref<MemoryInputAccessor> corepkgsFS;
/**
* In-memory filesystem for internal, non-user-callable Nix
* expressions like call-flake.nix.
*/
const ref<MemoryInputAccessor> internalFS;
const SourcePath derivationInternal;
const SourcePath callFlakeInternal;
/**
* Store used to materialise .drv files.
*/
@ -223,7 +243,6 @@ public:
*/
const ref<Store> buildStore;
RootValue vCallFlake = nullptr;
RootValue vImportedDrvToDerivation = nullptr;
/**
@ -405,16 +424,6 @@ public:
*/
void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
/**
* Like `evalFile`, but with an already parsed expression.
*/
void cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial = false);
void resetFileCache();
/**
@ -424,7 +433,7 @@ public:
SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/**
* Try to resolve a search path value (not the optinal key part)
* Try to resolve a search path value (not the optional key part)
*
* If the specified search path element is a URI, download it.
*
@ -829,8 +838,6 @@ struct InvalidPathError : EvalError
#endif
};
static const std::string corepkgsPrefix{"/__corepkgs__/"};
template<class ErrorType>
void ErrorBuilder::debugThrow()
{

View file

@ -223,9 +223,9 @@ static Flake getFlake(
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
Value vInfo;
state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack
state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack
expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1));
expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1));
if (auto description = vInfo.attrs->get(state.sDescription)) {
expectType(state, nString, *description->value, description->pos);
@ -737,14 +737,10 @@ void callFlake(EvalState & state,
vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
if (!state.vCallFlake) {
state.vCallFlake = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh"
, CanonPath::root), **state.vCallFlake);
}
auto vCallFlake = state.allocValue();
state.evalFile(state.callFlakeInternal, *vCallFlake);
state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
}

View file

@ -20,7 +20,6 @@ MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError);
MakeError(RestrictedPathError, Error);
/**
* Position objects.
@ -200,9 +199,13 @@ struct ExprString : Expr
struct ExprPath : Expr
{
ref<InputAccessor> accessor;
std::string s;
Value v;
ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); };
ExprPath(ref<InputAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
{
v.mkPath(&*accessor, this->s.c_str());
}
Value * maybeThunk(EvalState & state, Env & env) override;
COMMON_METHODS
};

View file

@ -520,7 +520,7 @@ path_start
/* add back in the trailing '/' to the first segment */
if ($1.p[$1.l-1] == '/' && $1.l > 1)
path += "/";
$$ = new ExprPath(std::move(path));
$$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
}
| HPATH {
if (evalSettings.pureEval) {
@ -530,7 +530,7 @@ path_start
);
}
Path path(getHome() + std::string($1.p + 1, $1.l - 1));
$$ = new ExprPath(std::move(path));
$$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
}
;
@ -649,6 +649,8 @@ formal
#include "tarball.hh"
#include "store-api.hh"
#include "flake/flake.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
namespace nix {
@ -756,11 +758,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
if (pathExists(res)) return CanonPath(canonPath(res));
if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
}
if (hasPrefix(path, "nix/"))
return CanonPath(concatStrings(corepkgsPrefix, path.substr(4)));
return {corepkgsFS, CanonPath(path.substr(3))};
debugThrow(ThrownError({
.msg = hintfmt(evalSettings.pureEval

View file

@ -1,10 +1,11 @@
#include "eval.hh"
#include "fs-input-accessor.hh"
namespace nix {
SourcePath EvalState::rootPath(CanonPath path)
{
return path;
return {rootFS, std::move(path)};
}
}

View file

@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
try {
StringMap rewrites = state.realiseContext(context);
auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context)));
if (!context.empty()) {
auto rewrites = state.realiseContext(context);
auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
return {path.accessor, CanonPath(realPath)};
}
return flags.checkForPureEval
? state.checkSourcePath(realPath)
: realPath;
? state.checkSourcePath(path)
: path;
} catch (Error & e) {
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
throw;
@ -202,7 +204,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "imported-drv-to-derivation.nix.gen.hh"
, CanonPath::root), **state.vImportedDrvToDerivation);
, state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation);
}
state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh");
@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
}
else if (path2 == corepkgsPrefix + "fetchurl.nix") {
state.eval(state.parseExprFromString(
#include "fetchurl.nix.gen.hh"
, CanonPath::root), v);
}
else {
if (!vScope)
state.evalFile(path, v);
@ -599,7 +595,10 @@ struct CompareValues
case nString:
return v1->string_view().compare(v2->string_view()) < 0;
case nPath:
return strcmp(v1->_path, v2->_path) < 0;
// Note: we don't take the accessor into account
// since it's not obvious how to compare them in a
// reproducible way.
return strcmp(v1->_path.path, v2->_path.path) < 0;
case nList:
// Lexicographic comparison
for (size_t i = 0;; i++) {
@ -1493,7 +1492,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
}));
NixStringContext context;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
/* Resolve symlinks in path, unless path itself is a symlink
directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */
@ -2211,7 +2210,7 @@ static void addPath(
path = evalSettings.pureEval && expectedHash
? path
: state.checkSourcePath(CanonPath(path)).path.abs();
: state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs();
PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path);
@ -2244,9 +2243,7 @@ static void addPath(
});
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
StorePath dstPath = settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs);
auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
if (expectedHash && expectedStorePath != dstPath)
state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
state.allowAndSetStorePathString(dstPath, v);
@ -2263,7 +2260,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
{
NixStringContext context;
auto path = state.coerceToPath(pos, *args[1], context,
"while evaluating the second argument (the path to filter) passed to builtins.filterSource");
"while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
}
@ -4545,12 +4542,7 @@ void EvalState::createBaseEnv()
/* Note: we have to initialize the 'derivation' constant *after*
building baseEnv/staticBaseEnv because it uses 'builtins'. */
char code[] =
#include "primops/derivation.nix.gen.hh"
// the parser needs two NUL bytes as terminators; one of them
// is implied by being a C string.
"\0";
eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
evalFile(derivationInternal, *vDerivation);
}

View file

@ -310,7 +310,7 @@ namespace nix {
ASSERT_TRACE2("storePath true",
TypeError,
hintfmt("cannot coerce %s to a string", "a Boolean"),
hintfmt("while evaluating the first argument passed to builtins.storePath"));
hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
}
@ -378,12 +378,12 @@ namespace nix {
ASSERT_TRACE2("filterSource [] []",
TypeError,
hintfmt("cannot coerce %s to a string", "a list"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] \"foo\"",
EvalError,
hintfmt("string '%s' doesn't represent an absolute path", "foo"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] ./.",
TypeError,

View file

@ -62,7 +62,7 @@ namespace nix {
// not supported by store 'dummy'" thrown in the test body.
TEST_F(JSONValueTest, DISABLED_Path) {
Value v;
v.mkPath("test");
v.mkPath(state.rootPath(CanonPath("/test")));
ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
}
} /* namespace nix */

View file

@ -103,14 +103,17 @@ namespace nix {
}
MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) {
if (arg.type() != nPath) {
*result_listener << "Expected a path got " << arg.type();
return false;
} else if (std::string_view(arg._path) != p) {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str();
if (arg.type() != nPath) {
*result_listener << "Expected a path got " << arg.type();
return false;
} else {
auto path = arg.path();
if (path.path != CanonPath(p)) {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path;
return false;
}
return true;
}
return true;
}

View file

@ -189,7 +189,12 @@ public:
const char * c_str;
const char * * context; // must be in sorted order
} string;
const char * _path;
struct {
InputAccessor * accessor;
const char * path;
} _path;
Bindings * attrs;
struct {
size_t size;
@ -286,11 +291,12 @@ public:
void mkPath(const SourcePath & path);
inline void mkPath(const char * path)
inline void mkPath(InputAccessor * accessor, const char * path)
{
clearValue();
internalType = tPath;
_path = path;
_path.accessor = accessor;
_path.path = path;
}
inline void mkNull()
@ -437,7 +443,10 @@ public:
SourcePath path() const
{
assert(internalType == tPath);
return SourcePath{CanonPath(_path)};
return SourcePath {
.accessor = ref(_path.accessor->shared_from_this()),
.path = CanonPath(CanonPath::unchecked_t(), _path.path)
};
}
std::string_view string_view() const