From b48e64162a77f09345c94826d9799ff578b2981b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 19 Dec 2022 13:45:06 +0100 Subject: [PATCH] Add builtins.filterPath This is like builtins.{filterSource,path}, but returns a virtual path that applies the filter lazily, rather than copying the result to the Nix store. Thus filterPath can be composed. --- src/libexpr/eval.hh | 7 ++ src/libexpr/primops.cc | 52 +++++---- src/libexpr/primops/filterPath.cc | 174 ++++++++++++++++++++++++++++++ tests/flakes/tree-operators.sh | 48 +++++++++ tests/lang/lib.nix | 2 + tests/local.mk | 1 + 6 files changed, 263 insertions(+), 21 deletions(-) create mode 100644 src/libexpr/primops/filterPath.cc create mode 100644 tests/flakes/tree-operators.sh diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index bb20f2641..8e981ad5f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -525,6 +525,13 @@ public: */ [[nodiscard]] StringMap realiseContext(const PathSet & context); + /* Call the binary path filter predicate used builtins.path etc. */ + bool callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos); + private: unsigned long nrEnvs = 0; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a58ea6151..e40f3480c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1974,6 +1974,30 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); +bool EvalState::callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos) +{ + auto st = path.lstat(); + + /* Call the filter function. The first argument is the path, the + second is a string indicating the type of the file. */ + Value arg1; + arg1.mkString(pathArg); + + Value arg2; + // assert that type is not "unknown" + arg2.mkString(fileTypeToString(st.type)); + + Value * args []{&arg1, &arg2}; + Value res; + callFunction(*filterFun, 2, args, res, pos); + + return forceBool(res, pos); +} + static void addPath( EvalState & state, const PosIdx pos, @@ -2010,26 +2034,11 @@ static void addPath( #endif std::unique_ptr filter; - if (filterFun) filter = std::make_unique([&](const Path & p) { - SourcePath path2{path.accessor, CanonPath(p)}; - - auto st = path2.lstat(); - - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - arg1.mkString(path2.path.abs()); - - Value arg2; - // assert that type is not "unknown" - arg2.mkString(fileTypeToString(st.type)); - - Value * args []{&arg1, &arg2}; - Value res; - state.callFunction(*filterFun, 2, args, res, pos); - - return state.forceBool(res, pos); - }); + if (filterFun) + filter = std::make_unique([&](const Path & p) { + auto p2 = CanonPath(p); + return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos); + }); std::optional expectedStorePath; if (expectedHash) @@ -2130,7 +2139,6 @@ static RegisterPrimOp primop_filterSource({ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); std::optional path; std::string name; Value * filterFun = nullptr; @@ -2138,6 +2146,8 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value std::optional expectedHash; PathSet context; + state.forceAttrs(*args[0], pos); + for (auto & attr : *args[0]->attrs) { auto n = state.symbols[attr.name]; if (n == "path") diff --git a/src/libexpr/primops/filterPath.cc b/src/libexpr/primops/filterPath.cc new file mode 100644 index 000000000..c2db5b029 --- /dev/null +++ b/src/libexpr/primops/filterPath.cc @@ -0,0 +1,174 @@ +#include "primops.hh" + +namespace nix { + +struct FilteringInputAccessor : InputAccessor +{ + EvalState & state; + PosIdx pos; + ref next; + CanonPath prefix; + Value * filterFun; + + std::map cache; + + FilteringInputAccessor(EvalState & state, PosIdx pos, const SourcePath & src, Value * filterFun) + : state(state) + , pos(pos) + , next(src.accessor) + , prefix(src.path) + , filterFun(filterFun) + { + } + + std::string readFile(const CanonPath & path) override + { + checkAccess(path); + return next->readFile(prefix + path); + } + + bool pathExists(const CanonPath & path) override + { + checkAccess(path); + return next->pathExists(prefix + path); + } + + Stat lstat(const CanonPath & path) override + { + checkAccess(path); + return next->lstat(prefix + path); + } + + DirEntries readDirectory(const CanonPath & path) override + { + checkAccess(path); + DirEntries entries; + for (auto & entry : next->readDirectory(prefix + path)) { + if (isAllowed(path + entry.first)) + entries.insert(std::move(entry)); + } + return entries; + } + + std::string readLink(const CanonPath & path) override + { + checkAccess(path); + return next->readLink(prefix + path); + } + + void checkAccess(const CanonPath & path) + { + if (!isAllowed(path)) + throw Error("access to path '%s' has been filtered out", showPath(path)); + } + + bool isAllowed(const CanonPath & path) + { + auto i = cache.find(path); + if (i != cache.end()) return i->second; + auto res = isAllowedUncached(path); + cache.emplace(path, res); + return res; + } + + bool isAllowedUncached(const CanonPath & path) + { + if (!path.isRoot() && !isAllowed(*path.parent())) return false; + // Note that unlike 'builtins.{path,filterSource}', we don't + // pass the prefix to the filter function. + return state.callPathFilter(filterFun, {next, prefix + path}, path.abs(), pos); + } + + std::string showPath(const CanonPath & path) override + { + return next->showPath(prefix + path); + } +}; + +static void prim_filterPath(EvalState & state, PosIdx pos, Value * * args, Value & v) +{ + std::optional path; + Value * filterFun = nullptr; + PathSet context; + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + auto n = state.symbols[attr.name]; + if (n == "path") + path.emplace(state.coerceToPath(attr.pos, *attr.value, context)); + else if (n == "filter") { + state.forceValue(*attr.value, pos); + filterFun = attr.value; + } + else + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("unsupported argument '%1%' to 'filterPath'", state.symbols[attr.name]), + .errPos = state.positions[attr.pos] + })); + } + + if (!path) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("'path' required"), + .errPos = state.positions[pos] + })); + + if (!filterFun) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("'filter' required"), + .errPos = state.positions[pos] + })); + + if (!context.empty()) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("'path' argument to 'filterPath' cannot have a context"), + .errPos = state.positions[pos] + })); + + auto accessor = make_ref(state, pos, *path, filterFun); + + state.registerAccessor(accessor); + + v.mkPath(accessor->root()); +} + +static RegisterPrimOp primop_filterPath({ + .name = "__filterPath", + .args = {"args"}, + .doc = R"( + This function lets you filter out files from a path. It takes a + path and a predicate function, and returns a new path from which + every file has been removed for which the predicate function + returns `false`. + + For example, the following filters out all regular files in + `./doc` that don't end with the extension `.md`: + + ```nix + builtins.filterPath { + path = ./doc; + filter = + path: type: + (type != "regular" || hasSuffix ".md" path); + } + ``` + + The filter function is called for all files in `path`. It takes + two arguments. The first is a string that represents the path of + the file to be filtered, relative to `path` (i.e. it does *not* + contain `./doc` in the example above). The second is the file + type, which can be one of `regular`, `directory` or `symlink`. + + Note that unlike `builtins.filterSource` and `builtins.path`, + this function does not copy the result to the Nix store. Rather, + the result is a virtual path that lazily applies the filter + predicate. The result will only be copied to the Nix store if + needed (e.g. if used in a derivation attribute like `src = + builtins.filterPath { ... }`). + )", + .fun = prim_filterPath, + .experimentalFeature = Xp::Flakes, +}); + +} diff --git a/tests/flakes/tree-operators.sh b/tests/flakes/tree-operators.sh new file mode 100644 index 000000000..9298afda2 --- /dev/null +++ b/tests/flakes/tree-operators.sh @@ -0,0 +1,48 @@ +source ./common.sh + +flake1Dir=$TEST_ROOT/flake1 + +mkdir -p $flake1Dir + +pwd=$(pwd) + +cat > $flake1Dir/flake.nix <