mirror of
https://github.com/NixOS/nix
synced 2025-07-03 06:11:46 +02:00
Add builtins.filterPath
This is like builtins.{filterSource,path}, but returns a virtual path that applies the filter lazily, rather than copying the result to the Nix store. Thus filterPath can be composed.
This commit is contained in:
parent
15d2e0e63b
commit
b48e64162a
6 changed files with 263 additions and 21 deletions
|
@ -525,6 +525,13 @@ public:
|
||||||
*/
|
*/
|
||||||
[[nodiscard]] StringMap realiseContext(const PathSet & context);
|
[[nodiscard]] StringMap realiseContext(const PathSet & context);
|
||||||
|
|
||||||
|
/* Call the binary path filter predicate used builtins.path etc. */
|
||||||
|
bool callPathFilter(
|
||||||
|
Value * filterFun,
|
||||||
|
const SourcePath & path,
|
||||||
|
std::string_view pathArg,
|
||||||
|
PosIdx pos);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
unsigned long nrEnvs = 0;
|
unsigned long nrEnvs = 0;
|
||||||
|
|
|
@ -1974,6 +1974,30 @@ static RegisterPrimOp primop_toFile({
|
||||||
.fun = prim_toFile,
|
.fun = prim_toFile,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
bool EvalState::callPathFilter(
|
||||||
|
Value * filterFun,
|
||||||
|
const SourcePath & path,
|
||||||
|
std::string_view pathArg,
|
||||||
|
PosIdx pos)
|
||||||
|
{
|
||||||
|
auto st = path.lstat();
|
||||||
|
|
||||||
|
/* Call the filter function. The first argument is the path, the
|
||||||
|
second is a string indicating the type of the file. */
|
||||||
|
Value arg1;
|
||||||
|
arg1.mkString(pathArg);
|
||||||
|
|
||||||
|
Value arg2;
|
||||||
|
// assert that type is not "unknown"
|
||||||
|
arg2.mkString(fileTypeToString(st.type));
|
||||||
|
|
||||||
|
Value * args []{&arg1, &arg2};
|
||||||
|
Value res;
|
||||||
|
callFunction(*filterFun, 2, args, res, pos);
|
||||||
|
|
||||||
|
return forceBool(res, pos);
|
||||||
|
}
|
||||||
|
|
||||||
static void addPath(
|
static void addPath(
|
||||||
EvalState & state,
|
EvalState & state,
|
||||||
const PosIdx pos,
|
const PosIdx pos,
|
||||||
|
@ -2010,26 +2034,11 @@ static void addPath(
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
std::unique_ptr<PathFilter> filter;
|
std::unique_ptr<PathFilter> filter;
|
||||||
if (filterFun) filter = std::make_unique<PathFilter>([&](const Path & p) {
|
if (filterFun)
|
||||||
SourcePath path2{path.accessor, CanonPath(p)};
|
filter = std::make_unique<PathFilter>([&](const Path & p) {
|
||||||
|
auto p2 = CanonPath(p);
|
||||||
auto st = path2.lstat();
|
return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos);
|
||||||
|
});
|
||||||
/* Call the filter function. The first argument is the path,
|
|
||||||
the second is a string indicating the type of the file. */
|
|
||||||
Value arg1;
|
|
||||||
arg1.mkString(path2.path.abs());
|
|
||||||
|
|
||||||
Value arg2;
|
|
||||||
// assert that type is not "unknown"
|
|
||||||
arg2.mkString(fileTypeToString(st.type));
|
|
||||||
|
|
||||||
Value * args []{&arg1, &arg2};
|
|
||||||
Value res;
|
|
||||||
state.callFunction(*filterFun, 2, args, res, pos);
|
|
||||||
|
|
||||||
return state.forceBool(res, pos);
|
|
||||||
});
|
|
||||||
|
|
||||||
std::optional<StorePath> expectedStorePath;
|
std::optional<StorePath> expectedStorePath;
|
||||||
if (expectedHash)
|
if (expectedHash)
|
||||||
|
@ -2130,7 +2139,6 @@ static RegisterPrimOp primop_filterSource({
|
||||||
|
|
||||||
static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v)
|
static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v)
|
||||||
{
|
{
|
||||||
state.forceAttrs(*args[0], pos);
|
|
||||||
std::optional<SourcePath> path;
|
std::optional<SourcePath> path;
|
||||||
std::string name;
|
std::string name;
|
||||||
Value * filterFun = nullptr;
|
Value * filterFun = nullptr;
|
||||||
|
@ -2138,6 +2146,8 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
|
||||||
std::optional<Hash> expectedHash;
|
std::optional<Hash> expectedHash;
|
||||||
PathSet context;
|
PathSet context;
|
||||||
|
|
||||||
|
state.forceAttrs(*args[0], pos);
|
||||||
|
|
||||||
for (auto & attr : *args[0]->attrs) {
|
for (auto & attr : *args[0]->attrs) {
|
||||||
auto n = state.symbols[attr.name];
|
auto n = state.symbols[attr.name];
|
||||||
if (n == "path")
|
if (n == "path")
|
||||||
|
|
174
src/libexpr/primops/filterPath.cc
Normal file
174
src/libexpr/primops/filterPath.cc
Normal file
|
@ -0,0 +1,174 @@
|
||||||
|
#include "primops.hh"
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
struct FilteringInputAccessor : InputAccessor
|
||||||
|
{
|
||||||
|
EvalState & state;
|
||||||
|
PosIdx pos;
|
||||||
|
ref<InputAccessor> next;
|
||||||
|
CanonPath prefix;
|
||||||
|
Value * filterFun;
|
||||||
|
|
||||||
|
std::map<CanonPath, bool> cache;
|
||||||
|
|
||||||
|
FilteringInputAccessor(EvalState & state, PosIdx pos, const SourcePath & src, Value * filterFun)
|
||||||
|
: state(state)
|
||||||
|
, pos(pos)
|
||||||
|
, next(src.accessor)
|
||||||
|
, prefix(src.path)
|
||||||
|
, filterFun(filterFun)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string readFile(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
checkAccess(path);
|
||||||
|
return next->readFile(prefix + path);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool pathExists(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
checkAccess(path);
|
||||||
|
return next->pathExists(prefix + path);
|
||||||
|
}
|
||||||
|
|
||||||
|
Stat lstat(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
checkAccess(path);
|
||||||
|
return next->lstat(prefix + path);
|
||||||
|
}
|
||||||
|
|
||||||
|
DirEntries readDirectory(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
checkAccess(path);
|
||||||
|
DirEntries entries;
|
||||||
|
for (auto & entry : next->readDirectory(prefix + path)) {
|
||||||
|
if (isAllowed(path + entry.first))
|
||||||
|
entries.insert(std::move(entry));
|
||||||
|
}
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string readLink(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
checkAccess(path);
|
||||||
|
return next->readLink(prefix + path);
|
||||||
|
}
|
||||||
|
|
||||||
|
void checkAccess(const CanonPath & path)
|
||||||
|
{
|
||||||
|
if (!isAllowed(path))
|
||||||
|
throw Error("access to path '%s' has been filtered out", showPath(path));
|
||||||
|
}
|
||||||
|
|
||||||
|
bool isAllowed(const CanonPath & path)
|
||||||
|
{
|
||||||
|
auto i = cache.find(path);
|
||||||
|
if (i != cache.end()) return i->second;
|
||||||
|
auto res = isAllowedUncached(path);
|
||||||
|
cache.emplace(path, res);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool isAllowedUncached(const CanonPath & path)
|
||||||
|
{
|
||||||
|
if (!path.isRoot() && !isAllowed(*path.parent())) return false;
|
||||||
|
// Note that unlike 'builtins.{path,filterSource}', we don't
|
||||||
|
// pass the prefix to the filter function.
|
||||||
|
return state.callPathFilter(filterFun, {next, prefix + path}, path.abs(), pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string showPath(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
return next->showPath(prefix + path);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
static void prim_filterPath(EvalState & state, PosIdx pos, Value * * args, Value & v)
|
||||||
|
{
|
||||||
|
std::optional<SourcePath> path;
|
||||||
|
Value * filterFun = nullptr;
|
||||||
|
PathSet context;
|
||||||
|
|
||||||
|
state.forceAttrs(*args[0], pos);
|
||||||
|
|
||||||
|
for (auto & attr : *args[0]->attrs) {
|
||||||
|
auto n = state.symbols[attr.name];
|
||||||
|
if (n == "path")
|
||||||
|
path.emplace(state.coerceToPath(attr.pos, *attr.value, context));
|
||||||
|
else if (n == "filter") {
|
||||||
|
state.forceValue(*attr.value, pos);
|
||||||
|
filterFun = attr.value;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
state.debugThrowLastTrace(EvalError({
|
||||||
|
.msg = hintfmt("unsupported argument '%1%' to 'filterPath'", state.symbols[attr.name]),
|
||||||
|
.errPos = state.positions[attr.pos]
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!path)
|
||||||
|
state.debugThrowLastTrace(EvalError({
|
||||||
|
.msg = hintfmt("'path' required"),
|
||||||
|
.errPos = state.positions[pos]
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (!filterFun)
|
||||||
|
state.debugThrowLastTrace(EvalError({
|
||||||
|
.msg = hintfmt("'filter' required"),
|
||||||
|
.errPos = state.positions[pos]
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (!context.empty())
|
||||||
|
state.debugThrowLastTrace(EvalError({
|
||||||
|
.msg = hintfmt("'path' argument to 'filterPath' cannot have a context"),
|
||||||
|
.errPos = state.positions[pos]
|
||||||
|
}));
|
||||||
|
|
||||||
|
auto accessor = make_ref<FilteringInputAccessor>(state, pos, *path, filterFun);
|
||||||
|
|
||||||
|
state.registerAccessor(accessor);
|
||||||
|
|
||||||
|
v.mkPath(accessor->root());
|
||||||
|
}
|
||||||
|
|
||||||
|
static RegisterPrimOp primop_filterPath({
|
||||||
|
.name = "__filterPath",
|
||||||
|
.args = {"args"},
|
||||||
|
.doc = R"(
|
||||||
|
This function lets you filter out files from a path. It takes a
|
||||||
|
path and a predicate function, and returns a new path from which
|
||||||
|
every file has been removed for which the predicate function
|
||||||
|
returns `false`.
|
||||||
|
|
||||||
|
For example, the following filters out all regular files in
|
||||||
|
`./doc` that don't end with the extension `.md`:
|
||||||
|
|
||||||
|
```nix
|
||||||
|
builtins.filterPath {
|
||||||
|
path = ./doc;
|
||||||
|
filter =
|
||||||
|
path: type:
|
||||||
|
(type != "regular" || hasSuffix ".md" path);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The filter function is called for all files in `path`. It takes
|
||||||
|
two arguments. The first is a string that represents the path of
|
||||||
|
the file to be filtered, relative to `path` (i.e. it does *not*
|
||||||
|
contain `./doc` in the example above). The second is the file
|
||||||
|
type, which can be one of `regular`, `directory` or `symlink`.
|
||||||
|
|
||||||
|
Note that unlike `builtins.filterSource` and `builtins.path`,
|
||||||
|
this function does not copy the result to the Nix store. Rather,
|
||||||
|
the result is a virtual path that lazily applies the filter
|
||||||
|
predicate. The result will only be copied to the Nix store if
|
||||||
|
needed (e.g. if used in a derivation attribute like `src =
|
||||||
|
builtins.filterPath { ... }`).
|
||||||
|
)",
|
||||||
|
.fun = prim_filterPath,
|
||||||
|
.experimentalFeature = Xp::Flakes,
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
48
tests/flakes/tree-operators.sh
Normal file
48
tests/flakes/tree-operators.sh
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
source ./common.sh
|
||||||
|
|
||||||
|
flake1Dir=$TEST_ROOT/flake1
|
||||||
|
|
||||||
|
mkdir -p $flake1Dir
|
||||||
|
|
||||||
|
pwd=$(pwd)
|
||||||
|
|
||||||
|
cat > $flake1Dir/flake.nix <<EOF
|
||||||
|
{
|
||||||
|
inputs.docs = {
|
||||||
|
url = "path://$pwd/../../doc";
|
||||||
|
flake = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs = { self, docs }: with import ./lib.nix; {
|
||||||
|
|
||||||
|
mdOnly = builtins.filterPath {
|
||||||
|
path = docs;
|
||||||
|
filter =
|
||||||
|
path: type:
|
||||||
|
(type != "regular" || hasSuffix ".md" path);
|
||||||
|
};
|
||||||
|
|
||||||
|
noReleaseNotes = builtins.filterPath {
|
||||||
|
path = self.mdOnly + "/manual";
|
||||||
|
filter =
|
||||||
|
path: type:
|
||||||
|
assert !hasPrefix "/manual" path;
|
||||||
|
(builtins.baseNameOf path != "release-notes");
|
||||||
|
};
|
||||||
|
|
||||||
|
};
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
cp ../lang/lib.nix $flake1Dir/
|
||||||
|
|
||||||
|
nix build --out-link $TEST_ROOT/result $flake1Dir#mdOnly
|
||||||
|
[[ -e $TEST_ROOT/result/manual/src/quick-start.md ]]
|
||||||
|
[[ -e $TEST_ROOT/result/manual/src/release-notes ]]
|
||||||
|
(! find $TEST_ROOT/result/ -type f | grep -v '.md$')
|
||||||
|
find $TEST_ROOT/result/ -type f | grep release-notes
|
||||||
|
|
||||||
|
nix build --out-link $TEST_ROOT/result $flake1Dir#noReleaseNotes
|
||||||
|
[[ -e $TEST_ROOT/result/src/quick-start.md ]]
|
||||||
|
(! [[ -e $TEST_ROOT/result/src/release-notes ]])
|
||||||
|
(! find $TEST_ROOT/result/ -type f | grep release-notes)
|
|
@ -19,6 +19,8 @@ rec {
|
||||||
|
|
||||||
sum = foldl' (x: y: add x y) 0;
|
sum = foldl' (x: y: add x y) 0;
|
||||||
|
|
||||||
|
hasPrefix = pref: str: substring 0 (stringLength pref) str == pref;
|
||||||
|
|
||||||
hasSuffix = ext: fileName:
|
hasSuffix = ext: fileName:
|
||||||
let lenFileName = stringLength fileName;
|
let lenFileName = stringLength fileName;
|
||||||
lenExt = stringLength ext;
|
lenExt = stringLength ext;
|
||||||
|
|
|
@ -10,6 +10,7 @@ nix_tests = \
|
||||||
flakes/unlocked-override.sh \
|
flakes/unlocked-override.sh \
|
||||||
flakes/absolute-paths.sh \
|
flakes/absolute-paths.sh \
|
||||||
flakes/build-paths.sh \
|
flakes/build-paths.sh \
|
||||||
|
flakes/tree-operators.sh \
|
||||||
ca/gc.sh \
|
ca/gc.sh \
|
||||||
gc.sh \
|
gc.sh \
|
||||||
remote-store.sh \
|
remote-store.sh \
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue