Add builtins.filterPath

This is like builtins.{filterSource,path}, but returns a virtual path
that applies the filter lazily, rather than copying the result to the
Nix store. Thus filterPath can be composed.

src/libexpr/eval.hh

@@ -525,6 +525,13 @@ public:
      */
     [[nodiscard]] StringMap realiseContext(const PathSet & context);
 
+    /* Call the binary path filter predicate used builtins.path etc. */
+    bool callPathFilter(
+        Value * filterFun,
+        const SourcePath & path,
+        std::string_view pathArg,
+        PosIdx pos);
+
 private:
 
     unsigned long nrEnvs = 0;

src/libexpr/primops.cc

@@ -1974,6 +1974,30 @@ static RegisterPrimOp primop_toFile({
     .fun = prim_toFile,
 });
 
+bool EvalState::callPathFilter(
+    Value * filterFun,
+    const SourcePath & path,
+    std::string_view pathArg,
+    PosIdx pos)
+{
+    auto st = path.lstat();
+
+    /* Call the filter function.  The first argument is the path, the
+       second is a string indicating the type of the file. */
+    Value arg1;
+    arg1.mkString(pathArg);
+
+    Value arg2;
+    // assert that type is not "unknown"
+    arg2.mkString(fileTypeToString(st.type));
+
+    Value * args []{&arg1, &arg2};
+    Value res;
+    callFunction(*filterFun, 2, args, res, pos);
+
+    return forceBool(res, pos);
+}
+
 static void addPath(
     EvalState & state,
     const PosIdx pos,
@@ -2010,25 +2034,10 @@ static void addPath(
         #endif
 
         std::unique_ptr<PathFilter> filter;
-        if (filterFun) filter = std::make_unique<PathFilter>([&](const Path & p) {
-            SourcePath path2{path.accessor, CanonPath(p)};
-
-            auto st = path2.lstat();
-
-            /* Call the filter function.  The first argument is the path,
-               the second is a string indicating the type of the file. */
-            Value arg1;
-            arg1.mkString(path2.path.abs());
-
-            Value arg2;
-            // assert that type is not "unknown"
-            arg2.mkString(fileTypeToString(st.type));
-
-            Value * args []{&arg1, &arg2};
-            Value res;
-            state.callFunction(*filterFun, 2, args, res, pos);
-
-            return state.forceBool(res, pos);
+        if (filterFun)
+            filter = std::make_unique<PathFilter>([&](const Path & p) {
+                auto p2 = CanonPath(p);
+                return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos);
             });
 
         std::optional<StorePath> expectedStorePath;
@@ -2130,7 +2139,6 @@ static RegisterPrimOp primop_filterSource({
 
 static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    state.forceAttrs(*args[0], pos);
     std::optional<SourcePath> path;
     std::string name;
     Value * filterFun = nullptr;
@@ -2138,6 +2146,8 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
     std::optional<Hash> expectedHash;
     PathSet context;
 
+    state.forceAttrs(*args[0], pos);
+
     for (auto & attr : *args[0]->attrs) {
         auto n = state.symbols[attr.name];
         if (n == "path")

src/libexpr/primops/filterPath.cc

@@ -0,0 +1,174 @@
+#include "primops.hh"
+
+namespace nix {
+
+struct FilteringInputAccessor : InputAccessor
+{
+    EvalState & state;
+    PosIdx pos;
+    ref<InputAccessor> next;
+    CanonPath prefix;
+    Value * filterFun;
+
+    std::map<CanonPath, bool> cache;
+
+    FilteringInputAccessor(EvalState & state, PosIdx pos, const SourcePath & src, Value * filterFun)
+        : state(state)
+        , pos(pos)
+        , next(src.accessor)
+        , prefix(src.path)
+        , filterFun(filterFun)
+    {
+    }
+
+    std::string readFile(const CanonPath & path) override
+    {
+        checkAccess(path);
+        return next->readFile(prefix + path);
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        checkAccess(path);
+        return next->pathExists(prefix + path);
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        checkAccess(path);
+        return next->lstat(prefix + path);
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        checkAccess(path);
+        DirEntries entries;
+        for (auto & entry : next->readDirectory(prefix + path)) {
+            if (isAllowed(path + entry.first))
+                entries.insert(std::move(entry));
+        }
+        return entries;
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        checkAccess(path);
+        return next->readLink(prefix + path);
+    }
+
+    void checkAccess(const CanonPath & path)
+    {
+        if (!isAllowed(path))
+            throw Error("access to path '%s' has been filtered out", showPath(path));
+    }
+
+    bool isAllowed(const CanonPath & path)
+    {
+        auto i = cache.find(path);
+        if (i != cache.end()) return i->second;
+        auto res = isAllowedUncached(path);
+        cache.emplace(path, res);
+        return res;
+    }
+
+    bool isAllowedUncached(const CanonPath & path)
+    {
+        if (!path.isRoot() && !isAllowed(*path.parent())) return false;
+        // Note that unlike 'builtins.{path,filterSource}', we don't
+        // pass the prefix to the filter function.
+        return state.callPathFilter(filterFun, {next, prefix + path}, path.abs(), pos);
+    }
+
+    std::string showPath(const CanonPath & path) override
+    {
+        return next->showPath(prefix + path);
+    }
+};
+
+static void prim_filterPath(EvalState & state, PosIdx pos, Value * * args, Value & v)
+{
+    std::optional<SourcePath> path;
+    Value * filterFun = nullptr;
+    PathSet context;
+
+    state.forceAttrs(*args[0], pos);
+
+    for (auto & attr : *args[0]->attrs) {
+        auto n = state.symbols[attr.name];
+        if (n == "path")
+            path.emplace(state.coerceToPath(attr.pos, *attr.value, context));
+        else if (n == "filter") {
+            state.forceValue(*attr.value, pos);
+            filterFun = attr.value;
+        }
+        else
+            state.debugThrowLastTrace(EvalError({
+                .msg = hintfmt("unsupported argument '%1%' to 'filterPath'", state.symbols[attr.name]),
+                .errPos = state.positions[attr.pos]
+            }));
+    }
+
+    if (!path)
+        state.debugThrowLastTrace(EvalError({
+            .msg = hintfmt("'path' required"),
+            .errPos = state.positions[pos]
+        }));
+
+    if (!filterFun)
+        state.debugThrowLastTrace(EvalError({
+            .msg = hintfmt("'filter' required"),
+            .errPos = state.positions[pos]
+        }));
+
+    if (!context.empty())
+        state.debugThrowLastTrace(EvalError({
+            .msg = hintfmt("'path' argument to 'filterPath' cannot have a context"),
+            .errPos = state.positions[pos]
+        }));
+
+    auto accessor = make_ref<FilteringInputAccessor>(state, pos, *path, filterFun);
+
+    state.registerAccessor(accessor);
+
+    v.mkPath(accessor->root());
+}
+
+static RegisterPrimOp primop_filterPath({
+    .name = "__filterPath",
+    .args = {"args"},
+    .doc = R"(
+      This function lets you filter out files from a path. It takes a
+      path and a predicate function, and returns a new path from which
+      every file has been removed for which the predicate function
+      returns `false`.
+
+      For example, the following filters out all regular files in
+      `./doc` that don't end with the extension `.md`:
+
+      ```nix
+      builtins.filterPath {
+        path = ./doc;
+        filter =
+          path: type:
+          (type != "regular" || hasSuffix ".md" path);
+      }
+      ```
+
+      The filter function is called for all files in `path`. It takes
+      two arguments. The first is a string that represents the path of
+      the file to be filtered, relative to `path` (i.e. it does *not*
+      contain `./doc` in the example above). The second is the file
+      type, which can be one of `regular`, `directory` or `symlink`.
+
+      Note that unlike `builtins.filterSource` and `builtins.path`,
+      this function does not copy the result to the Nix store. Rather,
+      the result is a virtual path that lazily applies the filter
+      predicate. The result will only be copied to the Nix store if
+      needed (e.g. if used in a derivation attribute like `src =
+      builtins.filterPath { ... }`).
+    )",
+    .fun = prim_filterPath,
+    .experimentalFeature = Xp::Flakes,
+});
+
+}

tests/flakes/tree-operators.sh

@@ -0,0 +1,48 @@
+source ./common.sh
+
+flake1Dir=$TEST_ROOT/flake1
+
+mkdir -p $flake1Dir
+
+pwd=$(pwd)
+
+cat > $flake1Dir/flake.nix <<EOF
+{
+  inputs.docs = {
+    url = "path://$pwd/../../doc";
+    flake = false;
+  };
+
+  outputs = { self, docs }: with import ./lib.nix; {
+
+    mdOnly = builtins.filterPath {
+      path = docs;
+      filter =
+        path: type:
+        (type != "regular" || hasSuffix ".md" path);
+    };
+
+    noReleaseNotes = builtins.filterPath {
+      path = self.mdOnly + "/manual";
+      filter =
+        path: type:
+        assert !hasPrefix "/manual" path;
+        (builtins.baseNameOf path != "release-notes");
+    };
+
+  };
+}
+EOF
+
+cp ../lang/lib.nix $flake1Dir/
+
+nix build --out-link $TEST_ROOT/result $flake1Dir#mdOnly
+[[ -e $TEST_ROOT/result/manual/src/quick-start.md ]]
+[[ -e $TEST_ROOT/result/manual/src/release-notes ]]
+(! find $TEST_ROOT/result/ -type f | grep -v '.md$')
+find $TEST_ROOT/result/ -type f | grep release-notes
+
+nix build --out-link $TEST_ROOT/result $flake1Dir#noReleaseNotes
+[[ -e $TEST_ROOT/result/src/quick-start.md ]]
+(! [[ -e $TEST_ROOT/result/src/release-notes ]])
+(! find $TEST_ROOT/result/ -type f | grep release-notes)

tests/lang/lib.nix

@@ -19,6 +19,8 @@ rec {
 
   sum = foldl' (x: y: add x y) 0;
 
+  hasPrefix = pref: str: substring 0 (stringLength pref) str == pref;
+
   hasSuffix = ext: fileName:
     let lenFileName = stringLength fileName;
         lenExt = stringLength ext;

tests/local.mk

@@ -10,6 +10,7 @@ nix_tests = \
   flakes/unlocked-override.sh \
   flakes/absolute-paths.sh \
   flakes/build-paths.sh \
+  flakes/tree-operators.sh \
   ca/gc.sh \
   gc.sh \
   remote-store.sh \