From 73b175481634ac447b5fcfa8d3f60f37b5c7c860 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Apr 2025 17:29:15 +0200 Subject: [PATCH] Mount flake input source accessors on top of storeFS This way, we don't need the PathDisplaySourceAccessor source accessor hack, since error messages are produced directly by the original source accessor. In fact, we don't even need to copy the inputs to the store at all, so this gets us very close to lazy trees. We just need to know the store path so that requires hashing the entire input, which isn't lazy. But the next step will be to use a virtual store path that gets rewritten to the actual store path only when needed. --- src/libexpr/eval.cc | 46 +++------------- src/libexpr/eval.hh | 10 ++-- src/libexpr/primops/fetchTree.cc | 3 +- src/libfetchers/filtering-source-accessor.cc | 7 ++- src/libfetchers/filtering-source-accessor.hh | 2 + src/libfetchers/git.cc | 1 + src/libflake/flake/flake.cc | 3 +- src/libutil/forwarding-source-accessor.hh | 57 -------------------- src/libutil/meson.build | 2 +- src/libutil/mounted-source-accessor.cc | 16 ++++-- src/libutil/mounted-source-accessor.hh | 14 +++++ src/libutil/source-accessor.hh | 4 +- tests/functional/flakes/source-paths.sh | 12 +++++ tests/functional/restricted.sh | 6 +-- 14 files changed, 66 insertions(+), 117 deletions(-) delete mode 100644 src/libutil/forwarding-source-accessor.hh create mode 100644 src/libutil/mounted-source-accessor.hh diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 18b8c2f91..0ad12b9b5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -14,8 +14,8 @@ #include "profiles.hh" #include "print.hh" #include "filtering-source-accessor.hh" -#include "forwarding-source-accessor.hh" #include "memory-source-accessor.hh" +#include "mounted-source-accessor.hh" #include "gc-small-vector.hh" #include "url.hh" #include "fetch-to-store.hh" @@ -181,34 +181,6 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } } -struct PathDisplaySourceAccessor : ForwardingSourceAccessor -{ - ref storePathAccessors; - - PathDisplaySourceAccessor( - ref next, - ref storePathAccessors) - : ForwardingSourceAccessor(next) - , storePathAccessors(storePathAccessors) - { - } - - std::string showPath(const CanonPath & path) override - { - /* Find the accessor that produced `path`, if any, and use it - to render a more informative path - (e.g. `«github:foo/bar»/flake.nix` rather than - `/nix/store/hash.../flake.nix`). */ - auto ub = storePathAccessors->upper_bound(path); - if (ub != storePathAccessors->begin()) - ub--; - if (ub != storePathAccessors->end() && path.isWithin(ub->first)) - return ub->second->showPath(path.removePrefix(ub->first)); - else - return next->showPath(path); - } -}; - static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( @@ -274,7 +246,12 @@ EvalState::EvalState( } , repair(NoRepair) , emptyBindings(0) - , storePathAccessors(make_ref()) + , storeFS( + makeMountedSourceAccessor( + { + {CanonPath::root, makeEmptySourceAccessor()}, + {CanonPath(store->storeDir), makeFSSourceAccessor(dirOf(store->toRealPath(StorePath::dummy)))} + })) , rootFS( ({ /* In pure eval mode, we provide a filesystem that only @@ -290,18 +267,11 @@ EvalState::EvalState( auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy)); if (settings.pureEval || store->storeDir != realStoreDir) { - auto storeFS = makeMountedSourceAccessor( - { - {CanonPath::root, makeEmptySourceAccessor()}, - {CanonPath(store->storeDir), makeFSSourceAccessor(realStoreDir)} - }); accessor = settings.pureEval - ? storeFS + ? storeFS.cast() : makeUnionSourceAccessor({accessor, storeFS}); } - accessor = make_ref(accessor, storePathAccessors); - /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) accessor = AllowListSourceAccessor::create(accessor, {}, {}, diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3797c40a4..4ae73de57 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -37,6 +37,7 @@ class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; struct MemorySourceAccessor; +struct MountedSourceAccessor; namespace eval_cache { class EvalCache; } @@ -262,15 +263,10 @@ public: /** `"unknown"` */ Value vStringUnknown; - using StorePathAccessors = std::map>; - /** - * A map back to the original `SourceAccessor`s used to produce - * store paths. We keep track of this to produce error messages - * that refer to the original flakerefs. - * FIXME: use Sync. + * The accessor corresponding to `store`. */ - ref storePathAccessors; + const ref storeFS; /** * The accessor for the root filesystem. diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 8bbc435e4..f5ca5fd3e 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -10,6 +10,7 @@ #include "url.hh" #include "value-to-json.hh" #include "fetch-to-store.hh" +#include "mounted-source-accessor.hh" #include @@ -204,7 +205,7 @@ static void fetchTree( state.allowPath(storePath); - state.storePathAccessors->insert_or_assign(CanonPath(state.store->printStorePath(storePath)), accessor); + state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor); emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); } diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index c6a00faef..10a22d026 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -20,9 +20,14 @@ bool FilteringSourceAccessor::pathExists(const CanonPath & path) } std::optional FilteringSourceAccessor::maybeLstat(const CanonPath & path) +{ + return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt; +} + +SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path) { checkAccess(path); - return next->maybeLstat(prefix / path); + return next->lstat(prefix / path); } SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path) diff --git a/src/libfetchers/filtering-source-accessor.hh b/src/libfetchers/filtering-source-accessor.hh index 41889cfd7..544b4a490 100644 --- a/src/libfetchers/filtering-source-accessor.hh +++ b/src/libfetchers/filtering-source-accessor.hh @@ -38,6 +38,8 @@ struct FilteringSourceAccessor : SourceAccessor bool pathExists(const CanonPath & path) override; + Stat lstat(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 6b82d9ae3..54c66d151 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -15,6 +15,7 @@ #include "fetch-settings.hh" #include "json-utils.hh" #include "archive.hh" +#include "mounted-source-accessor.hh" #include #include diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index a14b55c6a..aa0229793 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -13,6 +13,7 @@ #include "value-to-json.hh" #include "local-fs-store.hh" #include "fetch-to-store.hh" +#include "mounted-source-accessor.hh" #include @@ -92,7 +93,7 @@ static StorePath copyInputToStore( state.allowPath(storePath); - state.storePathAccessors->insert_or_assign(CanonPath(state.store->printStorePath(storePath)), accessor); + state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor); auto narHash = state.store->queryPathInfo(storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); diff --git a/src/libutil/forwarding-source-accessor.hh b/src/libutil/forwarding-source-accessor.hh deleted file mode 100644 index bdba2addc..000000000 --- a/src/libutil/forwarding-source-accessor.hh +++ /dev/null @@ -1,57 +0,0 @@ -#pragma once - -#include "source-accessor.hh" - -namespace nix { - -/** - * A source accessor that just forwards every operation to another - * accessor. This is not useful in itself but can be used as a - * superclass for accessors that do change some operations. - */ -struct ForwardingSourceAccessor : SourceAccessor -{ - ref next; - - ForwardingSourceAccessor(ref next) - : next(next) - { - } - - std::string readFile(const CanonPath & path) override - { - return next->readFile(path); - } - - void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override - { - next->readFile(path, sink, sizeCallback); - } - - std::optional maybeLstat(const CanonPath & path) override - { - return next->maybeLstat(path); - } - - DirEntries readDirectory(const CanonPath & path) override - { - return next->readDirectory(path); - } - - std::string readLink(const CanonPath & path) override - { - return next->readLink(path); - } - - std::string showPath(const CanonPath & path) override - { - return next->showPath(path); - } - - std::optional getPhysicalPath(const CanonPath & path) override - { - return next->getPhysicalPath(path); - } -}; - -} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index b2bc0b4ec..f698f04dd 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -215,7 +215,6 @@ headers = [config_h] + files( 'file-system.hh', 'finally.hh', 'fmt.hh', - 'forwarding-source-accessor.hh', 'fs-sink.hh', 'git.hh', 'hash.hh', @@ -225,6 +224,7 @@ headers = [config_h] + files( 'logging.hh', 'lru-cache.hh', 'memory-source-accessor.hh', + 'mounted-source-accessor.hh', 'muxable-pipe.hh', 'os-string.hh', 'pool.hh', diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 79223d155..e1442d686 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -1,12 +1,12 @@ -#include "source-accessor.hh" +#include "mounted-source-accessor.hh" namespace nix { -struct MountedSourceAccessor : SourceAccessor +struct MountedSourceAccessorImpl : MountedSourceAccessor { std::map> mounts; - MountedSourceAccessor(std::map> _mounts) + MountedSourceAccessorImpl(std::map> _mounts) : mounts(std::move(_mounts)) { displayPrefix.clear(); @@ -69,11 +69,17 @@ struct MountedSourceAccessor : SourceAccessor auto [accessor, subpath] = resolve(path); return accessor->getPhysicalPath(subpath); } + + void mount(CanonPath mountPoint, ref accessor) override + { + // FIXME: thread-safety + mounts.insert_or_assign(std::move(mountPoint), accessor); + } }; -ref makeMountedSourceAccessor(std::map> mounts) +ref makeMountedSourceAccessor(std::map> mounts) { - return make_ref(std::move(mounts)); + return make_ref(std::move(mounts)); } } diff --git a/src/libutil/mounted-source-accessor.hh b/src/libutil/mounted-source-accessor.hh new file mode 100644 index 000000000..4e75edfaf --- /dev/null +++ b/src/libutil/mounted-source-accessor.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +struct MountedSourceAccessor : SourceAccessor +{ + virtual void mount(CanonPath mountPoint, ref accessor) = 0; +}; + +ref makeMountedSourceAccessor(std::map> mounts); + +} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 79ae092ac..a069e024d 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -118,7 +118,7 @@ struct SourceAccessor : std::enable_shared_from_this std::string typeString(); }; - Stat lstat(const CanonPath & path); + virtual Stat lstat(const CanonPath & path); virtual std::optional maybeLstat(const CanonPath & path) = 0; @@ -214,8 +214,6 @@ ref getFSSourceAccessor(); */ ref makeFSSourceAccessor(std::filesystem::path root); -ref makeMountedSourceAccessor(std::map> mounts); - /** * Construct an accessor that presents a "union" view of a vector of * underlying accessors. Earlier accessors take precedence over later. diff --git a/tests/functional/flakes/source-paths.sh b/tests/functional/flakes/source-paths.sh index 1eb8d618d..10b834bc8 100644 --- a/tests/functional/flakes/source-paths.sh +++ b/tests/functional/flakes/source-paths.sh @@ -13,6 +13,7 @@ cat > "$repo/flake.nix" < "$repo/foo" + +expectStderr 1 nix eval "$repo#z" | grepQuiet "error: File 'foo' in the repository \"$repo\" is not tracked by Git." + +git -C "$repo" add "$repo/foo" + +[[ $(nix eval --raw "$repo#z") = foo ]] diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 00ee4ddc8..bc42ec891 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -23,7 +23,7 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=./simple.nix -I src2=./conf (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" +expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' #| grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. p=$(nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}") @@ -53,9 +53,9 @@ mkdir -p $TEST_ROOT/tunnel.d $TEST_ROOT/foo2 ln -sfn .. $TEST_ROOT/tunnel.d/tunnel echo foo > $TEST_ROOT/bar -expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d #| grepQuiet "forbidden in restricted mode" -expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d #| grepQuiet "forbidden in restricted mode" # Reading the parents of allowed paths should show only the ancestors of the allowed paths. [[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]]