From 2e0d63caf6d2b5a16be93b0830348490a9ef41d8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Aug 2022 20:03:22 +0200 Subject: [PATCH] Add InputAccessor::fetchToStore() --- src/libexpr/eval.cc | 8 +------- src/libexpr/primops.cc | 16 +++++++--------- src/libfetchers/fetchers.cc | 11 +---------- src/libfetchers/input-accessor.cc | 32 +++++++++++++++++++++++++++++++ src/libfetchers/input-accessor.hh | 17 ++++++++++++++++ tests/plugins/local.mk | 2 +- 6 files changed, 59 insertions(+), 27 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 69d51074b..2e5727f19 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2257,13 +2257,7 @@ StorePath EvalState::copyPathToStore(PathSet & context, const SourcePath & path) auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto source = sinkToSource([&](Sink & sink) { - path.dumpPath(sink); - }); - auto dstPath = - settings.readOnlyMode - ? store->computeStorePathFromDump(*source, path.baseName()).first - : store->addToStoreFromDump(*source, path.baseName(), FileIngestionMethod::Recursive, htSHA256, repair); + auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 15fb0f888..485994da0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2006,7 +2006,8 @@ static void addPath( } #endif - PathFilter filter = filterFun ? ([&](const Path & p) { + std::unique_ptr filter; + if (filterFun) filter = std::make_unique([&](const Path & p) { SourcePath path2{path.accessor, CanonPath(p)}; auto st = path2.lstat(); @@ -2025,7 +2026,7 @@ static void addPath( state.callFunction(*filterFun, 2, args, res, pos); return state.forceBool(res, pos); - }) : defaultPathFilter; + }); std::optional expectedStorePath; if (expectedHash) @@ -2036,13 +2037,10 @@ static void addPath( // store on-demand. if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto source = sinkToSource([&](Sink & sink) { - path.dumpPath(sink, filter); - }); - auto dstPath = - settings.readOnlyMode - ? state.store->computeStorePathFromDump(*source, name, method, htSHA256, refs).first - : state.store->addToStoreFromDump(*source, name, method, htSHA256, state.repair); + // FIXME + if (method != FileIngestionMethod::Recursive) + throw Error("'recursive = false' is not implemented"); + auto dstPath = path.fetchToStore(state.store, name, filter.get(), state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e69898c2b..65c332935 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -121,16 +121,7 @@ std::pair Input::fetchToStore(ref store) const auto [storePath, input] = [&]() -> std::pair { try { auto [accessor, input2] = getAccessor(store); - - // FIXME: add an optimisation for the case where the - // accessor is an FSInputAccessor pointing to a store - // path. - auto source = sinkToSource([&, accessor{accessor}](Sink & sink) { - accessor->dumpPath(CanonPath::root, sink); - }); - - auto storePath = store->addToStoreFromDump(*source, input2.getName()); - + auto storePath = accessor->root().fetchToStore(store, input2.getName()); return {storePath, input2}; } catch (Error & e) { e.addTrace({}, "while fetching the input '%s'", to_string()); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 782213211..f1a9a9e95 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -1,5 +1,6 @@ #include "input-accessor.hh" #include "util.hh" +#include "store-api.hh" #include @@ -85,6 +86,28 @@ void InputAccessor::dumpPath( dump(path); } +StorePath InputAccessor::fetchToStore( + ref store, + const CanonPath & path, + std::string_view name, + PathFilter * filter, + RepairFlag repair) +{ + // FIXME: add an optimisation for the case where the accessor is + // an FSInputAccessor pointing to a store path. + + auto source = sinkToSource([&](Sink & sink) { + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + }); + + auto storePath = + settings.readOnlyMode + ? store->computeStorePathFromDump(*source, name).first + : store->addToStoreFromDump(*source, name, FileIngestionMethod::Recursive, htSHA256, repair); + + return storePath; +} + std::optional InputAccessor::maybeLstat(const CanonPath & path) { // FIXME: merge these into one operation. @@ -164,6 +187,15 @@ ref makeMemoryInputAccessor() return make_ref(); } +StorePath SourcePath::fetchToStore( + ref store, + std::string_view name, + PathFilter * filter, + RepairFlag repair) const +{ + return accessor->fetchToStore(store, path, name, filter, repair); +} + std::string_view SourcePath::baseName() const { return path.baseName().value_or("source"); diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 1c6bd24cb..bbb0554b6 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -4,12 +4,16 @@ #include "types.hh" #include "archive.hh" #include "canon-path.hh" +#include "repair-flag.hh" namespace nix { MakeError(RestrictedPathError, Error); struct SourcePath; +struct StorePath; +class Store; +enum RepairFlag; struct InputAccessor : public std::enable_shared_from_this { @@ -52,6 +56,13 @@ struct InputAccessor : public std::enable_shared_from_this Sink & sink, PathFilter & filter = defaultPathFilter); + StorePath fetchToStore( + ref store, + const CanonPath & path, + std::string_view name, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + /* Return a corresponding path in the root filesystem, if possible. This is only possible for inputs that are materialized in the root filesystem. */ @@ -124,6 +135,12 @@ struct SourcePath PathFilter & filter = defaultPathFilter) const { return accessor->dumpPath(path, sink, filter); } + StorePath fetchToStore( + ref store, + std::string_view name, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair) const; + std::optional getPhysicalPath() const { return accessor->getPhysicalPath(path); } diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk index 3745e50b5..125a51abf 100644 --- a/tests/plugins/local.mk +++ b/tests/plugins/local.mk @@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 -libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr -I src/libfetchers +libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr -I src/libfetchers -I src/libstore