diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9d60676f5..10a33c042 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -19,6 +19,7 @@ #include "nix/util/url.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/tarball.hh" +#include "nix/fetchers/input-cache.hh" #include "parser-tab.hh" @@ -310,6 +311,7 @@ EvalState::EvalState( )} , store(store) , buildStore(buildStore ? buildStore : store) + , inputCache(fetchers::InputCache::create()) , debugRepl(nullptr) , debugStop(false) , trylevel(0) @@ -1152,6 +1154,7 @@ void EvalState::resetFileCache() { fileEvalCache.clear(); fileParseCache.clear(); + inputCache->clear(); } diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 61da225fc..6a6959bd8 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -33,7 +33,10 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; -namespace fetchers { struct Settings; } +namespace fetchers { +struct Settings; +struct InputCache; +} struct EvalSettings; class EvalState; class StorePath; @@ -300,6 +303,8 @@ public: RootValue vImportedDrvToDerivation = nullptr; + ref inputCache; + /** * Debugger */ diff --git a/src/libfetchers/include/nix/fetchers/input-cache.hh b/src/libfetchers/include/nix/fetchers/input-cache.hh new file mode 100644 index 000000000..a7ca34487 --- /dev/null +++ b/src/libfetchers/include/nix/fetchers/input-cache.hh @@ -0,0 +1,31 @@ +#include "fetchers.hh" + +namespace nix::fetchers { + +struct InputCache +{ + struct CachedResult + { + ref accessor; + Input resolvedInput; + Input lockedInput; + }; + + CachedResult getAccessor(ref store, const Input & originalInput, bool useRegistries); + + struct CachedInput + { + Input lockedInput; + ref accessor; + }; + + virtual std::optional lookup(const Input & originalInput) const = 0; + + virtual void upsert(Input key, CachedInput cachedInput) = 0; + + virtual void clear() = 0; + + static ref create(); +}; + +} diff --git a/src/libfetchers/include/nix/fetchers/meson.build b/src/libfetchers/include/nix/fetchers/meson.build index 3a752d9cb..e6ddedd97 100644 --- a/src/libfetchers/include/nix/fetchers/meson.build +++ b/src/libfetchers/include/nix/fetchers/meson.build @@ -9,6 +9,7 @@ headers = files( 'filtering-source-accessor.hh', 'git-lfs-fetch.hh', 'git-utils.hh', + 'input-cache.hh', 'registry.hh', 'store-path-accessor.hh', 'tarball.hh', diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc new file mode 100644 index 000000000..716143899 --- /dev/null +++ b/src/libfetchers/input-cache.cc @@ -0,0 +1,80 @@ +#include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/registry.hh" +#include "nix/util/sync.hh" +#include "nix/util/source-path.hh" + +namespace nix::fetchers { + +InputCache::CachedResult InputCache::getAccessor(ref store, const Input & originalInput, bool useRegistries) +{ + auto fetched = lookup(originalInput); + Input resolvedInput = originalInput; + + if (!fetched) { + if (originalInput.isDirect()) { + auto [accessor, lockedInput] = originalInput.getAccessor(store); + fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); + } else { + if (useRegistries) { + auto [res, extraAttrs] = + lookupInRegistries(store, originalInput, [](fetchers::Registry::RegistryType type) { + /* Only use the global registry and CLI flags + to resolve indirect flakerefs. */ + return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; + }); + resolvedInput = std::move(res); + fetched = lookup(resolvedInput); + if (!fetched) { + auto [accessor, lockedInput] = resolvedInput.getAccessor(store); + fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); + } + upsert(resolvedInput, *fetched); + } else { + throw Error( + "'%s' is an indirect flake reference, but registry lookups are not allowed", + originalInput.to_string()); + } + } + upsert(originalInput, *fetched); + } + + debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedInput.to_string()); + + return {fetched->accessor, resolvedInput, fetched->lockedInput}; +} + +struct InputCacheImpl : InputCache +{ + Sync> cache_; + + std::optional lookup(const Input & originalInput) const override + { + auto cache(cache_.readLock()); + auto i = cache->find(originalInput); + if (i == cache->end()) + return std::nullopt; + debug( + "mapping '%s' to previously seen input '%s' -> '%s", + originalInput.to_string(), + i->first.to_string(), + i->second.lockedInput.to_string()); + return i->second; + } + + void upsert(Input key, CachedInput cachedInput) override + { + cache_.lock()->insert_or_assign(std::move(key), std::move(cachedInput)); + } + + void clear() override + { + cache_.lock()->clear(); + } +}; + +ref InputCache::create() +{ + return make_ref(); +} + +} diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 6e7129f4c..321146ca4 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -44,6 +44,7 @@ sources = files( 'git.cc', 'github.cc', 'indirect.cc', + 'input-cache.cc', 'mercurial.cc', 'path.cc', 'registry.cc', diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index 06b139f91..f100bf146 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -177,6 +177,7 @@ nix_locked_flake * nix_flake_lock( { nix_clear_err(context); try { + eval_state->state.resetFileCache(); auto lockedFlake = nix::make_ref(nix::flake::lockFlake( *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags)); return new nix_locked_flake{lockedFlake}; diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 8856a03dd..afeefdaec 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -14,6 +14,7 @@ #include "nix/store/local-fs-store.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/fetchers/input-cache.hh" #include @@ -23,66 +24,6 @@ using namespace flake; namespace flake { -struct FetchedFlake -{ - FlakeRef lockedRef; - ref accessor; -}; - -typedef std::map FlakeCache; - -static std::optional lookupInFlakeCache( - const FlakeCache & flakeCache, - const FlakeRef & flakeRef) -{ - auto i = flakeCache.find(flakeRef); - if (i == flakeCache.end()) return std::nullopt; - debug("mapping '%s' to previously seen input '%s' -> '%s", - flakeRef, i->first, i->second.lockedRef); - return i->second; -} - -static std::tuple, FlakeRef, FlakeRef> fetchOrSubstituteTree( - EvalState & state, - const FlakeRef & originalRef, - bool useRegistries, - FlakeCache & flakeCache) -{ - auto fetched = lookupInFlakeCache(flakeCache, originalRef); - FlakeRef resolvedRef = originalRef; - - if (!fetched) { - if (originalRef.input.isDirect()) { - auto [accessor, lockedRef] = originalRef.lazyFetch(state.store); - fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor}); - } else { - if (useRegistries) { - resolvedRef = originalRef.resolve( - state.store, - [](fetchers::Registry::RegistryType type) { - /* Only use the global registry and CLI flags - to resolve indirect flakerefs. */ - return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; - }); - fetched = lookupInFlakeCache(flakeCache, originalRef); - if (!fetched) { - auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store); - fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor}); - } - flakeCache.insert_or_assign(resolvedRef, *fetched); - } - else { - throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); - } - } - flakeCache.insert_or_assign(originalRef, *fetched); - } - - debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedRef); - - return {fetched->accessor, resolvedRef, fetched->lockedRef}; -} - static StorePath copyInputToStore( EvalState & state, fetchers::Input & input, @@ -397,15 +338,16 @@ static Flake getFlake( EvalState & state, const FlakeRef & originalRef, bool useRegistries, - FlakeCache & flakeCache, const InputAttrPath & lockRootAttrPath) { // Fetch a lazy tree first. - auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, originalRef, useRegistries, flakeCache); + auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries); + + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), originalRef.subdir); + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), originalRef.subdir); // Parse/eval flake.nix to get at the input.self attributes. - auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {accessor}, lockRootAttrPath); + auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {cachedInput.accessor}, lockRootAttrPath); // Re-fetch the tree if necessary. auto newLockedRef = applySelfAttrs(lockedRef, flake); @@ -414,14 +356,13 @@ static Flake getFlake( debug("refetching input '%s' due to self attribute", newLockedRef); // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. newLockedRef.input.attrs.erase("narHash"); - auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree( - state, newLockedRef, false, flakeCache); - accessor = accessor2; - lockedRef = lockedRef2; + auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, useRegistries); + cachedInput.accessor = cachedInput2.accessor; + lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } // Copy the tree to the store. - auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, accessor); + auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor); // Re-parse flake.nix from the store. return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath); @@ -429,8 +370,7 @@ static Flake getFlake( Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistries) { - FlakeCache flakeCache; - return getFlake(state, originalRef, useRegistries, flakeCache, {}); + return getFlake(state, originalRef, useRegistries, {}); } static LockFile readLockFile( @@ -452,11 +392,9 @@ LockedFlake lockFlake( { experimentalFeatureSettings.require(Xp::Flakes); - FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); - auto flake = getFlake(state, topRef, useRegistries, flakeCache, {}); + auto flake = getFlake(state, topRef, useRegistries, {}); if (lockFlags.applyNixConfig) { flake.config.apply(settings); @@ -631,7 +569,7 @@ LockedFlake lockFlake( if (auto resolvedPath = resolveRelativePath()) { return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); } else { - return getFlake(state, ref, useRegistries, flakeCache, inputAttrPath); + return getFlake(state, ref, useRegistries, inputAttrPath); } }; @@ -779,11 +717,12 @@ LockedFlake lockFlake( if (auto resolvedPath = resolveRelativePath()) { return {*resolvedPath, *input.ref}; } else { - auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, *input.ref, useRegistries, flakeCache); + auto cachedInput = state.inputCache->getAccessor(state.store, input.ref->input, useRegistries); + + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); // FIXME: allow input to be lazy. - auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor); + auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, cachedInput.accessor); return {state.storePath(storePath), lockedRef}; }