Merge pull request #12992 from DeterminateSystems/input-cache

Rename FlakeCache to InputCache and move it to libfetchers

src/libexpr/eval.cc

@@ -19,6 +19,7 @@
 #include "nix/util/url.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/tarball.hh"
+#include "nix/fetchers/input-cache.hh"
 
 #include "parser-tab.hh"
 
@@ -310,6 +311,7 @@ EvalState::EvalState(
     )}
     , store(store)
     , buildStore(buildStore ? buildStore : store)
+    , inputCache(fetchers::InputCache::create())
     , debugRepl(nullptr)
     , debugStop(false)
     , trylevel(0)
@@ -1152,6 +1154,7 @@ void EvalState::resetFileCache()
 {
     fileEvalCache.clear();
     fileParseCache.clear();
+    inputCache->clear();
 }
 
 

src/libexpr/include/nix/expr/eval.hh

@@ -33,7 +33,10 @@ namespace nix {
 constexpr size_t maxPrimOpArity = 8;
 
 class Store;
-namespace fetchers { struct Settings; }
+namespace fetchers {
+struct Settings;
+struct InputCache;
+}
 struct EvalSettings;
 class EvalState;
 class StorePath;
@@ -300,6 +303,8 @@ public:
 
     RootValue vImportedDrvToDerivation = nullptr;
 
+    ref<fetchers::InputCache> inputCache;
+
     /**
      * Debugger
      */

src/libfetchers/include/nix/fetchers/input-cache.hh

@@ -0,0 +1,31 @@
+#include "fetchers.hh"
+
+namespace nix::fetchers {
+
+struct InputCache
+{
+    struct CachedResult
+    {
+        ref<SourceAccessor> accessor;
+        Input resolvedInput;
+        Input lockedInput;
+    };
+
+    CachedResult getAccessor(ref<Store> store, const Input & originalInput, bool useRegistries);
+
+    struct CachedInput
+    {
+        Input lockedInput;
+        ref<SourceAccessor> accessor;
+    };
+
+    virtual std::optional<CachedInput> lookup(const Input & originalInput) const = 0;
+
+    virtual void upsert(Input key, CachedInput cachedInput) = 0;
+
+    virtual void clear() = 0;
+
+    static ref<InputCache> create();
+};
+
+}

src/libfetchers/include/nix/fetchers/meson.build

@@ -9,6 +9,7 @@ headers = files(
   'filtering-source-accessor.hh',
   'git-lfs-fetch.hh',
   'git-utils.hh',
+  'input-cache.hh',
   'registry.hh',
   'store-path-accessor.hh',
   'tarball.hh',

src/libfetchers/input-cache.cc

@@ -0,0 +1,80 @@
+#include "nix/fetchers/input-cache.hh"
+#include "nix/fetchers/registry.hh"
+#include "nix/util/sync.hh"
+#include "nix/util/source-path.hh"
+
+namespace nix::fetchers {
+
+InputCache::CachedResult InputCache::getAccessor(ref<Store> store, const Input & originalInput, bool useRegistries)
+{
+    auto fetched = lookup(originalInput);
+    Input resolvedInput = originalInput;
+
+    if (!fetched) {
+        if (originalInput.isDirect()) {
+            auto [accessor, lockedInput] = originalInput.getAccessor(store);
+            fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
+        } else {
+            if (useRegistries) {
+                auto [res, extraAttrs] =
+                    lookupInRegistries(store, originalInput, [](fetchers::Registry::RegistryType type) {
+                        /* Only use the global registry and CLI flags
+                           to resolve indirect flakerefs. */
+                        return type == fetchers::Registry::Flag || type == fetchers::Registry::Global;
+                    });
+                resolvedInput = std::move(res);
+                fetched = lookup(resolvedInput);
+                if (!fetched) {
+                    auto [accessor, lockedInput] = resolvedInput.getAccessor(store);
+                    fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
+                }
+                upsert(resolvedInput, *fetched);
+            } else {
+                throw Error(
+                    "'%s' is an indirect flake reference, but registry lookups are not allowed",
+                    originalInput.to_string());
+            }
+        }
+        upsert(originalInput, *fetched);
+    }
+
+    debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedInput.to_string());
+
+    return {fetched->accessor, resolvedInput, fetched->lockedInput};
+}
+
+struct InputCacheImpl : InputCache
+{
+    Sync<std::map<Input, CachedInput>> cache_;
+
+    std::optional<CachedInput> lookup(const Input & originalInput) const override
+    {
+        auto cache(cache_.readLock());
+        auto i = cache->find(originalInput);
+        if (i == cache->end())
+            return std::nullopt;
+        debug(
+            "mapping '%s' to previously seen input '%s' -> '%s",
+            originalInput.to_string(),
+            i->first.to_string(),
+            i->second.lockedInput.to_string());
+        return i->second;
+    }
+
+    void upsert(Input key, CachedInput cachedInput) override
+    {
+        cache_.lock()->insert_or_assign(std::move(key), std::move(cachedInput));
+    }
+
+    void clear() override
+    {
+        cache_.lock()->clear();
+    }
+};
+
+ref<InputCache> InputCache::create()
+{
+    return make_ref<InputCacheImpl>();
+}
+
+}

src/libfetchers/meson.build

@@ -44,6 +44,7 @@ sources = files(
   'git.cc',
   'github.cc',
   'indirect.cc',
+  'input-cache.cc',
   'mercurial.cc',
   'path.cc',
   'registry.cc',

src/libflake-c/nix_api_flake.cc

@@ -177,6 +177,7 @@ nix_locked_flake * nix_flake_lock(
 {
     nix_clear_err(context);
     try {
+        eval_state->state.resetFileCache();
         auto lockedFlake = nix::make_ref<nix::flake::LockedFlake>(nix::flake::lockFlake(
             *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags));
         return new nix_locked_flake{lockedFlake};

src/libflake/flake/flake.cc

@@ -14,6 +14,7 @@
 #include "nix/store/local-fs-store.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/util/memory-source-accessor.hh"
+#include "nix/fetchers/input-cache.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -23,66 +24,6 @@ using namespace flake;
 
 namespace flake {
 
-struct FetchedFlake
-{
-    FlakeRef lockedRef;
-    ref<SourceAccessor> accessor;
-};
-
-typedef std::map<FlakeRef, FetchedFlake> FlakeCache;
-
-static std::optional<FetchedFlake> lookupInFlakeCache(
-    const FlakeCache & flakeCache,
-    const FlakeRef & flakeRef)
-{
-    auto i = flakeCache.find(flakeRef);
-    if (i == flakeCache.end()) return std::nullopt;
-    debug("mapping '%s' to previously seen input '%s' -> '%s",
-        flakeRef, i->first, i->second.lockedRef);
-    return i->second;
-}
-
-static std::tuple<ref<SourceAccessor>, FlakeRef, FlakeRef> fetchOrSubstituteTree(
-    EvalState & state,
-    const FlakeRef & originalRef,
-    bool useRegistries,
-    FlakeCache & flakeCache)
-{
-    auto fetched = lookupInFlakeCache(flakeCache, originalRef);
-    FlakeRef resolvedRef = originalRef;
-
-    if (!fetched) {
-        if (originalRef.input.isDirect()) {
-            auto [accessor, lockedRef] = originalRef.lazyFetch(state.store);
-            fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
-        } else {
-            if (useRegistries) {
-                resolvedRef = originalRef.resolve(
-                    state.store,
-                    [](fetchers::Registry::RegistryType type) {
-                        /* Only use the global registry and CLI flags
-                           to resolve indirect flakerefs. */
-                        return type == fetchers::Registry::Flag || type == fetchers::Registry::Global;
-                    });
-                fetched = lookupInFlakeCache(flakeCache, originalRef);
-                if (!fetched) {
-                    auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store);
-                    fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
-                }
-                flakeCache.insert_or_assign(resolvedRef, *fetched);
-            }
-            else {
-                throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
-            }
-        }
-        flakeCache.insert_or_assign(originalRef, *fetched);
-    }
-
-    debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedRef);
-
-    return {fetched->accessor, resolvedRef, fetched->lockedRef};
-}
-
 static StorePath copyInputToStore(
     EvalState & state,
     fetchers::Input & input,
@@ -397,15 +338,16 @@ static Flake getFlake(
     EvalState & state,
     const FlakeRef & originalRef,
     bool useRegistries,
-    FlakeCache & flakeCache,
     const InputAttrPath & lockRootAttrPath)
 {
     // Fetch a lazy tree first.
-    auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+    auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries);
-        state, originalRef, useRegistries, flakeCache);
+
+    auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), originalRef.subdir);
+    auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), originalRef.subdir);
 
     // Parse/eval flake.nix to get at the input.self attributes.
-    auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {accessor}, lockRootAttrPath);
+    auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {cachedInput.accessor}, lockRootAttrPath);
 
     // Re-fetch the tree if necessary.
     auto newLockedRef = applySelfAttrs(lockedRef, flake);
@@ -414,14 +356,13 @@ static Flake getFlake(
         debug("refetching input '%s' due to self attribute", newLockedRef);
         // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
         newLockedRef.input.attrs.erase("narHash");
-        auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree(
+        auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, useRegistries);
-            state, newLockedRef, false, flakeCache);
+        cachedInput.accessor = cachedInput2.accessor;
-        accessor = accessor2;
+        lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
-        lockedRef = lockedRef2;
     }
 
     // Copy the tree to the store.
-    auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, accessor);
+    auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor);
 
     // Re-parse flake.nix from the store.
     return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath);
@@ -429,8 +370,7 @@ static Flake getFlake(
 
 Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistries)
 {
-    FlakeCache flakeCache;
+    return getFlake(state, originalRef, useRegistries, {});
-    return getFlake(state, originalRef, useRegistries, flakeCache, {});
 }
 
 static LockFile readLockFile(
@@ -452,11 +392,9 @@ LockedFlake lockFlake(
 {
     experimentalFeatureSettings.require(Xp::Flakes);
 
-    FlakeCache flakeCache;
-
     auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
 
-    auto flake = getFlake(state, topRef, useRegistries, flakeCache, {});
+    auto flake = getFlake(state, topRef, useRegistries, {});
 
     if (lockFlags.applyNixConfig) {
         flake.config.apply(settings);
@@ -631,7 +569,7 @@ LockedFlake lockFlake(
                         if (auto resolvedPath = resolveRelativePath()) {
                             return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath);
                         } else {
-                            return getFlake(state, ref, useRegistries, flakeCache, inputAttrPath);
+                            return getFlake(state, ref, useRegistries, inputAttrPath);
                         }
                     };
 
@@ -779,11 +717,12 @@ LockedFlake lockFlake(
                                 if (auto resolvedPath = resolveRelativePath()) {
                                     return {*resolvedPath, *input.ref};
                                 } else {
-                                    auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+                                    auto cachedInput = state.inputCache->getAccessor(state.store, input.ref->input, useRegistries);
-                                        state, *input.ref, useRegistries, flakeCache);
+
+                                    auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
 
                                     // FIXME: allow input to be lazy.
-                                    auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor);
+                                    auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, cachedInput.accessor);
 
                                     return {state.storePath(storePath), lockedRef};
                                 }