diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index c3b3964f9..dda650d8d 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -135,21 +135,15 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
         size_t number = std::stoi(std::string(pos, 0, slash));
         pos = pos.substr(slash);
 
-        std::shared_ptr<InputAccessor> accessor;
-        for (auto & i : state.inputAccessors)
-            if (i.second->number == number) {
-                accessor = i.second;
-                break;
-            }
-
-        if (!accessor) fail();
+        auto accessor = state.inputAccessors.find(number);
+        if (accessor == state.inputAccessors.end()) fail();
 
         auto colon = pos.rfind(':');
         if (colon == std::string::npos) fail();
         std::string filename(pos, 0, colon);
         auto lineno = std::stoi(std::string(pos, colon + 1, std::string::npos));
 
-        return {SourcePath{ref(accessor), CanonPath(filename)}, lineno};
+        return {SourcePath{accessor->second, CanonPath(filename)}, lineno};
 
     } catch (std::invalid_argument & e) {
         fail();
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 4fee93b8a..2d1c5e1f5 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -114,7 +114,9 @@ public:
 
     const SourcePath derivationInternal;
 
-    std::unordered_map<InputAccessor *, ref<InputAccessor>> inputAccessors;
+    /* A map keyed by InputAccessor::number that keeps input accessors
+       alive. */
+    std::unordered_map<size_t, ref<InputAccessor>> inputAccessors;
 
     /* Store used to materialise .drv files. */
     const ref<Store> store;
diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc
index 9a5449695..4bdcfff78 100644
--- a/src/libexpr/paths.cc
+++ b/src/libexpr/paths.cc
@@ -11,7 +11,7 @@ SourcePath EvalState::rootPath(const Path & path)
 
 void EvalState::registerAccessor(ref<InputAccessor> accessor)
 {
-    inputAccessors.emplace(&*accessor, accessor);
+    inputAccessors.emplace(accessor->number, accessor);
 }
 
 }