diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index a41546c44..b54416b10 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -438,11 +438,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         {
             if (!(statusFlags & GIT_STATUS_INDEX_DELETED) &&
                 !(statusFlags & GIT_STATUS_WT_DELETED))
-                info.files.emplace(CanonPath(path),
-                    statusFlags == GIT_STATUS_CURRENT
-                    ? WorkdirInfo::State::Clean
-                    : WorkdirInfo::State::Dirty);
-            else
+            {
+                info.files.insert(CanonPath(path));
+                if (statusFlags != GIT_STATUS_CURRENT)
+                    info.dirtyFiles.insert(CanonPath(path));
+            } else
                 info.deletedFiles.insert(CanonPath(path));
             if (statusFlags != GIT_STATUS_CURRENT)
                 info.isDirty = true;
@@ -1208,15 +1208,6 @@ ref<SourceAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore
     }
 }
 
-template<typename K, typename V>
-std::set<K> getKeys(const std::map<K, V> & c)
-{
-    std::set<K> res;
-    for (auto & i : c)
-        res.insert(i.first);
-    return res;
-}
-
 ref<SourceAccessor> GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
@@ -1229,7 +1220,7 @@ ref<SourceAccessor> GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool export
         ? makeEmptySourceAccessor()
         : AllowListSourceAccessor::create(
             makeFSSourceAccessor(path),
-            std::set<CanonPath> { getKeys(wd.files) },
+            std::set<CanonPath> { wd.files },
             std::move(makeNotAllowedError)).cast<SourceAccessor>();
     if (exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index 2db64e438..ff115143f 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -55,11 +55,12 @@ struct GitRepo
            in the repo yet. */
         std::optional<Hash> headRev;
 
-        enum State { Clean, Dirty };
-
         /* All files in the working directory that are unchanged,
            modified or added, but excluding deleted files. */
-        std::map<CanonPath, State> files;
+        std::set<CanonPath> files;
+
+        /* All modified or added files. */
+        std::set<CanonPath> dirtyFiles;
 
         /* The deleted files. */
         std::set<CanonPath> deletedFiles;
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index a584542d1..c73f53765 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -686,7 +686,7 @@ struct GitInputScheme : InputScheme
         if (getSubmodulesAttr(input))
             /* Create mountpoints for the submodules. */
             for (auto & submodule : repoInfo.workdirInfo.submodules)
-                repoInfo.workdirInfo.files.emplace(submodule.path, GitRepo::WorkdirInfo::State::Clean);
+                repoInfo.workdirInfo.files.insert(submodule.path);
 
         auto repo = GitRepo::openRepo(repoInfo.url, false, false);
 
@@ -807,12 +807,11 @@ struct GitInputScheme : InputScheme
                 /* Calculate a fingerprint that takes into account the
                    deleted and modified/added files. */
                 HashSink hashSink{HashAlgorithm::SHA512};
-                for (auto & file : repoInfo.workdirInfo.files)
-                    if (file.second == GitRepo::WorkdirInfo::State::Dirty) {
-                        writeString("modified:", hashSink);
-                        writeString(file.first.abs(), hashSink);
-                        dumpPath(repoInfo.url + "/" + file.first.abs(), hashSink);
-                    }
+                for (auto & file : repoInfo.workdirInfo.dirtyFiles) {
+                    writeString("modified:", hashSink);
+                    writeString(file.abs(), hashSink);
+                    dumpPath(repoInfo.url + "/" + file.abs(), hashSink);
+                }
                 for (auto & file : repoInfo.workdirInfo.deletedFiles) {
                     writeString("deleted:", hashSink);
                     writeString(file.abs(), hashSink);