From cc838e81818fa6d78fb2e2686eb83dacac07d7d6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Jan 2025 14:23:02 +0100
Subject: [PATCH] addMultipleToStore(): Move pathsToCopy

This allows RemoteStore::addMultipleToStore() to free the Source
objects early (and in particular the associated sinkToSource()
buffers). This should fix #7359. For example, memory consumption of

  nix copy --derivation --to ssh-ng://localhost?remote-store=/tmp/nix --derivation --no-check-sigs \
    /nix/store/4p9xmfgnvclqpii8pxqcwcvl9bxqy2xf-nixos-system-...drv

went from 353 MB to 74 MB.
---
 src/libstore/remote-store.cc | 7 +++++--
 src/libstore/remote-store.hh | 2 +-
 src/libstore/store-api.cc    | 4 ++--
 src/libstore/store-api.hh    | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 69bbc64fc..87c58a690 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -534,14 +534,16 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 void RemoteStore::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
     auto source = sinkToSource([&](Sink & sink) {
         sink << pathsToCopy.size();
-        for (auto & [pathInfo, pathSource] : pathsToCopy) {
+        std::reverse(pathsToCopy.begin(), pathsToCopy.end());
+        while (!pathsToCopy.empty()) {
+            auto & [pathInfo, pathSource] = pathsToCopy.back();
             WorkerProto::Serialise<ValidPathInfo>::write(*this,
                  WorkerProto::WriteConn {
                      .to = sink,
@@ -549,6 +551,7 @@ void RemoteStore::addMultipleToStore(
                  },
                  pathInfo);
             pathSource->drainInto(sink);
+            pathsToCopy.pop_back();
         }
     });
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 4e1896268..ea6cd471e 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -102,7 +102,7 @@ public:
         CheckSigsFlag checkSigs) override;
 
     void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 3b5167730..6cd8e47f0 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -223,7 +223,7 @@ StorePath Store::addToStore(
 }
 
 void Store::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
@@ -1138,7 +1138,7 @@ std::map<StorePath, StorePath> copyPaths(
         pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
     }
 
-    dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
+    dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);
 
     return pathsMap;
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f45012061..474dffcb5 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -425,7 +425,7 @@ public:
         CheckSigsFlag checkSigs = CheckSigs);
 
     virtual void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair = NoRepair,
         CheckSigsFlag checkSigs = CheckSigs);