nix-copy-closure: Implement in C++.

Tests fail currently because the database is not given proper hashes in the VM

src/build-remote/build-remote.cc

@@ -9,7 +9,6 @@
 #include "shared.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
-#include "serve-protocol.hh"
 #include "serialise.hh"
 #include "store-api.hh"
 #include "derivations.hh"

src/build-remote/local.mk

@@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore
 
 build-remote_SOURCES := $(d)/build-remote.cc
 
-build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store
+build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""

src/libstore/ssh-store.cc

@@ -39,6 +39,8 @@ private:
     string uri;
 
     Path key;
+
+    bool compress;
 };
 
 SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
@@ -48,6 +50,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
     , socketPath((Path) tmpDir + "/ssh.sock")
     , uri(std::move(uri))
     , key(get(params, "ssh-key", ""))
+    , compress(get(params, "compress", "") == "true")
 {
     /* open a connection and perform the handshake to verify all is well */
     connections->get();
@@ -90,11 +93,12 @@ ref<FSAccessor> SSHStore::getFSAccessor()
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     if ((pid_t) sshMaster == -1) {
+        auto flags = compress ? "-NMCS" : "-NMS";
         sshMaster = startProcess([&]() {
             if (key.empty())
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL);
             else
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
             throw SysError("starting ssh master");
         });
     }

src/libstore/store-api.cc

@@ -4,6 +4,7 @@
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
+#include "derivations.hh"
 
 #include <future>
 
@@ -699,8 +700,27 @@ std::list<ref<Store>> getDefaultSubstituters()
 }
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths)
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
 {
+    if (substitute) {
+        /* Filter out .drv files (we don't want to build anything). */
+        PathSet paths2;
+        for (auto & path : storePaths)
+            if (!isDerivation(path)) paths2.insert(path);
+        unsigned long long downloadSize, narSize;
+        PathSet willBuild, willSubstitute, unknown;
+        to->queryMissing(PathSet(paths2.begin(), paths2.end()),
+            willBuild, willSubstitute, unknown, downloadSize, narSize);
+        /* FIXME: should use ensurePath(), but it only
+           does one path at a time. */
+        if (!willSubstitute.empty())
+            try {
+                to->buildPaths(willSubstitute);
+            } catch (Error & e) {
+                printMsg(lvlError, format("warning: %1%") % e.msg());
+            }
+    }
+
     std::string copiedLabel = "copied";
 
     logger->setExpected(copiedLabel, storePaths.size());

src/libstore/store-api.hh

@@ -625,7 +625,7 @@ void removeTempRoots();
 ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths);
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
 
 enum StoreType {
     tDaemon,

src/nix-copy-closure/local.mk

@@ -0,0 +1,7 @@
+programs += nix-copy-closure
+
+nix-copy-closure_DIR := $(d)
+
+nix-copy-closure_LIBS = libmain libutil libformat libstore
+
+nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc

src/nix-copy-closure/nix-copy-closure.cc

@@ -0,0 +1,60 @@
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+int main(int argc, char ** argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        auto gzip = false;
+        auto toMode = true;
+        auto includeOutputs = false;
+        auto dryRun = false;
+        auto useSubstitutes = false;
+        auto sshHost = string{};
+        auto storePaths = PathSet{};
+        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+            if (*arg == "--help")
+                showManPage("nix-copy-closure");
+            else if (*arg == "--version")
+                printVersion("nix-copy-closure");
+            else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
+                if (*arg != "--gzip")
+                    printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg);
+                gzip = true;
+            } else if (*arg == "--from")
+                toMode = false;
+            else if (*arg == "--to")
+                toMode = true;
+            else if (*arg == "--include-outputs")
+                includeOutputs = true;
+            else if (*arg == "--show-progress")
+                printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented");
+            else if (*arg == "--dry-run")
+                dryRun = true;
+            else if (*arg == "--use-substitutes" || *arg == "-s")
+                useSubstitutes = true;
+            else if (sshHost.empty())
+                sshHost = *arg;
+            else
+                storePaths.insert(*arg);
+            return true;
+        });
+        if (sshHost.empty())
+            throw UsageError("no host name specified");
+
+        auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
+        auto to = toMode ? openStore(remoteUri) : openStore();
+        auto from = toMode ? openStore() : openStore(remoteUri);
+        if (includeOutputs) {
+            auto newPaths = PathSet{};
+            for (const auto & p : storePaths) {
+                auto outputs = from->queryDerivationOutputs(p);
+                newPaths.insert(outputs.begin(), outputs.end());
+            }
+            storePaths.insert(newPaths.begin(), newPaths.end());
+        }
+        copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
+    });
+}