Merge remote-tracking branch 'upstream/master' into lfs

src/libstore/package.nix

@@ -1,25 +1,26 @@
-{ lib
-, stdenv
-, mkMesonLibrary
+{
+  lib,
+  stdenv,
+  mkMesonLibrary,
 
-, unixtools
-, darwin
+  unixtools,
+  darwin,
 
-, nix-util
-, boost
-, curl
-, aws-sdk-cpp
-, libseccomp
-, nlohmann_json
-, sqlite
+  nix-util,
+  boost,
+  curl,
+  aws-sdk-cpp,
+  libseccomp,
+  nlohmann_json,
+  sqlite,
 
-, busybox-sandbox-shell ? null
+  busybox-sandbox-shell ? null,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 
-, embeddedSandboxShell ? stdenv.hostPlatform.isStatic
+  embeddedSandboxShell ? stdenv.hostPlatform.isStatic,
 }:
 
 let
@@ -48,19 +49,20 @@ mkMesonLibrary (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "sql") ./.)
   ];
 
-  nativeBuildInputs =
-    lib.optional embeddedSandboxShell unixtools.hexdump;
+  nativeBuildInputs = lib.optional embeddedSandboxShell unixtools.hexdump;
 
-  buildInputs = [
-    boost
-    curl
-    sqlite
-  ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp
+  buildInputs =
+    [
+      boost
+      curl
+      sqlite
+    ]
+    ++ lib.optional stdenv.hostPlatform.isLinux libseccomp
     # There have been issues building these dependencies
     ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox
-    ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
-      aws-sdk-cpp
-  ;
+    ++ lib.optional (
+      stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)
+    ) aws-sdk-cpp;
 
   propagatedBuildInputs = [
     nix-util
@@ -75,12 +77,14 @@ mkMesonLibrary (finalAttrs: {
       echo ${version} > ../../.version
     '';
 
-  mesonFlags = [
-    (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux)
-    (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell)
-  ] ++ lib.optionals stdenv.hostPlatform.isLinux [
-    (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox")
-  ];
+  mesonFlags =
+    [
+      (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux)
+      (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell)
+    ]
+    ++ lib.optionals stdenv.hostPlatform.isLinux [
+      (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox")
+    ];
 
   env = {
     # Needed for Meson to find Boost.

src/libstore/remote-store.cc

@@ -539,11 +539,21 @@ void RemoteStore::addMultipleToStore(
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
+    // `addMultipleToStore` is single threaded
+    size_t bytesExpected = 0;
+    for (auto & [pathInfo, _] : pathsToCopy) {
+        bytesExpected += pathInfo.narSize;
+    }
+    act.setExpected(actCopyPath, bytesExpected);
+
     auto source = sinkToSource([&](Sink & sink) {
-        sink << pathsToCopy.size();
+        size_t nrTotal = pathsToCopy.size();
+        sink << nrTotal;
         // Reverse, so we can release memory at the original start
         std::reverse(pathsToCopy.begin(), pathsToCopy.end());
         while (!pathsToCopy.empty()) {
+            act.progress(nrTotal - pathsToCopy.size(), nrTotal, size_t(1), size_t(0));
+
             auto & [pathInfo, pathSource] = pathsToCopy.back();
             WorkerProto::Serialise<ValidPathInfo>::write(*this,
                  WorkerProto::WriteConn {

src/libstore/store-api.cc

@@ -242,8 +242,8 @@ void Store::addMultipleToStore(
         storePathsToAdd.insert(thingToAdd.first.path);
     }
 
-    auto showProgress = [&]() {
-        act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
+    auto showProgress = [&, nrTotal = pathsToCopy.size()]() {
+        act.progress(nrDone, nrTotal, nrRunning, nrFailed);
     };
 
     processGraph<StorePath>(
@@ -1104,9 +1104,6 @@ std::map<StorePath, StorePath> copyPaths(
         return storePathForDst;
     };
 
-    // total is accessed by each copy, which are each handled in separate threads
-    std::atomic<uint64_t> total = 0;
-
     for (auto & missingPath : sortedMissing) {
         auto info = srcStore.queryPathInfo(missingPath);
 
@@ -1116,9 +1113,10 @@ std::map<StorePath, StorePath> copyPaths(
         ValidPathInfo infoForDst = *info;
         infoForDst.path = storePathForDst;
 
-        auto source = sinkToSource([&](Sink & sink) {
+        auto source = sinkToSource([&, narSize = info->narSize](Sink & sink) {
             // We can reasonably assume that the copy will happen whenever we
             // read the path, so log something about that at that point
+            uint64_t total = 0;
             auto srcUri = srcStore.getUri();
             auto dstUri = dstStore.getUri();
             auto storePathS = srcStore.printStorePath(missingPath);
@@ -1129,13 +1127,13 @@ std::map<StorePath, StorePath> copyPaths(
 
             LambdaSink progressSink([&](std::string_view data) {
                 total += data.size();
-                act.progress(total, info->narSize);
+                act.progress(total, narSize);
             });
             TeeSink tee { sink, progressSink };
 
             srcStore.narFromPath(missingPath, tee);
         });
-        pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
+        pathsToCopy.emplace_back(std::move(infoForDst), std::move(source));
     }
 
     dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);

src/libstore/unix/build/local-derivation-goal.cc

@@ -2565,7 +2565,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 case FileIngestionMethod::Git: {
                     return git::dumpHash(
                         outputHash.hashAlgo,
-                        {getFSSourceAccessor(), CanonPath(tmpDir + "/tmp")}).hash;
+                        {getFSSourceAccessor(), CanonPath(actualPath)}).hash;
                 }
                 }
                 assert(false);
@@ -2657,10 +2657,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                             wanted.to_string(HashFormat::SRI, true),
                             got.to_string(HashFormat::SRI, true)));
                 }
-                if (!newInfo0.references.empty())
+                if (!newInfo0.references.empty()) {
+                    auto numViolations = newInfo.references.size();
                     delayedException = std::make_exception_ptr(
-                        BuildError("illegal path references in fixed-output derivation '%s'",
-                            worker.store.printStorePath(drvPath)));
+                        BuildError("fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'",
+                            worker.store.printStorePath(drvPath),
+                            numViolations,
+                            worker.store.printStorePath(*newInfo.references.begin())));
+                }
 
                 return newInfo0;
             },