Merge pull request NixOS#12336 from NaN-git/fix-progress

libstore: Fix progress bars

Author: roberth

src/libstore/remote-store.cc

@@ -539,11 +539,21 @@ void RemoteStore::addMultipleToStore(
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
+    // `addMultipleToStore` is single threaded
+    size_t bytesExpected = 0;
+    for (auto & [pathInfo, _] : pathsToCopy) {
+        bytesExpected += pathInfo.narSize;
+    }
+    act.setExpected(actCopyPath, bytesExpected);
+
     auto source = sinkToSource([&](Sink & sink) {
-        sink << pathsToCopy.size();
+        size_t nrTotal = pathsToCopy.size();
+        sink << nrTotal;
         // Reverse, so we can release memory at the original start
         std::reverse(pathsToCopy.begin(), pathsToCopy.end());
         while (!pathsToCopy.empty()) {
+            act.progress(nrTotal - pathsToCopy.size(), nrTotal, size_t(1), size_t(0));
+
             auto & [pathInfo, pathSource] = pathsToCopy.back();
             WorkerProto::Serialise<ValidPathInfo>::write(*this,
                  WorkerProto::WriteConn {

src/libstore/store-api.cc

@@ -242,8 +242,8 @@ void Store::addMultipleToStore(
         storePathsToAdd.insert(thingToAdd.first.path);
     }
 
-    auto showProgress = [&]() {
-        act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
+    auto showProgress = [&, nrTotal = pathsToCopy.size()]() {
+        act.progress(nrDone, nrTotal, nrRunning, nrFailed);
     };
 
     processGraph<StorePath>(
@@ -1104,9 +1104,6 @@ std::map<StorePath, StorePath> copyPaths(
         return storePathForDst;
     };
 
-    // total is accessed by each copy, which are each handled in separate threads
-    std::atomic<uint64_t> total = 0;
-
     for (auto & missingPath : sortedMissing) {
         auto info = srcStore.queryPathInfo(missingPath);
 
@@ -1116,9 +1113,10 @@ std::map<StorePath, StorePath> copyPaths(
         ValidPathInfo infoForDst = *info;
         infoForDst.path = storePathForDst;
 
-        auto source = sinkToSource([&](Sink & sink) {
+        auto source = sinkToSource([&, narSize = info->narSize](Sink & sink) {
             // We can reasonably assume that the copy will happen whenever we
             // read the path, so log something about that at that point
+            uint64_t total = 0;
             auto srcUri = srcStore.getUri();
             auto dstUri = dstStore.getUri();
             auto storePathS = srcStore.printStorePath(missingPath);
@@ -1129,13 +1127,13 @@ std::map<StorePath, StorePath> copyPaths(
 
             LambdaSink progressSink([&](std::string_view data) {
                 total += data.size();
-                act.progress(total, info->narSize);
+                act.progress(total, narSize);
             });
             TeeSink tee { sink, progressSink };
 
             srcStore.narFromPath(missingPath, tee);
         });
-        pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
+        pathsToCopy.emplace_back(std::move(infoForDst), std::move(source));
     }
 
     dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);