Improve Volume Project Handling

CHANGELOG.md

@@ -10,10 +10,10 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 [Commits](https://github.com/scalableminds/webknossos/compare/19.07.0...HEAD)
 
 ### Added
--
+- Volume tasks with only one finished instance can now be viewed as CompoundTask. [#4167](https://github.com/scalableminds/webknossos/pull/4167)
 
 ### Changed
--
+- Volume project download zips are reorganized to contain a zipfile for each annotation (that in turn contains a data.zip and an nml file). [#4167](https://github.com/scalableminds/webknossos/pull/4167)
 
 ### Fixed
 - Fixed a bug where volume tracings could not be converted to hybrid. [#4159](https://github.com/scalableminds/webknossos/pull/4159)

app/models/annotation/AnnotationMerger.scala

@@ -45,31 +45,59 @@ class AnnotationMerger @Inject()(dataSetDAO: DataSetDAO, tracingStoreService: Tr
       Fox.empty
     else {
       for {
-        mergedTracingReference <- mergeTracingsOfAnnotations(annotations, _dataSet, persistTracing)
+        (mergedSkeletonTracingReference, mergedVolumeTracingReference) <- mergeTracingsOfAnnotations(annotations,
+                                                                                                     _dataSet,
+                                                                                                     persistTracing)
       } yield {
         Annotation(
           newId,
           _dataSet,
           None,
           _team,
           _user,
-          Some(mergedTracingReference),
-          None,
+          mergedSkeletonTracingReference,
+          mergedVolumeTracingReference,
           typ = typ
         )
       }
     }
 
   private def mergeTracingsOfAnnotations(annotations: List[Annotation], dataSetId: ObjectId, persistTracing: Boolean)(
-      implicit ctx: DBAccessContext): Fox[String] =
+      implicit ctx: DBAccessContext): Fox[(Option[String], Option[String])] =
     for {
       dataSet <- dataSetDAO.findOne(dataSetId)
-      tracingStoreClient <- tracingStoreService.clientFor(dataSet)
-      _ <- bool2Fox(annotations.flatMap(_.volumeTracingId).isEmpty) ?~> "annotation.cantMergeVolumes"
-      skeletonTracingIds = annotations.map(_.skeletonTracingId)
-      tracingReference <- tracingStoreClient.mergeSkeletonTracingsByIds(skeletonTracingIds, persistTracing) ?~> "Failed to merge skeleton tracings."
+      tracingStoreClient: TracingStoreRpcClient <- tracingStoreService.clientFor(dataSet)
+      (skeletonTracingReference, volumeTracingReference) <- mergeOrDuplicate(tracingStoreClient,
+                                                                             persistTracing,
+                                                                             annotations)
     } yield {
-      tracingReference
+      (skeletonTracingReference, volumeTracingReference)
     }
 
+  private def mergeOrDuplicate(tracingStoreClient: TracingStoreRpcClient,
+                               persistTracing: Boolean,
+                               annotations: List[Annotation]): Fox[(Option[String], Option[String])] =
+    if (isSingleVolume(annotations))
+      for {
+        singleVolumeTracingId <- extractSingleVolumeTracingId(annotations) ?~> "Failed to duplicate volume tracing"
+        volumeTracingReference <- tracingStoreClient.duplicateVolumeTracing(singleVolumeTracingId) ?~> "Failed to duplicate volume tracing."
+      } yield (None, Some(volumeTracingReference))
+    else
+      for {
+        _ <- bool2Fox(containsNoVolumes(annotations))
+        skeletonTracingIds = annotations.map(_.skeletonTracingId)
+        skeletonTracingReference <- tracingStoreClient.mergeSkeletonTracingsByIds(skeletonTracingIds, persistTracing) ?~> "Failed to merge skeleton tracings."
+      } yield (Some(skeletonTracingReference), None)
+
+  private def isSingleVolume(annotations: List[Annotation]): Boolean =
+    annotations.flatMap(_.skeletonTracingId).lengthCompare(0) == 0 && annotations
+      .flatMap(_.volumeTracingId)
+      .lengthCompare(1) == 0
+
+  private def containsNoVolumes(annotations: List[Annotation]): Boolean =
+    annotations.flatMap(_.volumeTracingId).isEmpty
+
+  private def extractSingleVolumeTracingId(annotations: List[Annotation]): Fox[String] =
+    annotations.flatMap(_.volumeTracingId).headOption.toFox
+
 }

app/models/annotation/AnnotationService.scala

@@ -521,9 +521,21 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor
 
     def addToZip(nmls: List[(Enumerator[Array[Byte]], String, Option[Array[Byte]])]): Future[Boolean] =
       nmls match {
-        case head :: tail => {
-          head._3.foreach(volumeData => zipper.addFileFromBytes(head._2 + "_data.zip", volumeData))
-          zipper.addFileFromEnumerator(head._2 + ".nml", head._1).flatMap(_ => addToZip(tail))
+        case (nml, name, volumeDataOpt) :: tail => {
+          if (volumeDataOpt.isDefined) {
+            val subZip = temporaryFileCreator.create(normalize(name), ".zip")
+            val subZipper =
+              ZipIO.startZip(new BufferedOutputStream(new FileOutputStream(new File(subZip.path.toString))))
+            volumeDataOpt.foreach(volumeData => subZipper.addFileFromBytes(name + "_data.zip", volumeData))
+            for {
+              _ <- subZipper.addFileFromEnumerator(name + ".nml", nml)
+              _ = subZipper.close()
+              _ = zipper.addFileFromTemporaryFile(name + ".zip", subZip)
+              res <- addToZip(tail)
+            } yield res
+          } else {
+            zipper.addFileFromEnumerator(name + ".nml", nml).flatMap(_ => addToZip(tail))
+          }
         }
         case _ =>
           Future.successful(true)

app/models/annotation/AnnotationStore.scala

@@ -23,7 +23,7 @@ class AnnotationStore @Inject()(
 
   case class StoredResult(result: Fox[Annotation], timestamp: Long = System.currentTimeMillis)
 
-  def requestAnnotation(id: AnnotationIdentifier, user: Option[User])(implicit ctx: DBAccessContext) =
+  def requestAnnotation(id: AnnotationIdentifier, user: Option[User])(implicit ctx: DBAccessContext): Fox[Annotation] =
     requestFromCache(id).getOrElse(requestFromHandler(id, user)).futureBox.recover {
       case e =>
         logger.error("AnnotationStore ERROR: " + e)

util/src/main/scala/com/scalableminds/util/io/ZipIO.scala

@@ -1,7 +1,7 @@
 package com.scalableminds.util.io
 
 import java.io._
-import java.nio.file.{Path, Paths}
+import java.nio.file.{Files, Path, Paths}
 import java.util.zip.{GZIPOutputStream => DefaultGZIPOutputStream, _}
 
 import akka.stream.{ActorMaterializer, scaladsl}
@@ -15,6 +15,7 @@ import net.liftweb.util.Helpers.tryo
 
 import scala.concurrent.duration._
 import org.apache.commons.io.IOUtils
+import play.api.libs.Files.TemporaryFile
 import play.api.libs.iteratee.{Enumerator, Iteratee}
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -48,6 +49,12 @@ object ZipIO extends LazyLogging {
       stream.closeEntry()
     }
 
+    def addFileFromTemporaryFile(name: String, data: TemporaryFile): Unit = {
+      stream.putNextEntry(new ZipEntry(name))
+      stream.write(Files.readAllBytes(data))
+      stream.closeEntry()
+    }
+
     def addFileFromEnumerator(name: String, data: Enumerator[Array[Byte]])(
         implicit ec: ExecutionContext): Future[Unit] = {
       stream.putNextEntry(new ZipEntry(name))