Merge branch 'master' into reduce-save-requests

Author: MichaelBuessemeyer

CHANGELOG.unreleased.md

@@ -11,6 +11,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 [Commits](https://github.com/scalableminds/webknossos/compare/22.02.0...HEAD)
 
 ### Added
+- Added the option to make a segment's ID active via the right-click context menu in the segments list. [#5935](https://github.com/scalableminds/webknossos/pull/6006)
 - Added a button next to the histogram which adapts the contrast and brightness to the currently visible data. [#5961](https://github.com/scalableminds/webknossos/pull/5961)
 - Running uploads can now be cancelled. [#5958](https://github.com/scalableminds/webknossos/pull/5958)
 
@@ -21,11 +22,15 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Changed that webKnossos no longer tries to reach a save state where all updates are sent to the backend to be in sync with the frontend when the save is triggered by a timeout. [#5999](https://github.com/scalableminds/webknossos/pull/5999)
 - When changing which layers are visible in an annotation, this setting is persisted in the annotation, so when you share it, viewers will see the same visibility configuration. [#5967](https://github.com/scalableminds/webknossos/pull/5967)
 - Downloading public annotations is now also allowed without being authenticated. [#6001](https://github.com/scalableminds/webknossos/pull/6001)
+- Downloaded volume annotation layers no longer produce zero-byte zipfiles but rather a valid header-only zip file with no contents. [#6022](https://github.com/scalableminds/webknossos/pull/6022)
+- Changed a number of API routes from GET to POST to avoid unwanted side effects. [#6023](https://github.com/scalableminds/webknossos/pull/6023)
+- Removed unused datastore route `checkInbox` (use `checkInboxBlocking` instead). [#6023](https://github.com/scalableminds/webknossos/pull/6023)
 
 ### Fixed
 - Fixed volume-related bugs which could corrupt the volume data in certain scenarios. [#5955](https://github.com/scalableminds/webknossos/pull/5955)
 - Fixed the placeholder resolution computation for anisotropic layers with missing base resolutions. [#5983](https://github.com/scalableminds/webknossos/pull/5983)
 - Fixed a bug where ad-hoc meshes were computed for a mapping, although it was disabled. [#5982](https://github.com/scalableminds/webknossos/pull/5982)
+- Fixed a bug where volume annotation downloads would sometimes contain truncated zips. [#6009](https://github.com/scalableminds/webknossos/pull/6009)
 
 
 ### Removed

README.md

@@ -179,6 +179,9 @@ yarn run lint
 # Format frontend code
 yarn run pretty
 
+# Format backend code
+yarn pretty-backend
+
 # Frontend type checking
 yarn flow
 

app/controllers/AnnotationController.scala

@@ -355,17 +355,24 @@ class AnnotationController @Inject()(
       } yield JsonOk(Messages("annotation.edit.success"))
     }
 
-  @ApiOperation(hidden = true, value = "")
-  def annotationsForTask(taskId: String): Action[AnyContent] = sil.SecuredAction.async { implicit request =>
-    for {
-      taskIdValidated <- ObjectId.parse(taskId)
-      task <- taskDAO.findOne(taskIdValidated) ?~> "task.notFound" ~> NOT_FOUND
-      project <- projectDAO.findOne(task._project)
-      _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(request.identity, project._team))
-      annotations <- annotationService.annotationsFor(task._id) ?~> "task.annotation.failed"
-      jsons <- Fox.serialSequence(annotations)(a => annotationService.publicWrites(a, Some(request.identity)))
-    } yield Ok(JsArray(jsons.flatten))
-  }
+  @ApiOperation(value = "Information about all annotations for a specific task", nickname = "annotationInfosByTaskId")
+  @ApiResponses(
+    Array(
+      new ApiResponse(code = 200,
+                      message = "JSON list of objects containing information about the selected annotations."),
+      new ApiResponse(code = 400, message = badRequestLabel)
+    ))
+  def annotationsForTask(@ApiParam(value = "The id of the task") taskId: String): Action[AnyContent] =
+    sil.SecuredAction.async { implicit request =>
+      for {
+        taskIdValidated <- ObjectId.parse(taskId)
+        task <- taskDAO.findOne(taskIdValidated) ?~> "task.notFound" ~> NOT_FOUND
+        project <- projectDAO.findOne(task._project)
+        _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(request.identity, project._team))
+        annotations <- annotationService.annotationsFor(task._id) ?~> "task.annotation.failed"
+        jsons <- Fox.serialSequence(annotations)(a => annotationService.publicWrites(a, Some(request.identity)))
+      } yield Ok(JsArray(jsons.flatten))
+    }
 
   @ApiOperation(hidden = true, value = "")
   def cancel(typ: String, id: String): Action[AnyContent] = sil.SecuredAction.async { implicit request =>

app/controllers/AnnotationIOController.scala

@@ -1,10 +1,9 @@
 package controllers
 
-import java.io.File
+import java.io.{BufferedOutputStream, File, FileOutputStream}
 
 import akka.actor.ActorSystem
 import akka.stream.Materializer
-import akka.stream.scaladsl._
 import com.mohiva.play.silhouette.api.Silhouette
 import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.io.{NamedEnumeratorStream, ZipIO}
@@ -30,9 +29,7 @@ import models.task._
 import models.user._
 import oxalis.security.WkEnv
 import play.api.i18n.{Messages, MessagesProvider}
-import play.api.libs.Files.TemporaryFile
-import play.api.libs.iteratee.Enumerator
-import play.api.libs.iteratee.streams.IterateeStreams
+import play.api.libs.Files.{TemporaryFile, TemporaryFileCreator}
 import play.api.libs.json.Json
 import play.api.mvc.{Action, AnyContent, MultipartFormData}
 import utils.ObjectId
@@ -51,6 +48,7 @@ class AnnotationIOController @Inject()(
     taskDAO: TaskDAO,
     taskTypeDAO: TaskTypeDAO,
     tracingStoreService: TracingStoreService,
+    temporaryFileCreator: TemporaryFileCreator,
     annotationService: AnnotationService,
     analyticsService: AnalyticsService,
     sil: Silhouette[WkEnv],
@@ -223,8 +221,11 @@ Expects:
   @ApiOperation(value = "Download an annotation as NML/ZIP", nickname = "annotationDownload")
   @ApiResponses(
     Array(
-      new ApiResponse(code = 200,
-                      message = "NML or Zip file containing skeleton and/or volume data of this annotation."),
+      new ApiResponse(
+        code = 200,
+        message =
+          "NML or Zip file containing skeleton and/or volume data of this annotation. In case of Compound annotations, multiple such annotations wrapped in another zip"
+      ),
       new ApiResponse(code = 400, message = badRequestLabel)
     ))
   def download(
@@ -268,30 +269,34 @@ Expects:
                                     volumeVersion: Option[Long],
                                     skipVolumeData: Boolean)(implicit ctx: DBAccessContext) = {
 
-    def skeletonToDownloadStream(dataSet: DataSet, annotation: Annotation, name: String, organizationName: String) =
+    def skeletonToTemporaryFile(dataSet: DataSet,
+                                annotation: Annotation,
+                                organizationName: String): Fox[TemporaryFile] =
       for {
         tracingStoreClient <- tracingStoreService.clientFor(dataSet)
         fetchedAnnotationLayers <- Fox.serialCombined(annotation.skeletonAnnotationLayers)(
           tracingStoreClient.getSkeletonTracing(_, skeletonVersion))
         user <- userService.findOneById(annotation._user, useCache = true)
         taskOpt <- Fox.runOptional(annotation._task)(taskDAO.findOne)
-      } yield {
-        (nmlWriter.toNmlStream(fetchedAnnotationLayers,
-                               Some(annotation),
-                               dataSet.scale,
-                               None,
-                               organizationName,
-                               Some(user),
-                               taskOpt),
-         name + ".nml")
-      }
+        nmlStream = nmlWriter.toNmlStream(fetchedAnnotationLayers,
+                                          Some(annotation),
+                                          dataSet.scale,
+                                          None,
+                                          organizationName,
+                                          Some(user),
+                                          taskOpt)
+        nmlTemporaryFile = temporaryFileCreator.create()
+        temporaryFileStream = new BufferedOutputStream(new FileOutputStream(nmlTemporaryFile))
+        _ <- NamedEnumeratorStream("", nmlStream).writeTo(temporaryFileStream)
+        _ = temporaryFileStream.close()
+      } yield nmlTemporaryFile
 
-    def volumeOrHybridToDownloadStream(dataSet: DataSet,
-                                       annotation: Annotation,
-                                       name: String,
-                                       organizationName: String): Fox[(Enumerator[Array[Byte]], String)] =
+    def volumeOrHybridToTemporaryFile(dataset: DataSet,
+                                      annotation: Annotation,
+                                      name: String,
+                                      organizationName: String): Fox[TemporaryFile] =
       for {
-        tracingStoreClient <- tracingStoreService.clientFor(dataSet)
+        tracingStoreClient <- tracingStoreService.clientFor(dataset)
         fetchedVolumeLayers: List[FetchedAnnotationLayer] <- Fox.serialCombined(annotation.volumeAnnotationLayers) {
           volumeAnnotationLayer =>
             tracingStoreClient.getVolumeTracing(volumeAnnotationLayer, volumeVersion, skipVolumeData)
@@ -302,46 +307,61 @@ Expects:
         }
         user <- userService.findOneById(annotation._user, useCache = true)
         taskOpt <- Fox.runOptional(annotation._task)(taskDAO.findOne)
-      } yield {
-        val nmlStream = NamedEnumeratorStream(
-          name + ".nml",
-          nmlWriter.toNmlStream(fetchedSkeletonLayers ::: fetchedVolumeLayers,
-                                Some(annotation),
-                                dataSet.scale,
-                                None,
-                                organizationName,
-                                Some(user),
-                                taskOpt)
-        )
-        val dataStreams: List[NamedEnumeratorStream] =
-          fetchedVolumeLayers.zipWithIndex.flatMap {
-            case (volumeLayer, index) => volumeLayer.namedVolumeDataEnumerator(index, fetchedVolumeLayers.length == 1)
-          }
-        (Enumerator.outputStream { outputStream =>
-          ZipIO.zip(
-            nmlStream :: dataStreams,
-            outputStream
-          )
-        }, name + ".zip")
-      }
+        nmlStream = nmlWriter.toNmlStream(fetchedSkeletonLayers ::: fetchedVolumeLayers,
+                                          Some(annotation),
+                                          dataset.scale,
+                                          None,
+                                          organizationName,
+                                          Some(user),
+                                          taskOpt)
+        temporaryFile = temporaryFileCreator.create()
+        zipper = ZipIO.startZip(new BufferedOutputStream(new FileOutputStream(new File(temporaryFile.path.toString))))
+        _ <- zipper.addFileFromEnumerator(name + ".nml", nmlStream)
+        _ = fetchedVolumeLayers.zipWithIndex.map {
+          case (volumeLayer, index) =>
+            volumeLayer.volumeDataOpt.foreach { volumeData =>
+              val dataZipName = volumeLayer.volumeDataZipName(index, fetchedSkeletonLayers.length == 1)
+              zipper.addFileFromBytes(dataZipName, volumeData)
+            }
+        }
+        _ = zipper.close()
+      } yield temporaryFile
+
+    def annotationToTemporaryFile(dataSet: DataSet,
+                                  annotation: Annotation,
+                                  name: String,
+                                  organizationName: String): Fox[TemporaryFile] =
+      if (annotation.tracingType == TracingType.skeleton)
+        skeletonToTemporaryFile(dataSet, annotation, organizationName)
+      else
+        volumeOrHybridToTemporaryFile(dataSet, annotation, name, organizationName)
+
+    def exportExtensionForAnnotation(annotation: Annotation): String =
+      if (annotation.tracingType == TracingType.skeleton)
+        ".nml"
+      else
+        ".zip"
 
-    def tracingToDownloadStream(dataSet: DataSet, annotation: Annotation, name: String, organizationName: String) =
+    def exportMimeTypeForAnnotation(annotation: Annotation): String =
       if (annotation.tracingType == TracingType.skeleton)
-        skeletonToDownloadStream(dataSet, annotation, name, organizationName)
+        "application/xml"
       else
-        volumeOrHybridToDownloadStream(dataSet, annotation, name, organizationName)
+        "application/zip"
 
     for {
       annotation <- provider.provideAnnotation(typ, annotationId, issuingUser) ~> NOT_FOUND
       restrictions <- provider.restrictionsFor(typ, annotationId)
       name <- provider.nameFor(annotation) ?~> "annotation.name.impossible"
+      fileExtension = exportExtensionForAnnotation(annotation)
+      fileName = name + fileExtension
+      mimeType = exportMimeTypeForAnnotation(annotation)
       _ <- restrictions.allowDownload(issuingUser) ?~> "annotation.download.notAllowed" ~> FORBIDDEN
       dataSet <- dataSetDAO.findOne(annotation._dataSet)(GlobalAccessContext) ?~> "dataSet.notFoundForAnnotation" ~> NOT_FOUND
       organization <- organizationDAO.findOne(dataSet._organization)(GlobalAccessContext) ?~> "organization.notFound" ~> NOT_FOUND
-      (downloadStream, fileName) <- tracingToDownloadStream(dataSet, annotation, name, organization.name)
+      temporaryFile <- annotationToTemporaryFile(dataSet, annotation, name, organization.name)
     } yield {
-      Ok.chunked(Source.fromPublisher(IterateeStreams.enumeratorToPublisher(downloadStream)))
-        .as(if (fileName.toLowerCase.endsWith(".zip")) "application/zip" else "application/xml")
+      Ok.sendFile(temporaryFile, inline = false)
+        .as(mimeType)
         .withHeaders(CONTENT_DISPOSITION ->
           s"attachment;filename=${'"'}$fileName${'"'}")
     }