Hybrid task with nml

app/controllers/TaskController.scala

@@ -2,30 +2,29 @@ package controllers
 
 import java.io.File
 
-import javax.inject.Inject
+import com.mohiva.play.silhouette.api.Silhouette
+import com.mohiva.play.silhouette.api.actions.SecuredRequest
+import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.geometry.{BoundingBox, Point3D, Vector3D}
 import com.scalableminds.util.mvc.ResultBox
-import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper}
 import com.scalableminds.webknossos.tracingstore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings}
+import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.tracingstore.tracings.{ProtoGeometryImplicits, TracingType}
-import models.annotation.nml.{NmlResults, NmlService}
-import models.annotation.{AnnotationService, TracingStoreService}
+import javax.inject.Inject
+import models.annotation.nml.NmlResults.NmlParseResult
+import models.annotation.nml.NmlService
+import models.annotation.{Annotation, AnnotationDAO, AnnotationService, TracingStoreRpcClient, TracingStoreService}
 import models.binary.{DataSetDAO, DataSetService}
 import models.project.ProjectDAO
 import models.task._
 import models.team.TeamDAO
 import models.user._
-import net.liftweb.common.Box
+import net.liftweb.common.{Box, Full}
 import oxalis.security.WkEnv
-import com.mohiva.play.silhouette.api.Silhouette
-import com.mohiva.play.silhouette.api.actions.{SecuredRequest, UserAwareRequest}
-import com.scalableminds.webknossos.tracingstore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
-import models.annotation.nml.NmlResults.NmlParseResult
-import play.api.libs.Files
-import play.api.i18n.{Messages, MessagesApi, MessagesProvider}
+import play.api.i18n.{Messages, MessagesProvider}
 import play.api.libs.json._
-import play.api.mvc.{MultipartFormData, PlayBodyParsers, Result}
+import play.api.mvc.{PlayBodyParsers, Result}
 import utils.{ObjectId, WkConf}
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -41,7 +40,8 @@ case class TaskParameters(
     editPosition: Point3D,
     editRotation: Vector3D,
     creationInfo: Option[String],
-    description: Option[String]
+    description: Option[String],
+    baseAnnotation: Option[BaseAnnotation]
 )
 
 object TaskParameters {
@@ -59,7 +59,14 @@ object NmlTaskParameters {
   implicit val nmlTaskParametersFormat: Format[NmlTaskParameters] = Json.format[NmlTaskParameters]
 }
 
-class TaskController @Inject()(annotationService: AnnotationService,
+case class BaseAnnotation(baseId: String, skeletonId: Option[String] = None, volumeId: Option[String] = None) // baseId is the id of the old Annotation which should be used as base for the new annotation, skeletonId/volumeId are the ids of the dupliated tracings from baseId
+
+object BaseAnnotation {
+  implicit val baseAnnotationFormat: Format[BaseAnnotation] = Json.format[BaseAnnotation]
+}
+
+class TaskController @Inject()(annotationDAO: AnnotationDAO,
+                               annotationService: AnnotationService,
                                scriptDAO: ScriptDAO,
                                projectDAO: ProjectDAO,
                                taskTypeDAO: TaskTypeDAO,
@@ -92,21 +99,83 @@ class TaskController @Inject()(annotationService: AnnotationService,
   def create = sil.SecuredAction.async(validateJson[List[TaskParameters]]) { implicit request =>
     for {
       _ <- bool2Fox(request.body.length <= 1000) ?~> "task.create.limitExceeded"
-      skeletonBaseOpts: List[Option[SkeletonTracing]] <- createTaskSkeletonTracingBases(request.body)
-      volumeBaseOpts: List[Option[VolumeTracing]] <- createTaskVolumeTracingBases(request.body,
+      taskParameters <- duplicateAllBaseTracings(request.body, request.identity._organization)
+      skeletonBaseOpts: List[Option[SkeletonTracing]] <- createTaskSkeletonTracingBases(taskParameters)
+      volumeBaseOpts: List[Option[VolumeTracing]] <- createTaskVolumeTracingBases(taskParameters,
                                                                                   request.identity._organization)
-      result <- createTasks((request.body, skeletonBaseOpts, volumeBaseOpts).zipped.toList)
+      result <- createTasks((taskParameters, skeletonBaseOpts, volumeBaseOpts).zipped.toList)
     } yield result
   }
 
+  def duplicateAllBaseTracings(taskParametersList: List[TaskParameters],
+                               organizationId: ObjectId)(implicit ctx: DBAccessContext, m: MessagesProvider) =
+    Fox.serialCombined(taskParametersList)(
+      params =>
+        Fox
+          .runOptional(params.baseAnnotation)(duplicateBaseTracings(_, params, organizationId))
+          .map(baseAnnotation => params.copy(baseAnnotation = baseAnnotation)))
+
+  private def duplicateSkeletonTracingOrCreateSkeletonTracingBase(
+      annotation: Annotation,
+      params: TaskParameters,
+      tracingStoreClient: TracingStoreRpcClient): Fox[String] =
+    annotation.skeletonTracingId
+      .map(id => tracingStoreClient.duplicateSkeletonTracing(id))
+      .getOrElse(
+        tracingStoreClient.saveSkeletonTracing(
+          annotationService.createSkeletonTracingBase(
+            params.dataSet,
+            params.boundingBox,
+            params.editPosition,
+            params.editRotation
+          )))
+
+  private def duplicateVolumeTracingOrCreateVolumeTracingBase(
+      annotation: Annotation,
+      params: TaskParameters,
+      tracingStoreClient: TracingStoreRpcClient,
+      organizationId: ObjectId)(implicit ctx: DBAccessContext, m: MessagesProvider): Fox[String] =
+    annotation.volumeTracingId
+      .map(id => tracingStoreClient.duplicateVolumeTracing(id))
+      .getOrElse(
+        annotationService
+          .createVolumeTracingBase(
+            params.dataSet,
+            organizationId,
+            params.boundingBox,
+            params.editPosition,
+            params.editRotation,
+            false
+          )
+          .flatMap(tracingStoreClient.saveVolumeTracing(_)))
+
+  def duplicateBaseTracings(baseAnnotation: BaseAnnotation, taskParameters: TaskParameters, organizationId: ObjectId)(
+      implicit ctx: DBAccessContext,
+      m: MessagesProvider) =
+    for {
+      taskTypeIdValidated <- ObjectId.parse(taskParameters.taskTypeId) ?~> "taskType.id.invalid"
+      taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound"
+      dataSet <- dataSetDAO.findOneByNameAndOrganization(taskParameters.dataSet, organizationId)
+      baseAnnotationIdValidated <- ObjectId.parse(baseAnnotation.baseId)
+      annotation <- annotationDAO.findOne(baseAnnotationIdValidated)
+      tracingStoreClient <- tracingStoreService.clientFor(dataSet)
+      newSkeletonId <- if (taskType.tracingType == TracingType.skeleton || taskType.tracingType == TracingType.hybrid)
+        duplicateSkeletonTracingOrCreateSkeletonTracingBase(annotation, taskParameters, tracingStoreClient).map(Some(_))
+      else Fox.successful(None)
+      newVolumeId <- if (taskType.tracingType == TracingType.volume || taskType.tracingType == TracingType.hybrid)
+        duplicateVolumeTracingOrCreateVolumeTracingBase(annotation, taskParameters, tracingStoreClient, organizationId)
+          .map(Some(_))
+      else Fox.successful(None)
+    } yield BaseAnnotation(baseAnnotationIdValidated.id, newSkeletonId, newVolumeId)
+
   def createTaskSkeletonTracingBases(paramsList: List[TaskParameters])(
       implicit ctx: DBAccessContext,
       m: MessagesProvider): Fox[List[Option[SkeletonTracing]]] =
     Fox.serialCombined(paramsList) { params =>
       for {
         taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) ?~> "taskType.id.invalid"
         taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" ~> NOT_FOUND
-        skeletonTracingOpt <- if (taskType.tracingType == TracingType.skeleton || taskType.tracingType == TracingType.hybrid) {
+        skeletonTracingOpt <- if ((taskType.tracingType == TracingType.skeleton || taskType.tracingType == TracingType.hybrid) && params.baseAnnotation.isEmpty) {
           Fox.successful(
             Some(
               annotationService.createSkeletonTracingBase(
@@ -126,7 +195,7 @@ class TaskController @Inject()(annotationService: AnnotationService,
       for {
         taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) ?~> "taskType.id.invalid"
         taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" ~> NOT_FOUND
-        volumeTracingOpt <- if (taskType.tracingType == TracingType.volume || taskType.tracingType == TracingType.hybrid) {
+        volumeTracingOpt <- if ((taskType.tracingType == TracingType.volume || taskType.tracingType == TracingType.hybrid) && params.baseAnnotation.isEmpty) {
           annotationService
             .createVolumeTracingBase(
               params.dataSet,
@@ -154,16 +223,18 @@ class TaskController @Inject()(annotationService: AnnotationService,
       params <- JsonHelper.parseJsonToFox[NmlTaskParameters](jsonString) ?~> "task.create.failed"
       taskTypeIdValidated <- ObjectId.parse(params.taskTypeId) ?~> "taskType.id.invalid"
       taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" ~> NOT_FOUND
-      _ <- bool2Fox(taskType.tracingType == TracingType.skeleton) ?~> "task.create.fromFileVolume"
+      _ <- bool2Fox(taskType.tracingType == TracingType.skeleton || taskType.tracingType == TracingType.hybrid) ?~> "task.create.fromFileVolume"
       project <- projectDAO
         .findOneByName(params.projectName) ?~> Messages("project.notFound", params.projectName) ~> NOT_FOUND
       _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(request.identity, project._team))
       parseResults: List[NmlParseResult] = nmlService
         .extractFromFiles(inputFiles.map(f => (new File(f.ref.path.toString), f.filename)))
         .parseResults
       skeletonSuccesses <- Fox.serialCombined(parseResults)(_.toSkeletonSuccessFox) ?~> "task.create.failed"
-      result <- createTasks(skeletonSuccesses.map(s =>
-        (buildFullParams(params, s.skeletonTracing.get, s.fileName, s.description), s.skeletonTracing, None)))
+      fullParams = skeletonSuccesses.map(s => buildFullParams(params, s.skeletonTracing.get, s.fileName, s.description))
+      skeletonBaseOpts = skeletonSuccesses.map(_.skeletonTracing)
+      volumeBaseOpts <- createTaskVolumeTracingBases(fullParams, request.identity._organization)
+      result <- createTasks((fullParams, skeletonBaseOpts, volumeBaseOpts).zipped.toList)
     } yield {
       result
     }
@@ -186,16 +257,22 @@ class TaskController @Inject()(annotationService: AnnotationService,
       tracing.editPosition,
       tracing.editRotation,
       Some(fileName),
-      description
+      description,
+      None
     )
   }
 
+  private def mergeTracingIds(list: List[(TaskParameters, Box[Option[String]])], isSkeletonId: Boolean) =
+    list.map { tuple =>
+      tuple._1.baseAnnotation.map(bA => Full(if (isSkeletonId) bA.skeletonId else bA.volumeId)).getOrElse(tuple._2)
+    }
+
   def createTasks(requestedTasks: List[(TaskParameters, Option[SkeletonTracing], Option[VolumeTracing])])(
       implicit request: SecuredRequest[WkEnv, _]): Fox[Result] = {
 
     def assertEachHasEitherSkeletonOrVolume: Fox[Boolean] =
       bool2Fox(requestedTasks.forall { tuple =>
-        tuple._2.isDefined || tuple._3.isDefined
+        tuple._1.baseAnnotation.isDefined || tuple._2.isDefined || tuple._3.isDefined
       })
 
     def assertAllOnSameDataset(firstDatasetName: String): Fox[String] = {
@@ -232,10 +309,12 @@ class TaskController @Inject()(annotationService: AnnotationService,
         SkeletonTracings(requestedTasks.map(taskTuple => SkeletonTracingOpt(taskTuple._2))))
       volumeTracingIds: List[Box[Option[String]]] <- tracingStoreClient.saveVolumeTracings(
         VolumeTracings(requestedTasks.map(taskTuple => VolumeTracingOpt(taskTuple._3))))
-      requestedTasksWithTracingIds = (requestedTasks, skeletonTracingIds, volumeTracingIds).zipped.toList
+      skeletonTracingsIdsMerged = mergeTracingIds((requestedTasks.map(_._1), skeletonTracingIds).zipped.toList, true)
+      volumeTracingsIdsMerged = mergeTracingIds((requestedTasks.map(_._1), volumeTracingIds).zipped.toList, false)
+      requestedTasksWithTracingIds = (requestedTasks, skeletonTracingsIdsMerged, volumeTracingsIdsMerged).zipped.toList
       taskObjects: List[Fox[Task]] = requestedTasksWithTracingIds.map(r =>
         createTaskWithoutAnnotationBase(r._1._1, r._2, r._3))
-      zipped = (requestedTasks, skeletonTracingIds.zip(volumeTracingIds), taskObjects).zipped.toList
+      zipped = (requestedTasks, skeletonTracingsIdsMerged.zip(volumeTracingsIdsMerged), taskObjects).zipped.toList
       annotationBases = zipped.map(
         tuple =>
           annotationService.createAnnotationBase(

app/models/annotation/AnnotationService.scala

@@ -238,7 +238,7 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor
       implicit ctx: DBAccessContext,
       m: MessagesProvider): Fox[(Option[String], Option[String])] =
     for {
-      dataSource <- bool2Fox(dataSet.isUsable) ?~> Messages("dataSet.notImported", dataSet.name)
+      _ <- bool2Fox(dataSet.isUsable) ?~> Messages("dataSet.notImported", dataSet.name)
       tracingStoreClient <- tracingStoreService.clientFor(dataSet)
       newSkeletonId: Option[String] <- Fox.runOptional(annotationBase.skeletonTracingId)(skeletonId =>
         tracingStoreClient.duplicateSkeletonTracing(skeletonId))
@@ -345,7 +345,6 @@ class AnnotationService @Inject()(annotationInformationProvider: AnnotationInfor
       skeletonIdOpt <- skeletonTracingIdBox.toFox
       volumeIdOpt <- volumeTracingIdBox.toFox
       _ <- bool2Fox(skeletonIdOpt.isDefined || volumeIdOpt.isDefined) ?~> "annotation.needsAtleastOne"
-      taskType <- taskTypeDAO.findOne(task._taskType)(GlobalAccessContext)
       project <- projectDAO.findOne(task._project)
       annotationBase = Annotation(ObjectId.generate,
                                   dataSetId,

conf/messages

@@ -221,7 +221,7 @@ task.create.success=Task successfully created
 task.create.failed=Failed to create Task
 task.create.limitExceeded=Cannot create more than 1000 tasks in one request.
 task.create.needsEitherSkeletonOrVolume=Each task needs to either be skeleton or volume.
-task.create.fromFileVolume=Task creation with file upload is only supported for skeleton task types.
+task.create.fromFileVolume=Task creation with file upload is not supported for volume task types.
 task.finished=Task is finished
 task.assigned=You got a new task
 task.tooManyOpenOnes=You already have too many open tasks

frontend/javascripts/admin/admin_rest_api.js

@@ -560,9 +560,10 @@ export function copyAnnotationToUserAccount(
 export function getAnnotationInformation(
   annotationId: string,
   annotationType: APIAnnotationType,
+  options?: RequestOptions = {},
 ): Promise<APIAnnotation> {
   const infoUrl = `/api/annotations/${annotationType}/${annotationId}/info`;
-  return Request.receiveJSON(infoUrl);
+  return Request.receiveJSON(infoUrl, options);
 }
 
 export function createExplorational(

frontend/javascripts/admin/api_flow_types.js

@@ -243,6 +243,8 @@ export const APIAnnotationTypeEnum = Enum.make({
 
 export type APIAnnotationType = $Keys<typeof APIAnnotationTypeEnum>;
 
+export type TracingType = "skeleton" | "volume" | "hybrid";
+
 export type APITaskType = {
   +id: string,
   +summary: string,
@@ -251,11 +253,9 @@ export type APITaskType = {
   +teamName: string,
   +settings: APISettings,
   +recommendedConfiguration: ?RecommendedConfiguration,
-  +tracingType: "skeleton" | "volume",
+  +tracingType: TracingType,
 };
 
-export type TracingType = "skeleton" | "volume" | "hybrid";
-
 export type TaskStatus = { +open: number, +active: number, +finished: number };
 
 type APIScriptTypeBase = {

frontend/javascripts/admin/task/task_create_bulk_view.js

@@ -26,7 +26,7 @@ type State = {
   tasksProcessed: number,
 };
 
-export type NewTask = {
+export type NewTask = {|
   +boundingBox: ?BoundingBoxObject,
   +dataSet: string,
   +editPosition: Vector3,
@@ -42,7 +42,10 @@ export type NewTask = {
   +taskTypeId: string,
   +csvFile?: File,
   +nmlFiles?: File,
-};
+  +baseAnnotation?: ?{
+    baseId: string,
+  },
+|};
 
 export type TaskCreationResponse = {
   status: number,
@@ -101,10 +104,7 @@ class TaskCreateBulkView extends React.PureComponent<Props, State> {
   }
 
   splitToWords(string: string): Array<string> {
-    return string
-      .split(",")
-      .map(word => word.trim())
-      .filter(word => word !== "");
+    return string.split(",").map(word => word.trim());
   }
 
   parseText(bulkText: string): Array<NewTask> {
@@ -135,7 +135,13 @@ class TaskCreateBulkView extends React.PureComponent<Props, State> {
     const height = parseInt(words[16]);
     const depth = parseInt(words[17]);
     const projectName = words[18];
-    const scriptId = words[19] || undefined;
+
+    // mapOptional takes care of treating empty strings as null
+    function mapOptional<U>(word, fn: string => U): ?U {
+      return word != null && word !== "" ? fn(word) : undefined;
+    }
+    const scriptId = mapOptional(words[19], a => a);
+    const baseAnnotation = mapOptional(words[20], word => ({ baseId: word }));
 
     // BoundingBox is optional and can be set to null by using the format [0, 0, 0, 0, 0, 0]
     const boundingBox =
@@ -162,7 +168,7 @@ class TaskCreateBulkView extends React.PureComponent<Props, State> {
       },
       editPosition: [x, y, z],
       editRotation: [rotX, rotY, rotZ],
-      isForAnonymous: false,
+      baseAnnotation,
     };
   }
 
@@ -265,7 +271,12 @@ class TaskCreateBulkView extends React.PureComponent<Props, State> {
               <a href="/dashboard">dataSet</a>, <a href="/taskTypes">taskTypeId</a>,{" "}
               experienceDomain, minExperience, x, y, z, rotX, rotY, rotZ, instances,{" "}
               <a href="/teams">team</a>, minX, minY, minZ, width, height, depth,{" "}
-              <a href="/projects">project</a> [, <a href="/scripts">scriptId</a>]
+              <a href="/projects">project</a> [, <a href="/scripts">scriptId</a>, baseAnnotationId]
+              <br />
+              If you want to define some (but not all) of the optional values, please list all
+              optional values and use an empty value for the ones you do not want to set (e.g.,
+              someValue,,someOtherValue if you want to omit the second value). If you don not want
+              to define a bounding box, you may use 0, 0, 0, 0, 0, 0 for the corresponding values.
             </p>
             <Form onSubmit={this.handleSubmit} layout="vertical">
               <FormItem label="Bulk Task Specification" hasFeedback>
@@ -289,7 +300,7 @@ class TaskCreateBulkView extends React.PureComponent<Props, State> {
                 })(
                   <TextArea
                     className="input-monospace"
-                    placeholder="dataSet, taskTypeId, experienceDomain, minExperience, x, y, z, rotX, rotY, rotZ, instances, team, minX, minY, minZ, width, height, depth, project[, scriptId]"
+                    placeholder="dataSet, taskTypeId, experienceDomain, minExperience, x, y, z, rotX, rotY, rotZ, instances, team, minX, minY, minZ, width, height, depth, project[, scriptId, baseAnnotationId]"
                     autosize={{ minRows: 6 }}
                     style={{
                       fontFamily: 'Monaco, Consolas, "Lucida Console", "Courier New", monospace',