Add frontend and backend part for job to materialize a volume annotation

CHANGELOG.unreleased.md

@@ -12,6 +12,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Added
 - Added a long-running job that applies the merging done via a merger mode tracing to a new output dataset. The job is accessible via a button next to the merger mode button once the merger mode is active. [#6086](https://github.com/scalableminds/webknossos/pull/6086)
+ - Added support to stream zarr files using the corresponding [zarr spec](https://zarr.readthedocs.io/en/stable/spec/v2.html#storage). [#6144](https://github.com/scalableminds/webknossos/pull/6144)
 
 ### Changed
 

app/controllers/JobsController.scala

@@ -267,14 +267,15 @@ class JobsController @Inject()(jobDAO: JobDAO,
       }
     }
 
-  def runApplyMergerModeJob(organizationName: String,
-                            dataSetName: String,
-                            fallbackLayerName: String,
-                            annotationId: String,
-                            annotationType: String,
-                            newDatasetName: String,
-                            outputSegmentationLayerName: String,
-                            volumeLayerName: Option[String]): Action[AnyContent] =
+  def runMaterializeVolumeAnnotationJob(organizationName: String,
+                                        dataSetName: String,
+                                        fallbackLayerName: String,
+                                        annotationId: String,
+                                        annotationType: String,
+                                        newDatasetName: String,
+                                        outputSegmentationLayerName: String,
+                                        mergeSegments: Boolean,
+                                        volumeLayerName: Option[String]): Action[AnyContent] =
     sil.SecuredAction.async { implicit request =>
       log(Some(slackNotificationService.noticeFailedJobRequest)) {
         for {
@@ -286,7 +287,7 @@ class JobsController @Inject()(jobDAO: JobDAO,
             dataSetName) ~> NOT_FOUND
           userAuthToken <- wkSilhouetteEnvironment.combinedAuthenticatorService.findOrCreateToken(
             request.identity.loginInfo)
-          command = "apply_merger_mode"
+          command = "materialize_volume_annotation"
           commandArgs = Json.obj(
             "organization_name" -> organizationName,
             "dataset_name" -> dataSetName,
@@ -297,6 +298,7 @@ class JobsController @Inject()(jobDAO: JobDAO,
             "output_segmentation_layer_name" -> outputSegmentationLayerName,
             "annotation_type" -> annotationType,
             "new_dataset_name" -> newDatasetName,
+            "merge_segments" -> mergeSegments,
             "volume_layer_name" -> volumeLayerName
           )
           job <- jobService.submitJob(command, commandArgs, request.identity, dataSet._dataStore) ?~> "job.couldNotRunApplyMergerMode"

app/models/job/Job.scala

@@ -72,7 +72,7 @@ case class Job(
           }
         case "export_tiff" =>
           Some(s"$dataStorePublicUrl/data/exports/${_id.id}/download")
-        case "infer_nuclei" | "infer_neurons" | "apply_merger_mode" =>
+        case "infer_nuclei" | "infer_neurons" | "materialize_volume_annotation" =>
           returnValue.map { resultDatasetName =>
             s"/datasets/$organizationName/$resultDatasetName/view"
           }

frontend/javascripts/admin/admin_rest_api.ts

@@ -948,6 +948,7 @@ export async function getJobs(): Promise<APIJob[]> {
     tracingId: job.commandArgs.volume_tracing_id,
     annotationId: job.commandArgs.annotation_id,
     annotationType: job.commandArgs.annotation_type,
+    mergeSegments: job.commandArgs.merge_segments,
     state: adaptJobState(job.command, job.state, job.manualState),
     manualState: job.manualState,
     result: job.returnValue,
@@ -1084,6 +1085,7 @@ function startSegmentationAnnotationDependentJob(
   annotationId: string,
   annotationType: APIAnnotationType,
   outputSegmentationLayerName?: string,
+  mergeSegments?: boolean,
 ): Promise<APIJob> {
   const requestURL = new URL(
     `/api/jobs/run/${jobURLPath}/${organizationName}/${datasetName}`,
@@ -1100,6 +1102,9 @@ function startSegmentationAnnotationDependentJob(
   if (outputSegmentationLayerName != null) {
     requestURL.searchParams.append("outputSegmentationLayerName", outputSegmentationLayerName);
   }
+  if (mergeSegments != null) {
+    requestURL.searchParams.append("mergeSegments", mergeSegments.toString());
+  }
   return Request.receiveJSON(requestURL.href, {
     method: "POST",
   });
@@ -1126,7 +1131,7 @@ export function startGlobalizeFloodfillsJob(
   );
 }
 
-export function startApplyMergerModeJob(
+export function startMaterializingVolumeAnnotationJob(
   organizationName: string,
   datasetName: string,
   fallbackLayerName: string,
@@ -1135,9 +1140,10 @@ export function startApplyMergerModeJob(
   outputSegmentationLayerName: string,
   annotationId: string,
   annotationType: APIAnnotationType,
+  mergeSegments: boolean,
 ): Promise<APIJob> {
   return startSegmentationAnnotationDependentJob(
-    "applyMergerMode",
+    "materializeVolumeAnnotation",
     organizationName,
     datasetName,
     fallbackLayerName,
@@ -1146,6 +1152,7 @@ export function startApplyMergerModeJob(
     annotationId,
     annotationType,
     outputSegmentationLayerName,
+    mergeSegments,
   );
 }
 

frontend/javascripts/admin/job/job_list_view.tsx

@@ -175,17 +175,19 @@ class JobListView extends React.PureComponent<Props, State> {
         </span>
       );
     } else if (
-      job.type === "apply_merger_mode" &&
+      job.type === "materialize_volume_annotation" &&
       job.organizationName &&
-      job.datasetName &&
-      job.layerName
+      job.datasetName
     ) {
       return (
         <span>
-          Applied merger mode to layer {job.layerName} of{" "}
+          Materialize annotation for {job.layerName ? ` layer ${job.layerName} of ` : " "}
           <Link to={`/datasets/${job.organizationName}/${job.datasetName}/view`}>
             {job.datasetName}
-          </Link>{" "}
+          </Link>
+          {job.mergeSegments
+            ? ". This includes merging the segments that were merged via merger mode."
+            : null}
         </span>
       );
     } else {
@@ -239,7 +241,7 @@ class JobListView extends React.PureComponent<Props, State> {
     } else if (
       job.type === "infer_nuclei" ||
       job.type === "infer_neurons" ||
-      job.type === "apply_merger_mode"
+      job.type === "materialize_volume_annotation"
     ) {
       return (
         <span>

frontend/javascripts/libs/utils.ts

@@ -172,9 +172,6 @@ export function roundTo(value: number, digits: number): number {
 export function capitalize(str: string): string {
   return str[0].toUpperCase() + str.slice(1);
 }
-export function capitalizeWords(str: string): string {
-  return str.split(" ").map(capitalize).join(" ");
-}
 
 function intToHex(int: number, digits: number = 6): string {
   return (_.repeat("0", digits) + int.toString(16)).slice(-digits);

frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx

@@ -30,7 +30,7 @@ import { updateUserSettingAction } from "oxalis/model/actions/settings_actions";
 import { usePrevious, useKeyPress } from "libs/react_hooks";
 import { userSettings } from "types/schemas/user_settings.schema";
 import ButtonComponent from "oxalis/view/components/button_component";
-import { ApplyMergerModeModal } from "oxalis/view/right-border-tabs/starting_job_modals";
+import { MaterializeVolumeAnnotationModal } from "oxalis/view/right-border-tabs/starting_job_modals";
 import type { AnnotationTool, OverwriteMode } from "oxalis/constants";
 import Constants, {
   ToolsWithOverwriteCapabilities,
@@ -276,7 +276,7 @@ function AdditionalSkeletonModesButtons() {
         </Tooltip>
       )}
       {features().jobsEnabled && showApplyMergerModeModal && (
-        <ApplyMergerModeModal handleClose={() => setShowApplyMergerModeModal(false)} />
+        <MaterializeVolumeAnnotationModal handleClose={() => setShowApplyMergerModeModal(false)} />
       )}
     </React.Fragment>
   );

frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx

@@ -14,7 +14,7 @@ import React from "react";
 import _ from "lodash";
 
 import classnames from "classnames";
-import type { APIDataset, EditableLayerProperties } from "types/api_flow_types";
+import type { APIDataLayer, APIDataset, EditableLayerProperties } from "types/api_flow_types";
 import { ValueOf } from "types/globals";
 import { AsyncIconButton } from "components/async_clickables";
 import {
@@ -81,6 +81,7 @@ import Toast from "libs/toast";
 import * as Utils from "libs/utils";
 import api from "oxalis/api/internal_api";
 import { settings } from "messages";
+import { MaterializeVolumeAnnotationModal } from "oxalis/view/right-border-tabs/starting_job_modals";
 import AddVolumeLayerModal from "./modals/add_volume_layer_modal";
 import DownsampleVolumeModal from "./modals/downsample_volume_modal";
 import Histogram, { isHistogramSupported } from "./histogram_view";
@@ -115,13 +116,15 @@ type State = {
   // is shown for that VolumeTracing
   volumeTracingToDownsample: VolumeTracing | null | undefined;
   isAddVolumeLayerModalVisible: boolean;
+  layerToMergeWithFallback: APIDataLayer | null | undefined;
 };
 
 class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
   onChangeUser: Record<keyof UserConfiguration, (...args: Array<any>) => any>;
   state: State = {
     volumeTracingToDownsample: null,
     isAddVolumeLayerModalVisible: false,
+    layerToMergeWithFallback: null,
   };
 
   constructor(props: DatasetSettingsProps) {
@@ -196,6 +199,19 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
     );
   };
 
+  getMergeWithFallbackLayerButton = (layer: APIDataLayer) => (
+    <Tooltip title="Merge this volume annotation with its fallback layer.">
+      <i
+        onClick={() => this.setState({ layerToMergeWithFallback: layer })}
+        className="fas fa-object-ungroup"
+        style={{
+          cursor: "pointer",
+          opacity: 0.7,
+        }}
+      />
+    </Tooltip>
+  );
+
   getClipButton = (layerName: string, isInEditMode: boolean) => {
     const editModeAddendum = isInEditMode
       ? "In Edit Mode, the histogram's range will be adjusted, too."
@@ -457,6 +473,9 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
           <div className="flex-item">
             {hasHistogram && !isDisabled ? this.getEditMinMaxButton(layerName, isInEditMode) : null}
           </div>
+          <div className="flex-item">
+            {isVolumeTracing && !isDisabled ? this.getMergeWithFallbackLayerButton(layer) : null}
+          </div>
           <div className="flex-item">
             {this.getFindDataButton(layerName, isDisabled, isColorLayer, maybeVolumeTracing)}
           </div>
@@ -639,12 +658,14 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
       `Jumping to position ${foundPosition.join(", ")} and zooming to ${zoomValue.toFixed(2)}`,
     );
   };
+
   reloadLayerData = async (layerName: string): Promise<void> => {
     await clearCache(this.props.dataset, layerName);
     await api.data.reloadBuckets(layerName);
     window.needsRerender = true;
     Toast.success(`Successfully reloaded data of layer ${layerName}.`);
   };
+
   getVolumeMagsToDownsample = (volumeTracing: VolumeTracing | null | undefined): Array<Vector3> => {
     if (this.props.task != null) {
       return [];
@@ -887,6 +908,13 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
           />
         ) : null}
 
+        {this.state.layerToMergeWithFallback != null ? (
+          <MaterializeVolumeAnnotationModal
+            selectedVolumeLayer={this.state.layerToMergeWithFallback}
+            handleClose={() => this.setState({ layerToMergeWithFallback: null })}
+          />
+        ) : null}
+
         {this.state.isAddVolumeLayerModalVisible ? (
           <AddVolumeLayerModal
             dataset={this.props.dataset}