Use new mesh API for animation job

CHANGELOG.unreleased.md

@@ -35,6 +35,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Only display UI elements to launch background jobs if the (worker) backend actually supports them. [#7591](https://github.com/scalableminds/webknossos/pull/7591)
 - If the current dataset folder in the dashboard cannot be found (e.g., because somebody else deleted it), the page navigates to the root folder automatically. [#7669](https://github.com/scalableminds/webknossos/pull/7669)
 - Voxelytics logs are now stored by organization name, rather than id, in Loki. This is inpreparation of the unificaction of these two concepts. [#7687](https://github.com/scalableminds/webknossos/pull/7687)
+- Updated dataset animations to use the new meshing API. Animitation now support ad-hoc meshes and mappings. [#7692](https://github.com/scalableminds/webknossos/pull/7692) 
 
 ### Fixed
 - Fixed rare SIGBUS crashes of the datastore module that were caused by memory mapping on unstable file systems. [#7528](https://github.com/scalableminds/webknossos/pull/7528)

app/controllers/JobsController.scala

@@ -31,12 +31,11 @@ object CameraPositionSetting extends ExtendedEnumeration {
 }
 
 case class AnimationJobOptions(
-    layerName: String,
+    colorLayerName: String,
     boundingBox: BoundingBox,
     includeWatermark: Boolean,
-    segmentationLayerName: Option[String],
-    meshFileName: Option[String],
-    meshSegmentIds: Array[Int],
+    isViewMode: Boolean,
+    meshes: JsValue,
     movieResolution: MovieResolutionSetting.Value,
     cameraPosition: CameraPositionSetting.Value,
     intensityMin: Double,
@@ -380,22 +379,20 @@ class JobsController @Inject()(
           _ <- Fox.runIf(userOrganization.pricingPlan == PricingPlan.Basic) {
             bool2Fox(animationJobOptions.movieResolution == MovieResolutionSetting.SD) ?~> "job.renderAnimation.resolutionMustBeSD"
           }
-          layerName = animationJobOptions.layerName
-          _ <- datasetService.assertValidLayerName(layerName)
-          _ <- Fox.runOptional(animationJobOptions.segmentationLayerName)(datasetService.assertValidLayerName)
-          exportFileName = s"webknossos_animation_${formatDateForFilename(new Date())}__${datasetName}__$layerName.mp4"
+          colorLayerName = animationJobOptions.colorLayerName
+          _ <- datasetService.assertValidLayerName(colorLayerName)
+          exportFileName = s"webknossos_animation_${formatDateForFilename(new Date())}__${datasetName}__$colorLayerName.mp4"
           command = JobCommand.render_animation
           commandArgs = Json.obj(
             "organization_name" -> organizationName,
             "dataset_name" -> datasetName,
             "export_file_name" -> exportFileName,
             "user_auth_token" -> userAuthToken.id,
-            "layer_name" -> animationJobOptions.layerName,
-            "segmentation_layer_name" -> animationJobOptions.segmentationLayerName,
+            "is_view_mode" -> animationJobOptions.isViewMode,
+            "color_layer_name" -> animationJobOptions.colorLayerName,
             "bounding_box" -> animationJobOptions.boundingBox.toLiteral,
             "include_watermark" -> animationJobOptions.includeWatermark,
-            "mesh_segment_ids" -> animationJobOptions.meshSegmentIds,
-            "meshfile_name" -> animationJobOptions.meshFileName,
+            "meshes" -> animationJobOptions.meshes,
             "movie_resolution" -> animationJobOptions.movieResolution,
             "camera_position" -> animationJobOptions.cameraPosition,
             "intensity_min" -> animationJobOptions.intensityMin,

docs/animations.md

@@ -9,7 +9,7 @@ A picture is worth a thousand words. In this spirit, you can use WEBKNOSSOS to c
 Creating an animation is easy:
 
 1. Open any dataset or annotation that you want to use for your animation. 
-2. Optionally, load any [pre-computed 3D meshes](./mesh_visualization.md#pre-computed-mesh-generation) for any segments that you wish to highlight.
+2. Optionally, load some [3D meshes](./mesh_visualization.md#pre-computed-mesh-generation) for any segments that you wish to highlight.
 3. For larger datasets, use the bounding box tool to create a bounding box around your area of interest. Smaller datasets can be used in their entirety.
 4. From the `Menu` dropdown in navbar at the top of the screen, select "Create Animation".
 5. Configure the animation options as desired, i.e. camera movement or resolution.

frontend/javascripts/oxalis/view/action-bar/create_animation_modal.tsx

@@ -11,6 +11,8 @@ import {
   getColorLayers,
   getEffectiveIntensityRange,
   getLayerByName,
+  getResolutionInfo,
+  hasFallbackLayer,
   is2dDataset,
 } from "oxalis/model/accessors/dataset_accessor";
 import {
@@ -24,16 +26,16 @@ import {
   MOVIE_RESOLUTIONS,
   APIDataLayer,
   APIJobType,
+  APISegmentationLayer,
 } from "types/api_flow_types";
 import { InfoCircleOutlined } from "@ant-design/icons";
 import { PricingEnforcedSpan } from "components/pricing_enforcers";
 import {
   PricingPlanEnum,
   isFeatureAllowedByPricingPlan,
 } from "admin/organization/pricing_plan_utils";
-import { BoundingBoxType, Vector3 } from "oxalis/constants";
+import { BoundingBoxType, ControlModeEnum, Vector3 } from "oxalis/constants";
 import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box";
-import { Model } from "oxalis/singletons";
 import { BoundingBoxSelection, LayerSelection } from "./starting_job_modals";
 
 type Props = {
@@ -80,7 +82,10 @@ export function CreateAnimationModalWrapper(props: Props) {
 
   // early stop if no color layer exists
   const colorLayers = getColorLayers(dataset);
-  if (colorLayers.length === 0) return null;
+  if (colorLayers.length === 0) {
+    console.warn("WK can not create animations for datasets without color layers.");
+    return null;
+  }
 
   return <CreateAnimationModal {...props} />;
 }
@@ -128,7 +133,7 @@ function CreateAnimationModal(props: Props) {
   const validateAnimationOptions = (
     colorLayer: APIDataLayer,
     selectedBoundingBox: BoundingBoxType,
-    meshSegmentIds: number[],
+    meshes: Partial<MeshInformation>[],
   ) => {
     //  Validate the select parameters and dataset to make sure it actually works and does not overload the server
 
@@ -151,7 +156,7 @@ function CreateAnimationModal(props: Props) {
       !is2dDataset(state.dataset) && (colorLayer.additionalAxes?.length || 0) === 0;
     if (isDataset3D) errorMessages.push("Sorry, animations are only supported for 3D datasets.");
 
-    const isTooManyMeshes = meshSegmentIds.length > MAX_MESHES_PER_ANIMATION;
+    const isTooManyMeshes = meshes.length > MAX_MESHES_PER_ANIMATION;
     if (isTooManyMeshes)
       errorMessages.push(
         `You selected too many meshes for the animation. Please keep the number of meshes below ${MAX_MESHES_PER_ANIMATION} to create an animation.`,
@@ -171,32 +176,34 @@ function CreateAnimationModal(props: Props) {
       (bb) => bb.id === selectedBoundingBoxId,
     )!.boundingBox;
 
-    // Submit currently visible pre-computed meshes
-    let meshSegmentIds: number[] = [];
-    let meshFileName: string | undefined;
-    let segmentationLayerName: string | undefined;
-
-    const visibleSegmentationLayer = Model.getVisibleSegmentationLayer();
-
-    if (visibleSegmentationLayer) {
-      const availableMeshes = state.localSegmentationData[visibleSegmentationLayer.name].meshes;
-      if (availableMeshes == null) {
-        throw new Error("There is no mesh data in localSegmentationData.");
-      }
-      meshSegmentIds = Object.values(availableMeshes as Record<number, MeshInformation>)
-        .filter((mesh) => mesh.isVisible && mesh.isPrecomputed)
-        .map((mesh) => mesh.segmentId);
-
-      const currentMeshFile =
-        state.localSegmentationData[visibleSegmentationLayer.name].currentMeshFile;
-      meshFileName = currentMeshFile?.meshFileName;
-
-      if (visibleSegmentationLayer.fallbackLayerInfo) {
-        segmentationLayerName = visibleSegmentationLayer.fallbackLayerInfo.name;
-      } else {
-        segmentationLayerName = visibleSegmentationLayer.name;
-      }
-    }
+    // Submit currently visible pre-computed & ad-hoc meshes
+    const axis = "";
+    const layerNames = Object.keys(state.localSegmentationData);
+    const { preferredQualityForMeshAdHocComputation } = state.temporaryConfiguration;
+
+    const meshes: RenderAnimationOptions["meshes"] = layerNames.flatMap((layerName) => {
+      const meshInfos = state.localSegmentationData[layerName]?.meshes?.[axis] || {};
+
+      return Object.values(meshInfos)
+        .filter((meshInfo: MeshInformation) => meshInfo.isVisible)
+        .flatMap((meshInfo: MeshInformation) => {
+          const layer = getLayerByName(state.dataset, layerName) as APISegmentationLayer;
+          const hasAFallbackLayer = hasFallbackLayer(layer);
+          const fullLayerName = layer.fallbackLayerInfo?.name || layerName;
+
+          const adhoc_mag_index = getResolutionInfo(layer.resolutions).getClosestExistingIndex(
+            preferredQualityForMeshAdHocComputation,
+          );
+          const adhoc_mag = layer.resolutions[adhoc_mag_index];
+
+          return {
+            layerName: fullLayerName,
+            hasFallbackLayer: hasAFallbackLayer,
+            adhoc_mag,
+            ...meshInfo,
+          };
+        });
+    });
 
     // Submit the configured min/max intensity info to support float datasets
     const [intensityMin, intensityMax] = getEffectiveIntensityRange(
@@ -206,22 +213,22 @@ function CreateAnimationModal(props: Props) {
     );
 
     const [magForTextures, _] = selectMagForTextureCreation(colorLayer, boundingBox);
+    const isViewMode = state.temporaryConfiguration.controlMode === ControlModeEnum.VIEW;
 
     const animationOptions: RenderAnimationOptions = {
-      layerName: selectedColorLayerName,
-      segmentationLayerName,
-      meshFileName,
-      meshSegmentIds,
+      colorLayerName: selectedColorLayerName,
+      meshes,
       intensityMin,
       intensityMax,
       magForTextures,
+      isViewMode,
       boundingBox: computeBoundingBoxObjectFromBoundingBox(boundingBox),
       includeWatermark: isWatermarkEnabled,
       movieResolution: selectedMovieResolution,
       cameraPosition: selectedCameraPosition,
     };
 
-    if (!validateAnimationOptions(colorLayer, boundingBox, meshSegmentIds)) return;
+    if (!validateAnimationOptions(colorLayer, boundingBox, meshes)) return;
 
     startRenderAnimationJob(state.dataset.owningOrganization, state.dataset.name, animationOptions);
 
@@ -338,7 +345,7 @@ function CreateAnimationModal(props: Props) {
               >
                 Include the currently selected 3D meshes
                 <Tooltip
-                  title="When enabled, all (pre-computed) meshes currently visible in WEBKNOSSOS will be included in the animation."
+                  title="When enabled, all meshes currently visible in WEBKNOSSOS will be included in the animation."
                   placement="right"
                 >
                   <InfoCircleOutlined style={{ marginLeft: 10 }} />

frontend/javascripts/types/api_flow_types.ts

@@ -6,6 +6,7 @@ import type {
   TreeGroup,
   RecommendedConfiguration,
   SegmentGroup,
+  MeshInformation,
 } from "oxalis/store";
 import type { ServerUpdateAction } from "oxalis/model/sagas/update_actions";
 import type {
@@ -1085,11 +1086,14 @@ export enum MOVIE_RESOLUTIONS {
 }
 
 export type RenderAnimationOptions = {
-  layerName: string;
-  segmentationLayerName?: string;
-  meshFileName?: string;
-  meshSegmentIds: number[];
+  colorLayerName: string;
+  meshes: ({
+    layerName: string;
+    hasFallbackLayer: boolean;
+    adhoc_mag: Vector3;
+  } & MeshInformation)[];
   boundingBox: BoundingBoxObject;
+  isViewMode: boolean;
   includeWatermark: boolean;
   intensityMin: number;
   intensityMax: number;