Support transforming the skeleton layer with affine and TSP transforms

CHANGELOG.unreleased.md

@@ -15,6 +15,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Segment statistics are now available for ND datasets. [#7411](https://github.com/scalableminds/webknossos/pull/7411)
 - Added support for uploading N5 and Neuroglancer Precomputed datasets. [#7578](https://github.com/scalableminds/webknossos/pull/7578)
 - Webknossos can now open ND Zarr datasets with arbitrary axis orders (not limited to `**xyz` anymore). [#7592](https://github.com/scalableminds/webknossos/pull/7592)
+- Added support for skeleton annotations within datasets that have transformed layers. The skeleton nodes will move according to the transforms when rendering a specific layer natively. Also, downloading visible trees can be done by incorporating the current transforms. However, note that the back-end export does not take transforms into account. [#7588](https://github.com/scalableminds/webknossos/pull/7588)
 - Added a new "Split from all neighboring segments" feature for the proofreading mode. [#7611](https://github.com/scalableminds/webknossos/pull/7611)
 - If storage scan is enabled, the measured used storage is now displayed in the dashboard’s dataset detail view. [#7677](https://github.com/scalableminds/webknossos/pull/7677)
 - Prepared support to download full stl meshes via the HTTP api. [#7587](https://github.com/scalableminds/webknossos/pull/7587)

frontend/javascripts/admin/dataset/composition_wizard/02_upload_files.tsx

@@ -188,8 +188,8 @@ async function parseNmlFiles(fileList: FileList): Promise<Partial<WizardContext>
         );
       }
       if (node1 != null && node2 != null) {
-        sourcePoints.push(node1.position);
-        targetPoints.push(node2.position);
+        sourcePoints.push(node1.untransformedPosition);
+        targetPoints.push(node2.untransformedPosition);
       }
     }
   }

frontend/javascripts/oxalis/api/api_latest.ts

@@ -60,6 +60,7 @@ import {
   getFlatTreeGroups,
   getTreeGroupsMap,
   mapGroups,
+  getNodePosition,
 } from "oxalis/model/accessors/skeletontracing_accessor";
 import {
   getActiveCellId,
@@ -159,6 +160,7 @@ import type {
   OxalisState,
   SegmentGroup,
   Segment,
+  MutableNode,
 } from "oxalis/store";
 import Store from "oxalis/store";
 import type { ToastStyle } from "libs/toast";
@@ -986,9 +988,9 @@ class TracingApi {
    */
   centerNode = (nodeId?: number): void => {
     const skeletonTracing = assertSkeleton(Store.getState().tracing);
-    getNodeAndTree(skeletonTracing, nodeId).map(([, node]) =>
-      Store.dispatch(setPositionAction(node.position)),
-    );
+    getNodeAndTree(skeletonTracing, nodeId).map(([, node]) => {
+      return Store.dispatch(setPositionAction(getNodePosition(node, Store.getState())));
+    });
   };
 
   /**
@@ -1032,23 +1034,26 @@ class TracingApi {
    * Measures the length of the given tree and returns the length in nanometer and in voxels.
    */
   measureTreeLength(treeId: number): [number, number] {
-    const skeletonTracing = assertSkeleton(Store.getState().tracing);
+    const state = Store.getState();
+    const skeletonTracing = assertSkeleton(state.tracing);
     const tree = skeletonTracing.trees[treeId];
 
     if (!tree) {
       throw new Error(`Tree with id ${treeId} not found.`);
     }
 
-    const datasetScale = Store.getState().dataset.dataSource.scale;
+    const datasetScale = state.dataset.dataSource.scale;
     // Pre-allocate vectors
     let lengthNmAcc = 0;
     let lengthVxAcc = 0;
 
+    const getPos = (node: Readonly<MutableNode>) => getNodePosition(node, state);
+
     for (const edge of tree.edges.all()) {
       const sourceNode = tree.nodes.get(edge.source);
       const targetNode = tree.nodes.get(edge.target);
-      lengthNmAcc += V3.scaledDist(sourceNode.position, targetNode.position, datasetScale);
-      lengthVxAcc += V3.length(V3.sub(sourceNode.position, targetNode.position));
+      lengthNmAcc += V3.scaledDist(getPos(sourceNode), getPos(targetNode), datasetScale);
+      lengthVxAcc += V3.length(V3.sub(getPos(sourceNode), getPos(targetNode)));
     }
 
     return [lengthNmAcc, lengthVxAcc];
@@ -1125,14 +1130,17 @@ class TracingApi {
     });
     priorityQueue.queue([sourceNodeId, 0]);
 
+    const state = Store.getState();
+    const getPos = (node: Readonly<MutableNode>) => getNodePosition(node, state);
+
     while (priorityQueue.length > 0) {
       const [nextNodeId, distance] = priorityQueue.dequeue();
-      const nextNodePosition = sourceTree.nodes.get(nextNodeId).position;
+      const nextNodePosition = getPos(sourceTree.nodes.get(nextNodeId));
 
       // Calculate the distance to all neighbours and update the distances.
       for (const { source, target } of sourceTree.edges.getEdgesForNode(nextNodeId)) {
         const neighbourNodeId = source === nextNodeId ? target : source;
-        const neighbourPosition = sourceTree.nodes.get(neighbourNodeId).position;
+        const neighbourPosition = getPos(sourceTree.nodes.get(neighbourNodeId));
         const neighbourDistance =
           distance + V3.scaledDist(nextNodePosition, neighbourPosition, datasetScale);
 
@@ -2605,11 +2613,7 @@ class UtilsApi {
    */
   registerOverwrite<S, A>(
     actionName: string,
-    overwriteFunction: (
-      store: S,
-      next: (action: A) => void,
-      originalAction: A,
-    ) => void | Promise<void>,
+    overwriteFunction: (store: S, next: (action: A) => void, originalAction: A) => A | Promise<A>,
   ) {
     return overwriteAction(actionName, overwriteFunction);
   }

frontend/javascripts/oxalis/api/api_v2.ts

@@ -21,6 +21,7 @@ import {
   getActiveNode,
   getActiveTree,
   getTree,
+  getNodePosition,
 } from "oxalis/model/accessors/skeletontracing_accessor";
 import { setActiveCellAction } from "oxalis/model/actions/volumetracing_actions";
 import { getActiveCellId } from "oxalis/model/accessors/volumetracing_accessor";
@@ -346,9 +347,10 @@ class TracingApi {
    * api.tracing.centerNode()
    */
   centerNode = (nodeId?: number): void => {
-    const skeletonTracing = assertSkeleton(Store.getState().tracing);
+    const state = Store.getState();
+    const skeletonTracing = assertSkeleton(state.tracing);
     getNodeAndTree(skeletonTracing, nodeId).map(([, node]) =>
-      Store.dispatch(setPositionAction(node.position)),
+      Store.dispatch(setPositionAction(getNodePosition(node, state))),
     );
   };
 
@@ -841,7 +843,7 @@ class UtilsApi {
    */
   registerOverwrite<S, A>(
     actionName: string,
-    overwriteFunction: (store: S, next: (action: A) => void, originalAction: A) => void,
+    overwriteFunction: (store: S, next: (action: A) => void, originalAction: A) => A | Promise<A>,
   ) {
     overwriteAction(actionName, overwriteFunction);
   }

frontend/javascripts/oxalis/constants.ts

@@ -376,5 +376,9 @@ export enum BLEND_MODES {
 }
 
 export const Identity4x4 = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
-export const IdentityTransform = { type: "affine", affineMatrix: Identity4x4 } as const;
+export const IdentityTransform = {
+  type: "affine",
+  affineMatrix: Identity4x4,
+  affineMatrixInv: Identity4x4,
+} as const;
 export const EMPTY_OBJECT = {} as const;

frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts

@@ -16,6 +16,8 @@ import {
   getActiveNode,
   getNodeAndTree,
   getNodeAndTreeOrNull,
+  getNodePosition,
+  untransformNodePosition,
 } from "oxalis/model/accessors/skeletontracing_accessor";
 import {
   getInputCatcherRect,
@@ -197,10 +199,10 @@ export function moveNode(
         op(vector[1] * zoomFactor * scaleFactor[1]),
         op(vector[2] * zoomFactor * scaleFactor[2]),
       ];
-      const [x, y, z] = activeNode.position;
+      const [x, y, z] = getNodePosition(activeNode, state);
       Store.dispatch(
         setNodePositionAction(
-          [x + delta[0], y + delta[1], z + delta[2]],
+          untransformNodePosition([x + delta[0], y + delta[1], z + delta[2]], state),
           activeNode.id,
           activeTree.treeId,
         ),
@@ -213,7 +215,11 @@ export function finishNodeMovement(nodeId: number) {
   getSkeletonTracing(Store.getState().tracing).map((skeletonTracing) =>
     getNodeAndTree(skeletonTracing, nodeId).map(([activeTree, node]) => {
       Store.dispatch(
-        setNodePositionAction(V3.round(node.position, [0, 0, 0]), node.id, activeTree.treeId),
+        setNodePositionAction(
+          V3.round(node.untransformedPosition, [0, 0, 0]),
+          node.id,
+          activeTree.treeId,
+        ),
       );
     }),
   );
@@ -228,15 +234,16 @@ export function setWaypoint(
   const activeNodeMaybe = getActiveNode(skeletonTracing);
   const rotation = getRotationOrtho(activeViewport);
   // set the new trace direction
-  activeNodeMaybe.map((activeNode) =>
-    Store.dispatch(
+  activeNodeMaybe.map((activeNode) => {
+    const activeNodePosition = getNodePosition(activeNode, Store.getState());
+    return Store.dispatch(
       setDirectionAction([
-        position[0] - activeNode.position[0],
-        position[1] - activeNode.position[1],
-        position[2] - activeNode.position[2],
+        position[0] - activeNodePosition[0],
+        position[1] - activeNodePosition[1],
+        position[2] - activeNodePosition[2],
       ]),
-    ),
-  );
+    );
+  });
   const state = Store.getState();
   // Create a new tree automatically if the corresponding setting is true and allowed
   const createNewTree =
@@ -276,7 +283,7 @@ function addNode(
 
   Store.dispatch(
     createNodeAction(
-      position,
+      untransformNodePosition(position, state),
       state.flycam.additionalCoordinates,
       rotation,
       OrthoViewToNumber[Store.getState().viewModeData.plane.activeViewport],
@@ -293,12 +300,10 @@ function addNode(
   if (center) {
     // we created a new node, so get a new reference from the current store state
     const newState = Store.getState();
-    enforce(getActiveNode)(newState.tracing.skeleton).map(
-      (
-        newActiveNode, // Center the position of the active node without modifying the "third" dimension (see centerPositionAnimated)
-      ) =>
-        // This is important because otherwise the user cannot continue to trace until the animation is over
-        api.tracing.centerPositionAnimated(newActiveNode.position, true),
+    enforce(getActiveNode)(newState.tracing.skeleton).map((newActiveNode) =>
+      // Center the position of the active node without modifying the "third" dimension (see centerPositionAnimated)
+      // This is important because otherwise the user cannot continue to trace until the animation is over
+      api.tracing.centerPositionAnimated(getNodePosition(newActiveNode, state), true),
     );
   }
 

frontend/javascripts/oxalis/controller/td_controller.tsx

@@ -26,7 +26,7 @@ import {
   moveTDViewYAction,
   moveTDViewByVectorWithoutTimeTrackingAction,
 } from "oxalis/model/actions/view_mode_actions";
-import { getActiveNode } from "oxalis/model/accessors/skeletontracing_accessor";
+import { getActiveNode, getNodePosition } from "oxalis/model/accessors/skeletontracing_accessor";
 import { voxelToNm } from "oxalis/model/scaleinfo";
 import CameraController from "oxalis/controller/camera_controller";
 import PlaneView from "oxalis/view/plane_view";
@@ -128,7 +128,7 @@ class TDController extends React.PureComponent<Props> {
       // This happens because the selection of the node does not trigger a call to setTargetAndFixPosition directly.
       // Thus we do it manually whenever the active node changes.
       getActiveNode(this.props.tracing.skeleton).map((activeNode) =>
-        this.setTargetAndFixPosition(activeNode.position),
+        this.setTargetAndFixPosition(getNodePosition(activeNode, Store.getState())),
       );
     }
   }

frontend/javascripts/oxalis/controller/viewmodes/arbitrary_controller.tsx

@@ -3,7 +3,12 @@ import type { ModifierKeys } from "libs/input";
 import { InputKeyboard, InputKeyboardNoLoop, InputMouse } from "libs/input";
 import type { Matrix4x4 } from "libs/mjs";
 import { V3 } from "libs/mjs";
-import { getActiveNode, getMaxNodeId } from "oxalis/model/accessors/skeletontracing_accessor";
+import {
+  getActiveNode,
+  getMaxNodeId,
+  getNodePosition,
+  untransformNodePosition,
+} from "oxalis/model/accessors/skeletontracing_accessor";
 import { getRotation, getPosition, getMoveOffset3d } from "oxalis/model/accessors/flycam_accessor";
 import { getViewportScale } from "oxalis/model/accessors/view_mode_accessor";
 import { listenToStoreProperty } from "oxalis/model/helpers/listener_helpers";
@@ -198,14 +203,19 @@ class ArbitraryController extends React.PureComponent<Props> {
       },
       // Recenter active node
       s: () => {
-        const skeletonTracing = Store.getState().tracing.skeleton;
+        const state = Store.getState();
+        const skeletonTracing = state.tracing.skeleton;
 
         if (!skeletonTracing) {
           return;
         }
 
         getActiveNode(skeletonTracing).map((activeNode) =>
-          api.tracing.centerPositionAnimated(activeNode.position, false, activeNode.rotation),
+          api.tracing.centerPositionAnimated(
+            getNodePosition(activeNode, state),
+            false,
+            activeNode.rotation,
+          ),
         );
       },
       ".": () => this.nextNode(true),
@@ -357,12 +367,18 @@ class ArbitraryController extends React.PureComponent<Props> {
     if (!Store.getState().temporaryConfiguration.flightmodeRecording) {
       return;
     }
-
-    const position = getPosition(Store.getState().flycam);
-    const rotation = getRotation(Store.getState().flycam);
-    const additionalCoordinates = Store.getState().flycam.additionalCoordinates;
+    const state = Store.getState();
+    const position = getPosition(state.flycam);
+    const rotation = getRotation(state.flycam);
+    const additionalCoordinates = state.flycam.additionalCoordinates;
     Store.dispatch(
-      createNodeAction(position, additionalCoordinates, rotation, constants.ARBITRARY_VIEW, 0),
+      createNodeAction(
+        untransformNodePosition(position, state),
+        additionalCoordinates,
+        rotation,
+        constants.ARBITRARY_VIEW,
+        0,
+      ),
     );
   }