Allow to compose datasets without transforms or with bigwarp/NML landmarks

CHANGELOG.unreleased.md

@@ -17,6 +17,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added route for triggering the compute segment index worker job. [#7471](https://github.com/scalableminds/webknossos/pull/7471)
 - Added thumbnails to the dashboard dataset list. [#7479](https://github.com/scalableminds/webknossos/pull/7479)
 - Adhoc mesh rendering is now available for ND datasets.[#7394](https://github.com/scalableminds/webknossos/pull/7394)
+- Added the ability to compose a new dataset from existing dataset layers. This can be done with or without transforms (transforms will be derived from landmarks given via BigWarp CSV or WK NMLs). [#7395](https://github.com/scalableminds/webknossos/pull/7395)
 
 ### Changed
 - Improved loading speed of the annotation list. [#7410](https://github.com/scalableminds/webknossos/pull/7410)

app/controllers/WKRemoteDataStoreController.scala

@@ -5,11 +5,8 @@ import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.controllers.JobExportProperties
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
-import com.scalableminds.webknossos.datastore.services.{
-  DataStoreStatus,
-  LinkedLayerIdentifier,
-  ReserveUploadInformation
-}
+import com.scalableminds.webknossos.datastore.services.uploading.{LinkedLayerIdentifier, ReserveUploadInformation}
+import com.scalableminds.webknossos.datastore.services.DataStoreStatus
 import com.typesafe.scalalogging.LazyLogging
 import mail.{MailchimpClient, MailchimpTag}
 

docs/datasets.md

@@ -126,6 +126,18 @@ For manual conversion, we provide the following software tools and libraries:
 - The [WEBKNOSSOS CLI](https://docs.webknossos.org/cli) is a CLI tool that can convert many formats to WKW. 
 - For other file formats, the [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/index.html) can be an option for custom scripting.
 
+### Composing Datasets
+New datasets can also be composed from existing ones.
+This feature allows to combine layers from previously added datasets to create a new dataset.
+During compositions, transforms can optionally be defined in case the datasets are not in the same coordinate system.
+There are three different ways to compose a new dataset:
+
+1) Combine datasets by selecting from existing datasets. No transforms between these datasets will be added.
+2) Create landmark annotations (using the skeleton tool) for each dataset. Then, these datasets can be combined while transforming one dataset to match the other.
+3) Similar to (2), two datasets can be combined while respecting landmarks that were generated with BigWarp.
+
+See the "Compose from existing datasets" tab in the "Add Dataset" screen for more details.
+
 ## Configuring Datasets
 You can configure the metadata, permission, and other properties of a dataset at any time. 
 

frontend/javascripts/admin/admin_rest_api.ts

@@ -66,6 +66,7 @@ import type {
   MaintenanceInfo,
   AdditionalCoordinate,
   RenderAnimationOptions,
+  LayerLink,
 } from "types/api_flow_types";
 import { APIAnnotationTypeEnum } from "types/api_flow_types";
 import type { LOG_LEVELS, Vector2, Vector3, Vector6 } from "oxalis/constants";
@@ -1429,10 +1430,12 @@ export async function getActiveDatasetsOfMyOrganization(): Promise<Array<APIData
 export function getDataset(
   datasetId: APIDatasetId,
   sharingToken?: string | null | undefined,
+  options: RequestOptions = {},
 ): Promise<APIDataset> {
   const sharingTokenSuffix = sharingToken != null ? `?sharingToken=${sharingToken}` : "";
   return Request.receiveJSON(
     `/api/datasets/${datasetId.owningOrganization}/${datasetId.name}${sharingTokenSuffix}`,
+    options,
   );
 }
 
@@ -1513,6 +1516,22 @@ export function getDatasetAccessList(datasetId: APIDatasetId): Promise<Array<API
   );
 }
 
+type DatasetCompositionArgs = {
+  newDatasetName: string;
+  targetFolderId: string;
+  organizationName: string;
+  scale: Vector3;
+  layers: LayerLink[];
+};
+
+export function createDatasetComposition(datastoreUrl: string, payload: DatasetCompositionArgs) {
+  return doWithToken((token) =>
+    Request.sendJSONReceiveJSON(`${datastoreUrl}/data/datasets/compose?token=${token}`, {
+      data: payload,
+    }),
+  );
+}
+
 export function createResumableUpload(datastoreUrl: string, uploadId: string): Promise<any> {
   // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'file' implicitly has an 'any' type.
   const generateUniqueIdentifier = (file) => {

frontend/javascripts/admin/dataset/composition_wizard/01_select_import_type.tsx

@@ -0,0 +1,55 @@
+import { Button, Radio, RadioChangeEvent, Space } from "antd";
+import React from "react";
+import { WizardComponentProps } from "./common";
+
+export default function SelectImportType({
+  wizardContext,
+  setWizardContext,
+}: WizardComponentProps) {
+  const { composeMode } = wizardContext;
+
+  const onNext = () => {
+    setWizardContext((oldContext) => ({
+      ...oldContext,
+      currentWizardStep: composeMode === "WITHOUT_TRANSFORMS" ? "SelectDatasets" : "UploadFiles",
+    }));
+  };
+  const onChange = (e: RadioChangeEvent) => {
+    setWizardContext((oldContext) => ({
+      ...oldContext,
+      composeMode: e.target.value,
+    }));
+  };
+
+  return (
+    <div>
+      <div style={{ marginBottom: 8 }}>
+        You can create a new dataset by composing existing datasets together. There are three
+        different ways to accomplish this:
+        <ul>
+          <li>Select existing datasets which should be combined without any transforms</li>
+          <li>
+            Create landmarks nodes using the skeleton tool in two datasets. Download the annotations
+            as NML and upload these here again.
+          </li>
+          <li>Import a landmark CSV as it can be exported by Big Warp.</li>
+        </ul>
+        In all three cases, you can tweak which layers should be used later.
+      </div>
+      <div>
+        <Radio.Group onChange={onChange} value={composeMode}>
+          <Space direction="vertical">
+            <Radio value={"WITHOUT_TRANSFORMS"}>Combine datasets without any transforms</Radio>
+            <Radio value={"WK_ANNOTATIONS"}>
+              Combine datasets by using skeleton annotations (NML)
+            </Radio>
+            <Radio value={"BIG_WARP"}>Combine datasets by using BigWarp landmarks (CSV)</Radio>
+          </Space>
+        </Radio.Group>
+      </div>
+      <Button type="primary" style={{ marginTop: 16 }} onClick={onNext}>
+        Next
+      </Button>
+    </div>
+  );
+}

frontend/javascripts/admin/dataset/composition_wizard/02_upload_files.tsx

@@ -0,0 +1,204 @@
+import { FileExcelOutlined } from "@ant-design/icons";
+import { Button } from "antd";
+import Upload, { UploadChangeParam, UploadFile } from "antd/lib/upload";
+import { AsyncButton } from "components/async_clickables";
+import { readFileAsText } from "libs/read_file";
+import Toast from "libs/toast";
+import { SoftError, values } from "libs/utils";
+import _ from "lodash";
+import { Vector3 } from "oxalis/constants";
+import { parseNml } from "oxalis/model/helpers/nml_helpers";
+import React from "react";
+import { tryToFetchDatasetsByName, WizardComponentProps, WizardContext, FileList } from "./common";
+import ErrorHandling from "libs/error_handling";
+
+const EXPECTED_VALUE_COUNT_PER_CSV_LINE = 8;
+
+export default function UploadFiles({ wizardContext, setWizardContext }: WizardComponentProps) {
+  const fileList = wizardContext.fileList;
+  const handleChange = async (info: UploadChangeParam<UploadFile<any>>) => {
+    setWizardContext((oldContext) => ({
+      ...oldContext,
+      fileList: info.fileList,
+    }));
+  };
+
+  const onPrev = () => {
+    setWizardContext((oldContext) => ({
+      ...oldContext,
+      currentWizardStep: "SelectImportType",
+    }));
+  };
+  const onNext = async () => {
+    try {
+      let newContextPartial: Partial<WizardContext> | null = null;
+      if (wizardContext.composeMode === "BIG_WARP") {
+        newContextPartial = await parseBigWarpFile(fileList);
+      } else if (wizardContext.composeMode === "WK_ANNOTATIONS") {
+        newContextPartial = await parseNmlFiles(fileList);
+      } else {
+        throw new Error("Unexpected compose mode: " + wizardContext.composeMode);
+      }
+      setWizardContext((oldContext) => ({
+        ...oldContext,
+        ...newContextPartial,
+      }));
+    } catch (exception) {
+      if (exception instanceof SoftError) {
+        Toast.warning(exception.message);
+      } else {
+        Toast.error(
+          "An error occurred while importing the uploaded files. See the Browser's console for more details.",
+        );
+        ErrorHandling.notify(exception as Error);
+        console.error(exception);
+      }
+    }
+  };
+
+  return (
+    <div>
+      {wizardContext.composeMode === "BIG_WARP" ? (
+        <p>
+          Please upload one CSV file that was exported by BigWarp. Note that the first dataset
+          referenced by the CSV file will be transformed to the second referenced dataset.
+        </p>
+      ) : (
+        <p>
+          Please upload two NML files that contain landmarks that you created with WEBKNOSSOS. Note
+          that the dataset that belongs to the first NML will be transformed to the dataset that
+          belongs to the second NML file. The skeletons in the NML files should have exactly one
+          node per tree. The n-th tree of the first NML is aligned with the n-th tree of the second
+          NML.
+        </p>
+      )}
+
+      <div>
+        <p>
+          Landmark files ({wizardContext.composeMode === "BIG_WARP" ? "1 CSV file" : "2 NML files"}
+          ):
+        </p>
+        <Upload.Dragger
+          name="files"
+          fileList={fileList}
+          onChange={handleChange}
+          beforeUpload={() => false}
+          maxCount={wizardContext.composeMode === "BIG_WARP" ? 1 : 2}
+          multiple
+        >
+          <p className="ant-upload-drag-icon">
+            <FileExcelOutlined
+              style={{
+                margin: 0,
+                fontSize: 35,
+              }}
+            />
+          </p>
+          <p className="ant-upload-text">Drag your landmark file(s) to this area</p>
+        </Upload.Dragger>
+      </div>
+
+      <Button style={{ marginTop: 16 }} onClick={onPrev}>
+        Back
+      </Button>
+
+      <AsyncButton type="primary" style={{ marginTop: 16, marginLeft: 8 }} onClick={onNext}>
+        Next
+      </AsyncButton>
+    </div>
+  );
+}
+
+async function parseBigWarpFile(fileList: FileList): Promise<Partial<WizardContext>> {
+  const sourcePoints: Vector3[] = [];
+  const targetPoints: Vector3[] = [];
+  if (fileList.length !== 1 || fileList[0]?.originFileObj == null) {
+    throw new SoftError("Expected exactly one CSV file.");
+  }
+
+  const csv = await readFileAsText(fileList[0]?.originFileObj);
+  const lines = csv.split("\n");
+  for (const line of lines) {
+    const fields = line.split(",");
+    if (fields.length !== EXPECTED_VALUE_COUNT_PER_CSV_LINE) {
+      if (line.trim() !== "") {
+        throw new SoftError(
+          `Cannot interpret line in CSV file. Expected ${EXPECTED_VALUE_COUNT_PER_CSV_LINE} values, got ${fields.length}.`,
+        );
+      }
+      continue;
+    }
+    const [_pointName, enabled, x1, y1, z1, x2, y2, z2] = fields;
+
+    if (enabled) {
+      const source = [x1, y1, z1].map((el) => parseInt(el.replaceAll('"', ""))) as Vector3;
+      const target = [x2, y2, z2].map((el) => parseInt(el.replaceAll('"', ""))) as Vector3;
+      sourcePoints.push(source);
+      targetPoints.push(target);
+    }
+  }
+
+  return {
+    sourcePoints,
+    targetPoints,
+    datasets: [],
+    currentWizardStep: "SelectDatasets",
+  };
+}
+
+async function parseNmlFiles(fileList: FileList): Promise<Partial<WizardContext> | null> {
+  const sourcePoints: Vector3[] = [];
+  const targetPoints: Vector3[] = [];
+  if (fileList.length !== 2) {
+    throw new SoftError("Expected exactly two NML files.");
+  }
+
+  const nmlString1 = await readFileAsText(fileList[0]?.originFileObj!);
+  const nmlString2 = await readFileAsText(fileList[1]?.originFileObj!);
+
+  if (nmlString1 === "" || nmlString2 === "") {
+    throw new SoftError("NML files should not be empty.");
+  }
+
+  const { trees: trees1, datasetName: datasetName1 } = await parseNml(nmlString1);
+  const { trees: trees2, datasetName: datasetName2 } = await parseNml(nmlString2);
+
+  if (!datasetName1 || !datasetName2) {
+    throw new SoftError("Could not extract dataset names.");
+  }
+
+  const nodes1 = Array.from(
+    values(trees1)
+      .map((tree) => Array.from(tree.nodes.values())[0])
+      .values(),
+  );
+  const nodes2 = Array.from(
+    values(trees2)
+      .map((tree) => Array.from(tree.nodes.values())[0])
+      .values(),
+  );
+
+  for (const [node1, node2] of _.zip(nodes1, nodes2)) {
+    if ((node1 == null) !== (node2 == null)) {
+      throw new SoftError(
+        "A tree was empty while its corresponding tree wasn't. Ensure that the NML structures match each other.",
+      );
+    }
+    if (node1 != null && node2 != null) {
+      sourcePoints.push(node1.position);
+      targetPoints.push(node2.position);
+    }
+  }
+
+  const datasets = await tryToFetchDatasetsByName(
+    [datasetName1, datasetName2],
+    "Could not derive datasets from NML. Please specify these manually.",
+  );
+
+  return {
+    datasets: datasets || [],
+    sourcePoints,
+    targetPoints,
+    currentWizardStep: "SelectDatasets",
+  };
+}