Allow uploading zarr datasets

CHANGELOG.unreleased.md

@@ -11,6 +11,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 [Commits](https://github.com/scalableminds/webknossos/compare/23.11.0...HEAD)
 
 ### Added
+- Zarr datasets can now be directly uploaded to WEBKNOSSOS. [#7397](https://github.com/scalableminds/webknossos/pull/7397)
 - Added support for reading uint24 rgb layers in datasets with zarr2/zarr3/n5/neuroglancerPrecomputed format, as used for voxelytics predictions. [#7413](https://github.com/scalableminds/webknossos/pull/7413)
 
 ### Changed

app/models/dataset/explore/ExploreRemoteLayerService.scala

@@ -3,18 +3,18 @@ package models.dataset.explore
 import com.scalableminds.util.accesscontext.DBAccessContext
 import com.scalableminds.util.geometry.{Vec3Double, Vec3Int}
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
-import com.scalableminds.webknossos.datastore.dataformats.n5.{N5DataLayer, N5SegmentationLayer}
-import com.scalableminds.webknossos.datastore.dataformats.precomputed.{
-  PrecomputedDataLayer,
-  PrecomputedSegmentationLayer
-}
-import com.scalableminds.webknossos.datastore.dataformats.zarr3.{Zarr3DataLayer, Zarr3SegmentationLayer}
-import com.scalableminds.webknossos.datastore.dataformats.zarr._
-import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header
-import com.scalableminds.webknossos.datastore.datareaders.precomputed.PrecomputedHeader
-import com.scalableminds.webknossos.datastore.datareaders.zarr._
-import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3ArrayHeader
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
+import com.scalableminds.webknossos.datastore.explore.{
+  ExploreLayerService,
+  N5ArrayExplorer,
+  N5MultiscalesExplorer,
+  NgffExplorer,
+  PrecomputedExplorer,
+  RemoteLayerExplorer,
+  WebknossosZarrExplorer,
+  Zarr3ArrayExplorer,
+  ZarrArrayExplorer
+}
 import com.scalableminds.webknossos.datastore.models.datasource._
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptor}
@@ -33,7 +33,6 @@ import java.net.URI
 import javax.inject.Inject
 import scala.collection.mutable.ListBuffer
 import scala.concurrent.ExecutionContext
-import scala.util.Try
 
 case class ExploreRemoteDatasetParameters(remoteUri: String,
                                           credentialIdentifier: Option[String],
@@ -57,6 +56,7 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
                                           dataStoreDAO: DataStoreDAO,
                                           datasetService: DatasetService,
                                           wkSilhouetteEnvironment: WkSilhouetteEnvironment,
+                                          exploreLayerService: ExploreLayerService,
                                           rpc: RPC,
                                           wkConf: WkConf)
     extends FoxImplicits
@@ -79,16 +79,10 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
       layersWithVoxelSizes = exploredLayersNested.flatten
       preferredVoxelSize = parameters.flatMap(_.preferredVoxelSize).headOption
       _ <- bool2Fox(layersWithVoxelSizes.nonEmpty) ?~> "Detected zero layers"
-      rescaledLayersAndVoxelSize <- rescaleLayersByCommonVoxelSize(layersWithVoxelSizes, preferredVoxelSize) ?~> "Could not extract common voxel size from layers"
-      rescaledLayers = rescaledLayersAndVoxelSize._1
-      voxelSize = rescaledLayersAndVoxelSize._2
-      renamedLayers = makeLayerNamesUnique(rescaledLayers)
-      layersWithCoordinateTransformations = addCoordinateTransformationsToLayers(renamedLayers,
-                                                                                 preferredVoxelSize,
-                                                                                 voxelSize)
+      (layers, voxelSize) <- exploreLayerService.adaptLayersAndVoxelSize(layersWithVoxelSizes, preferredVoxelSize)
       dataSource = GenericDataSource[DataLayer](
         DataSourceId("", ""), // Frontend will prompt user for a good name
-        layersWithCoordinateTransformations,
+        layers,
         voxelSize
       )
     } yield dataSource
@@ -107,166 +101,14 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
       _ <- client.addDataSource(organization.name, datasetName, dataSource, folderId, userToken)
     } yield ()
 
-  private def makeLayerNamesUnique(layers: List[DataLayer]): List[DataLayer] = {
-    val namesSetMutable = scala.collection.mutable.Set[String]()
-    layers.map { layer: DataLayer =>
-      var nameCandidate = layer.name
-      var index = 1
-      while (namesSetMutable.contains(nameCandidate)) {
-        index += 1
-        nameCandidate = f"${layer.name}_$index"
-      }
-      namesSetMutable.add(nameCandidate)
-      if (nameCandidate == layer.name) {
-        layer
-      } else
-        layer match {
-          case l: ZarrDataLayer         => l.copy(name = nameCandidate)
-          case l: ZarrSegmentationLayer => l.copy(name = nameCandidate)
-          case l: N5DataLayer           => l.copy(name = nameCandidate)
-          case l: N5SegmentationLayer   => l.copy(name = nameCandidate)
-          case _                        => throw new Exception("Encountered unsupported layer format during explore remote")
-        }
-    }
-  }
-
-  private def addCoordinateTransformationsToLayers(layers: List[DataLayer],
-                                                   preferredVoxelSize: Option[Vec3Double],
-                                                   voxelSize: Vec3Double): List[DataLayer] =
-    layers.map(l => {
-      val coordinateTransformations = coordinateTransformationForVoxelSize(voxelSize, preferredVoxelSize)
-      l match {
-        case l: ZarrDataLayer                => l.copy(coordinateTransformations = coordinateTransformations)
-        case l: ZarrSegmentationLayer        => l.copy(coordinateTransformations = coordinateTransformations)
-        case l: N5DataLayer                  => l.copy(coordinateTransformations = coordinateTransformations)
-        case l: N5SegmentationLayer          => l.copy(coordinateTransformations = coordinateTransformations)
-        case l: PrecomputedDataLayer         => l.copy(coordinateTransformations = coordinateTransformations)
-        case l: PrecomputedSegmentationLayer => l.copy(coordinateTransformations = coordinateTransformations)
-        case l: Zarr3DataLayer               => l.copy(coordinateTransformations = coordinateTransformations)
-        case l: Zarr3SegmentationLayer       => l.copy(coordinateTransformations = coordinateTransformations)
-        case _                               => throw new Exception("Encountered unsupported layer format during explore remote")
-      }
-    })
-
-  private def isPowerOfTwo(x: Int): Boolean =
-    x != 0 && (x & (x - 1)) == 0
-
-  private def isPowerOfTwo(x: Double): Boolean = {
-    val epsilon = 0.0001
-    val l = math.log(x) / math.log(2)
-    math.abs(l - l.round.toDouble) < epsilon
-  }
-
-  private def magFromVoxelSize(minVoxelSize: Vec3Double, voxelSize: Vec3Double)(
-      implicit ec: ExecutionContext): Fox[Vec3Int] = {
-
-    val mag = (voxelSize / minVoxelSize).round.toVec3Int
-    for {
-      _ <- bool2Fox(isPowerOfTwo(mag.x) && isPowerOfTwo(mag.x) && isPowerOfTwo(mag.x)) ?~> s"invalid mag: $mag. Must all be powers of two"
-    } yield mag
-  }
-
-  private def checkForDuplicateMags(magGroup: List[Vec3Int])(implicit ec: ExecutionContext): Fox[Unit] =
-    for {
-      _ <- bool2Fox(magGroup.length == 1) ?~> s"detected mags are not unique, found $magGroup"
-    } yield ()
-
-  private def findBaseVoxelSize(minVoxelSize: Vec3Double, preferredVoxelSizeOpt: Option[Vec3Double]): Vec3Double =
-    preferredVoxelSizeOpt match {
-      case Some(preferredVoxelSize) =>
-        val baseMag = minVoxelSize / preferredVoxelSize
-        if (isPowerOfTwo(baseMag.x) && isPowerOfTwo(baseMag.y) && isPowerOfTwo(baseMag.z)) {
-          preferredVoxelSize
-        } else {
-          minVoxelSize
-        }
-      case None => minVoxelSize
-    }
-
-  private def coordinateTransformationForVoxelSize(
-      foundVoxelSize: Vec3Double,
-      preferredVoxelSize: Option[Vec3Double]): Option[List[CoordinateTransformation]] =
-    preferredVoxelSize match {
-      case None => None
-      case Some(voxelSize) =>
-        if (voxelSize == foundVoxelSize) { None } else {
-          val scale = foundVoxelSize / voxelSize
-          Some(
-            List(
-              CoordinateTransformation(CoordinateTransformationType.affine,
-                                       matrix = Some(
-                                         List(
-                                           List(scale.x, 0, 0, 0),
-                                           List(0, scale.y, 0, 0),
-                                           List(0, 0, scale.z, 0),
-                                           List(0, 0, 0, 1)
-                                         )))))
-        }
-    }
-
-  private def rescaleLayersByCommonVoxelSize(
-      layersWithVoxelSizes: List[(DataLayer, Vec3Double)],
-      preferredVoxelSize: Option[Vec3Double])(implicit ec: ExecutionContext): Fox[(List[DataLayer], Vec3Double)] = {
-    val allVoxelSizes = layersWithVoxelSizes
-      .flatMap(layerWithVoxelSize => {
-        val layer = layerWithVoxelSize._1
-        val voxelSize = layerWithVoxelSize._2
-
-        layer.resolutions.map(resolution => voxelSize * resolution.toVec3Double)
-      })
-      .toSet
-    val minVoxelSizeOpt = Try(allVoxelSizes.minBy(_.toTuple)).toOption
-
-    for {
-      minVoxelSize <- option2Fox(minVoxelSizeOpt)
-      baseVoxelSize = findBaseVoxelSize(minVoxelSize, preferredVoxelSize)
-      allMags <- Fox.combined(allVoxelSizes.map(magFromVoxelSize(baseVoxelSize, _)).toList) ?~> s"voxel sizes for layers are not uniform, got ${layersWithVoxelSizes
-        .map(_._2)}"
-      groupedMags = allMags.groupBy(_.maxDim)
-      _ <- Fox.combined(groupedMags.values.map(checkForDuplicateMags).toList)
-      rescaledLayers = layersWithVoxelSizes.map(layerWithVoxelSize => {
-        val layer = layerWithVoxelSize._1
-        val layerVoxelSize = layerWithVoxelSize._2
-        val magFactors = (layerVoxelSize / baseVoxelSize).toVec3Int
-        layer match {
-          case l: ZarrDataLayer =>
-            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
-                   boundingBox = l.boundingBox * magFactors)
-          case l: ZarrSegmentationLayer =>
-            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
-                   boundingBox = l.boundingBox * magFactors)
-          case l: N5DataLayer =>
-            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
-                   boundingBox = l.boundingBox * magFactors)
-          case l: N5SegmentationLayer =>
-            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
-                   boundingBox = l.boundingBox * magFactors)
-          case l: PrecomputedDataLayer =>
-            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
-                   boundingBox = l.boundingBox * magFactors)
-          case l: PrecomputedSegmentationLayer =>
-            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
-                   boundingBox = l.boundingBox * magFactors)
-          case l: Zarr3DataLayer =>
-            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
-                   boundingBox = l.boundingBox * magFactors)
-          case l: Zarr3SegmentationLayer =>
-            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
-                   boundingBox = l.boundingBox * magFactors)
-          case _ => throw new Exception("Encountered unsupported layer format during explore remote")
-        }
-      })
-    } yield (rescaledLayers, baseVoxelSize)
-  }
-
   private def exploreRemoteLayersForUri(
       layerUri: String,
       credentialIdentifier: Option[String],
       credentialSecret: Option[String],
       reportMutable: ListBuffer[String],
       requestingUser: User)(implicit ec: ExecutionContext): Fox[List[(DataLayer, Vec3Double)]] =
     for {
-      uri <- tryo(new URI(removeHeaderFileNamesFromUriSuffix(layerUri))) ?~> s"Received invalid URI: $layerUri"
+      uri <- tryo(new URI(exploreLayerService.removeHeaderFileNamesFromUriSuffix(layerUri))) ?~> s"Received invalid URI: $layerUri"
       _ <- bool2Fox(uri.getScheme != null) ?~> s"Received invalid URI: $layerUri"
       _ <- assertLocalPathInWhitelist(uri)
       credentialOpt = credentialService.createCredentialOpt(uri,
@@ -282,7 +124,7 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
         credentialId.map(_.toString),
         reportMutable,
         List(
-          new ZarrArrayExplorer,
+          new ZarrArrayExplorer(Vec3Int.ones, ec),
           new NgffExplorer,
           new WebknossosZarrExplorer,
           new N5ArrayExplorer,
@@ -298,17 +140,6 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
       bool2Fox(wkConf.Datastore.localFolderWhitelist.exists(whitelistEntry => uri.getPath.startsWith(whitelistEntry))) ?~> s"Absolute path ${uri.getPath} in local file system is not in path whitelist. Consider adding it to datastore.pathWhitelist"
     } else Fox.successful(())
 
-  private def removeHeaderFileNamesFromUriSuffix(uri: String): String =
-    if (uri.endsWith(N5Header.FILENAME_ATTRIBUTES_JSON)) uri.dropRight(N5Header.FILENAME_ATTRIBUTES_JSON.length)
-    else if (uri.endsWith(ZarrHeader.FILENAME_DOT_ZARRAY)) uri.dropRight(ZarrHeader.FILENAME_DOT_ZARRAY.length)
-    else if (uri.endsWith(NgffMetadata.FILENAME_DOT_ZATTRS)) uri.dropRight(NgffMetadata.FILENAME_DOT_ZATTRS.length)
-    else if (uri.endsWith(NgffGroupHeader.FILENAME_DOT_ZGROUP))
-      uri.dropRight(NgffGroupHeader.FILENAME_DOT_ZGROUP.length)
-    else if (uri.endsWith(PrecomputedHeader.FILENAME_INFO)) uri.dropRight(PrecomputedHeader.FILENAME_INFO.length)
-    else if (uri.endsWith(Zarr3ArrayHeader.FILENAME_ZARR_JSON))
-      uri.dropRight(Zarr3ArrayHeader.FILENAME_ZARR_JSON.length)
-    else uri
-
   private def exploreRemoteLayersForRemotePath(
       remotePath: VaultPath,
       credentialId: Option[String],

docs/datasets.md

@@ -76,7 +76,7 @@ For those wishing to automate dataset upload or to do it programmatically, check
 ### Uploading through the File System
 -- (Self-Hosted Instances Only)-- 
 
-On self-hosted instances, large datasets can be efficiently imported by placing them directly in the file system (WKW-format only):
+On self-hosted instances, large datasets can be efficiently imported by placing them directly in the file system (WKW-format or Zarr only):
 
 * Place the dataset at `<WEBKNOSSOS directory>/binaryData/<Organization name>/<Dataset name>`. For example `/opt/webknossos/binaryData/Springfield_University/great_dataset`.
 * Go to the [dataset view on the dashboard](./dashboard.md)

frontend/javascripts/admin/dataset/dataset_upload_view.tsx

@@ -477,12 +477,12 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
       return;
     }
 
-    let needsConversion = true;
     const fileExtensions = [];
+    const fileNames = [];
 
     for (const file of files) {
-      const filenameParts = file.name.split(".");
-      const fileExtension = filenameParts[filenameParts.length - 1].toLowerCase();
+      fileNames.push(file.name);
+      const fileExtension = Utils.getFileExtension(file.name);
       fileExtensions.push(fileExtension);
       sendAnalyticsEvent("add_files_to_upload", {
         fileExtension,
@@ -493,19 +493,9 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
           const reader = new Zip.ZipReader(new Zip.BlobReader(file));
           const entries = await reader.getEntries();
           await reader.close();
-          const wkwFile = entries.find((entry) =>
-            Utils.isFileExtensionEqualTo(entry.filename, "wkw"),
-          );
-          const needsConversion = wkwFile == null;
-          this.handleNeedsConversionInfo(needsConversion);
-
-          const nmlFile = entries.find((entry) =>
-            Utils.isFileExtensionEqualTo(entry.filename, "nml"),
-          );
-          if (nmlFile) {
-            Modal.error({
-              content: messages["dataset.upload_zip_with_nml"],
-            });
+          for (const entry of entries) {
+            fileNames.push(entry.filename);
+            fileExtensions.push(Utils.getFileExtension(entry.filename));
           }
         } catch (e) {
           console.error(e);
@@ -514,7 +504,6 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
             content: messages["dataset.upload_invalid_zip"],
           });
           const form = this.formRef.current;
-
           if (!form) {
             return;
           }
@@ -523,15 +512,30 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
             zipFile: [],
           });
         }
-        // We return here since not more than 1 zip archive is supported anyway.
+        // We return here since not more than 1 zip archive is supported anyway. This is guarded
+        // against via form validation.
         return;
-      } else if (fileExtension === "wkw") {
-        needsConversion = false;
       }
     }
 
     const countedFileExtensions = _.countBy(fileExtensions, (str) => str);
+    const containsExtension = (extension: string) => countedFileExtensions[extension] > 0;
+
+    if (containsExtension("nml")) {
+      Modal.error({
+        content: messages["dataset.upload_zip_with_nml"],
+      });
+    }
 
+    let needsConversion = true;
+    if (
+      containsExtension("wkw") ||
+      containsExtension("zarray") ||
+      fileNames.includes("datasource-properties.json") ||
+      fileNames.includes("zarr.json")
+    ) {
+      needsConversion = false;
+    }
     Object.entries(countedFileExtensions).map(([fileExtension, count]) =>
       sendAnalyticsEvent("add_files_to_upload", {
         fileExtension,
@@ -559,8 +563,8 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
       Modal.info({
         content: (
           <div>
-            The selected dataset does not seem to be in the WKW format. Please convert the dataset
-            using{" "}
+            The selected dataset does not seem to be in the WKW or Zarr format. Please convert the
+            dataset using{" "}
             <a
               target="_blank"
               href="https://github.com/scalableminds/webknossos-libs/tree/master/wkcuber#webknossos-cuber-wkcuber"

frontend/javascripts/libs/utils.ts

@@ -1170,3 +1170,9 @@ export const deepIterate = (obj: Obj | Obj[] | null, callback: (val: unknown) =>
     }
   });
 };
+
+export function getFileExtension(fileName: string): string {
+  const filenameParts = fileName.split(".");
+  const fileExtension = filenameParts[filenameParts.length - 1].toLowerCase();
+  return fileExtension;
+}

util/src/main/scala/com/scalableminds/util/io/PathUtils.scala

@@ -15,10 +15,10 @@ object PathUtils extends PathUtils
 
 trait PathUtils extends LazyLogging {
 
-  def directoryFilter(path: Path): Boolean =
+  private def directoryFilter(path: Path): Boolean =
     Files.isDirectory(path) && !Files.isHidden(path)
 
-  def fileFilter(path: Path): Boolean =
+  private def fileFilter(path: Path): Boolean =
     !Files.isDirectory(path)
 
   def fileExtensionFilter(ext: String)(path: Path): Boolean =