Support uploading N5 layers without multiscale attributes

CHANGELOG.unreleased.md

@@ -22,6 +22,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added an action to delete erronous, unimported datasets directly from the dashboard. [#7448](https://github.com/scalableminds/webknossos/pull/7448)
 - Added support for `window`, `active`, `inverted` keys from the `omero` info in the NGFF metadata. [7685](https://github.com/scalableminds/webknossos/pull/7685)
 - Added getSegment function to JavaScript API. Also, createSegmentGroup returns the id of the new group now. [#7694](https://github.com/scalableminds/webknossos/pull/7694)
+- Added support for importing N5 datasets without multiscales metadata. [#7664](https://github.com/scalableminds/webknossos/pull/7664)
 
 ### Changed
 - Datasets stored in WKW format are no longer loaded with memory mapping, reducing memory demands. [#7528](https://github.com/scalableminds/webknossos/pull/7528)

util/src/main/scala/com/scalableminds/util/io/PathUtils.scala

@@ -196,4 +196,24 @@ trait PathUtils extends LazyLogging {
     FileUtils.moveDirectory(tmpPath.toFile, dst.toFile)
   }
 
+  def recurseSubdirsUntil(path: Path, condition: Path => Boolean, maxDepth: Int = 10): Box[Path] = {
+    def recurse(p: Path, depth: Int): Box[Path] =
+      if (depth > maxDepth) {
+        Failure("Max depth reached")
+      } else if (condition(p)) {
+        Full(p)
+      } else {
+        val subdirs = listDirectories(p, silent = true)
+        subdirs.flatMap { dirs =>
+          dirs.foldLeft(Failure("No matching subdir found"): Box[Path]) { (acc, dir) =>
+            acc match {
+              case Full(_) => acc
+              case _       => recurse(dir, depth + 1)
+            }
+          }
+        }
+      }
+    recurse(path, 0)
+  }
+
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreLocalLayerService.scala

@@ -1,7 +1,9 @@
 package com.scalableminds.webknossos.datastore.explore
 
 import com.scalableminds.util.geometry.Vec3Int
+import com.scalableminds.util.io.PathUtils
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header
 import com.scalableminds.webknossos.datastore.models.datasource.{
   DataLayerWithMagLocators,
   DataSource,
@@ -31,7 +33,8 @@ class ExploreLocalLayerService @Inject()(dataVaultService: DataVaultService)
         exploreLocalNgffArray(path, dataSourceId),
         exploreLocalZarrArray(path, dataSourceId, layerDirectory),
         exploreLocalNeuroglancerPrecomputed(path, dataSourceId, layerDirectory),
-        exploreLocalN5Multiscales(path, dataSourceId, layerDirectory)
+        exploreLocalN5Multiscales(path, dataSourceId, layerDirectory),
+        exploreLocalN5Array(path, dataSourceId)
       )
       dataSource <- Fox.firstSuccess(explored) ?~> "Could not explore local data source"
     } yield dataSource
@@ -74,6 +77,31 @@ class ExploreLocalLayerService @Inject()(dataVaultService: DataVaultService)
       new N5MultiscalesExplorer
     )(path, dataSourceId, layerDirectory)
 
+  private def exploreLocalN5Array(path: Path, dataSourceId: DataSourceId)(
+      implicit ec: ExecutionContext): Fox[DataSourceWithMagLocators] =
+    for {
+      _ <- Fox.successful(())
+      // Go down subdirectories until we find a directory with an attributes.json file that matches N5Header
+      layerPath <- PathUtils
+        .recurseSubdirsUntil(
+          path,
+          (p: Path) =>
+            try {
+              val attributesBytes = Files.readAllBytes(p.resolve(N5Header.FILENAME_ATTRIBUTES_JSON))
+              Json.parse(new String(attributesBytes)).validate[N5Header].isSuccess
+            } catch {
+              case _: Exception => false
+          }
+        )
+        .getOrElse(path)
+      explored <- exploreLocalLayer(
+        layers =>
+          layers.map(l =>
+            l.mapped(magMapping = m => m.copy(path = m.path.map(_.stripPrefix(path.toAbsolutePath.toUri.toString))))),
+        new N5ArrayExplorer
+      )(layerPath, dataSourceId, "")
+    } yield explored
+
   private def selectLastDirectory(l: DataLayerWithMagLocators) =
     l.mapped(magMapping = m => m.copy(path = m.path.map(_.split("/").last)))
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala

@@ -7,7 +7,7 @@ import com.scalableminds.util.tools.{BoxImplicits, Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.dataformats.wkw.WKWDataFormat.FILENAME_HEADER_WKW
 import com.scalableminds.webknossos.datastore.dataformats.wkw.{WKWDataLayer, WKWSegmentationLayer}
 import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header.FILENAME_ATTRIBUTES_JSON
-import com.scalableminds.webknossos.datastore.datareaders.n5.N5Metadata
+import com.scalableminds.webknossos.datastore.datareaders.n5.{N5Header, N5Metadata}
 import com.scalableminds.webknossos.datastore.datareaders.precomputed.PrecomputedHeader.FILENAME_INFO
 import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadata.FILENAME_DOT_ZATTRS
 import com.scalableminds.webknossos.datastore.datareaders.zarr.ZarrHeader.FILENAME_DOT_ZARRAY
@@ -240,7 +240,7 @@ class UploadService @Inject()(dataSourceRepository: DataSourceRepository,
         uploadedDataSourceType = guessTypeOfUploadedDataSource(unpackToDir)
         _ <- uploadedDataSourceType match {
           case UploadedDataSourceType.ZARR | UploadedDataSourceType.NEUROGLANCER_PRECOMPUTED |
-              UploadedDataSourceType.N5 =>
+              UploadedDataSourceType.N5_MULTISCALES | UploadedDataSourceType.N5_ARRAY =>
             exploreLocalDatasource(unpackToDir, dataSourceId, uploadedDataSourceType)
           case UploadedDataSourceType.EXPLORED => Fox.successful(())
           case UploadedDataSourceType.ZARR_MULTILAYER | UploadedDataSourceType.NEUROGLANCER_MULTILAYER |
@@ -376,8 +376,10 @@ class UploadService @Inject()(dataSourceRepository: DataSourceRepository,
       UploadedDataSourceType.NEUROGLANCER_MULTILAYER
     } else if (looksLikeN5Multilayer(dataSourceDir).openOr(false)) {
       UploadedDataSourceType.N5_MULTILAYER
-    } else if (looksLikeN5Layer(dataSourceDir).openOr(false)) {
-      UploadedDataSourceType.N5
+    } else if (looksLikeN5MultiscalesLayer(dataSourceDir).openOr(false)) {
+      UploadedDataSourceType.N5_MULTISCALES
+    } else if (looksLikeN5Array(dataSourceDir).openOr(false)) {
+      UploadedDataSourceType.N5_ARRAY
     } else {
       UploadedDataSourceType.WKW
     }
@@ -396,23 +398,55 @@ class UploadService @Inject()(dataSourceRepository: DataSourceRepository,
   private def looksLikeNeuroglancerPrecomputed(dataSourceDir: Path, maxDepth: Int): Box[Boolean] =
     containsMatchingFile(List(FILENAME_INFO), dataSourceDir, maxDepth)
 
-  private def looksLikeN5Layer(dataSourceDir: Path): Box[Boolean] =
+  private def looksLikeN5MultiscalesLayer(dataSourceDir: Path): Box[Boolean] =
     for {
       attributesFiles <- PathUtils.listFilesRecursive(dataSourceDir,
                                                       silent = false,
                                                       maxDepth = 1,
                                                       filters = p => p.getFileName.toString == FILENAME_ATTRIBUTES_JSON)
+      _ <- bool2Box(attributesFiles.nonEmpty)
       _ <- Json.parse(new String(Files.readAllBytes(attributesFiles.head))).validate[N5Metadata]
     } yield true
 
+  /*
+  private def getN5MultilayerLayerDirectories(baseDir: Path): Box[List[Path]] =
+    for {
+      baseDirectories <- PathUtils.listDirectories(baseDir, silent = false)
+      directories <- if (baseDirectories.length == 1) { // Layers wrapped in another directory
+        PathUtils.listDirectories(baseDirectories.head, silent = false)
+      } else {
+        Full(baseDirectories)
+      }
+    } yield directories*/
+
   private def looksLikeN5Multilayer(dataSourceDir: Path): Box[Boolean] =
     for {
       _ <- containsMatchingFile(List(FILENAME_ATTRIBUTES_JSON), dataSourceDir, 1) // root attributes.json
       directories <- PathUtils.listDirectories(dataSourceDir, silent = false)
-      detectedLayerBoxes = directories.map(looksLikeN5Layer)
+      detectedLayerBoxes = directories.map(looksLikeN5MultiscalesLayer)
       _ <- bool2Box(detectedLayerBoxes.forall(_.openOr(false)))
     } yield true
 
+  private def looksLikeN5Array(dataSourceDir: Path): Box[Boolean] =
+    // Expected structure:
+    // dataSourceDir
+    //  - attributes.json (Root attributes, only contains N5 version)
+    //  - dataset
+    //     - scale dir
+    //        - directories 0 to n
+    //        - attributes.json (N5Header, dimension, compression,...)
+    for {
+      _ <- containsMatchingFile(List(FILENAME_ATTRIBUTES_JSON), dataSourceDir, 1) // root attributes.json
+      datasetDir <- PathUtils.listDirectories(dataSourceDir, silent = false).map(_.headOption)
+      scaleDirs <- datasetDir.map(PathUtils.listDirectories(_, silent = false)).getOrElse(Full(Seq.empty))
+      _ <- bool2Box(scaleDirs.length == 1) // Must be 1, otherwise it is a multiscale dataset
+      attributesFiles <- PathUtils.listFilesRecursive(scaleDirs.head,
+                                                      silent = false,
+                                                      maxDepth = 1,
+                                                      filters = p => p.getFileName.toString == FILENAME_ATTRIBUTES_JSON)
+      _ <- Json.parse(new String(Files.readAllBytes(attributesFiles.head))).validate[N5Header]
+    } yield true
+
   private def looksLikeExploredDataSource(dataSourceDir: Path): Box[Boolean] =
     containsMatchingFile(List(FILENAME_DATASOURCE_PROPERTIES_JSON), dataSourceDir, 1)
 
@@ -541,5 +575,6 @@ class UploadService @Inject()(dataSourceRepository: DataSourceRepository,
 }
 
 object UploadedDataSourceType extends Enumeration {
-  val ZARR, EXPLORED, ZARR_MULTILAYER, WKW, NEUROGLANCER_PRECOMPUTED, NEUROGLANCER_MULTILAYER, N5, N5_MULTILAYER = Value
+  val ZARR, EXPLORED, ZARR_MULTILAYER, WKW, NEUROGLANCER_PRECOMPUTED, NEUROGLANCER_MULTILAYER, N5_MULTISCALES,
+  N5_MULTILAYER, N5_ARRAY = Value
 }