-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* WIP add zarr3 streaming - Added mag/zarr.json & mag/coords route for datasets (not annotations) * manage minimal zarr3 support for viewing a dataset and viewing an annotation * add basic zarr 3 ngff v2 group header route * add zarr 3 ngff v2 group header route for annotations * merge Zarr3StreamController into ZarrStreamingController * ensure full match when parsing zarr coordinates to avoid parsing any non-numerical characters * make NgffMetadataV2 nd compatible * refactor code * remove Zarr3StreamingController.scala * minor code fixes; mostly remove unused imports * remove fixed full length match of zarr coordinate regex parsing; - wk seems to send c.0.x.y.z requests and the c at the start does not match with the regex resulting in invalid coordinate parsing * add c. as a prefix to the coordinate parsing regex as defined by the zarr spec * exclude leading c from the zarr3 coordinate parsing * apply pr feedback * remove unused imports and format code * add datasource-properties.json and dir listing routes to zarr 3 streaming * format code * add changelog entry * fix Zarr3GroupHeader json serialization format * fix backend format * Update webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala * add comments about why manual json serializer is needed * use chunk key encoding version 2 as zarr 2 stream also has no prefix of "c." when addressing an array * do not include a prefixed "c." when parsing coordinates -> works with zarr 2 and 3 streaming * remove outdated comment * update error message (remove hint to include a prepending "c." --------- Co-authored-by: Norman Rzepka <[email protected]>
- Loading branch information
1 parent
5fedb9c
commit c126859
Showing
17 changed files
with
729 additions
and
293 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
410 changes: 271 additions & 139 deletions
410
...tore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 34 additions & 10 deletions
44
...e/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,15 +1,39 @@ | ||
| package com.scalableminds.webknossos.datastore.dataformats.zarr | ||
|
|
||
| import com.scalableminds.util.tools.Fox | ||
| import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox} | ||
| import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate | ||
| import play.api.http.Status.NOT_FOUND | ||
| import play.api.i18n.{Messages, MessagesProvider} | ||
|
|
||
| import scala.concurrent.ExecutionContext | ||
|
|
||
| object ZarrCoordinatesParser { | ||
| def parseDotCoordinates( | ||
| cxyz: String, | ||
| ): Option[(Int, Int, Int, Int)] = { | ||
| val singleRx = "\\s*([0-9]+).([0-9]+).([0-9]+).([0-9]+)\\s*".r | ||
|
|
||
| cxyz match { | ||
| case singleRx(c, x, y, z) => | ||
| Some(Integer.parseInt(c), Integer.parseInt(x), Integer.parseInt(y), Integer.parseInt(z)) | ||
| case _ => None | ||
| } | ||
|
|
||
| def parseNDimensionalDotCoordinates( | ||
| coordinates: String, | ||
| )(implicit ec: ExecutionContext, m: MessagesProvider): Fox[(Int, Int, Int, Option[List[AdditionalCoordinate]])] = { | ||
| val ndCoordinatesRx = "^\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r | ||
|
|
||
| for { | ||
| parsedCoordinates <- ndCoordinatesRx | ||
| .findFirstIn(coordinates) | ||
| .map(m => m.split('.').map(coord => Integer.parseInt(coord))) ?~> | ||
| Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND | ||
| channelCoordinate <- parsedCoordinates.headOption ~> NOT_FOUND | ||
| _ <- bool2Fox(channelCoordinate == 0) ?~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND | ||
| _ <- bool2Fox(parsedCoordinates.length >= 4) ?~> "zarr.notEnoughCoordinates" ~> NOT_FOUND | ||
| (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), | ||
| parsedCoordinates(parsedCoordinates.length - 2), | ||
| parsedCoordinates(parsedCoordinates.length - 1)) | ||
| additionalCoordinates = if (parsedCoordinates.length > 4) | ||
| Some( | ||
| parsedCoordinates | ||
| .slice(1, parsedCoordinates.length - 3) | ||
| .zipWithIndex | ||
| .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) | ||
| .toList) | ||
| else None | ||
| } yield (x, y, z, additionalCoordinates) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
...astore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package com.scalableminds.webknossos.datastore.datareaders.zarr | ||
|
|
||
| import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} | ||
| import com.scalableminds.webknossos.datastore.models.VoxelSize | ||
| import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis | ||
| import play.api.libs.json.{Json, OFormat} | ||
|
|
||
| // See suggested changes to version v0.5 here together with an example: https://ngff.openmicroscopy.org/rfc/2/index.html#examples | ||
| case class NgffMultiscalesItemV0_5( | ||
| // Ngff V0.5 no longer has the version inside the multiscale field. | ||
| name: Option[String], | ||
| axes: List[NgffAxis] = List( | ||
| NgffAxis(name = "c", `type` = "channel"), | ||
| NgffAxis(name = "x", `type` = "space", unit = Some("nanometer")), | ||
| NgffAxis(name = "y", `type` = "space", unit = Some("nanometer")), | ||
| NgffAxis(name = "z", `type` = "space", unit = Some("nanometer")), | ||
| ), | ||
| datasets: List[NgffDataset] | ||
| ) | ||
|
|
||
| object NgffMultiscalesItemV0_5 { | ||
| implicit val jsonFormat: OFormat[NgffMultiscalesItemV0_5] = Json.format[NgffMultiscalesItemV0_5] | ||
| } | ||
|
|
||
| case class NgffMetadataV0_5(version: String, | ||
| multiscales: List[NgffMultiscalesItemV0_5], | ||
| omero: Option[NgffOmeroMetadata]) | ||
|
|
||
| object NgffMetadataV0_5 { | ||
| def fromNameVoxelSizeAndMags(dataLayerName: String, | ||
| dataSourceVoxelSize: VoxelSize, | ||
| mags: List[Vec3Int], | ||
| additionalAxes: Option[Seq[AdditionalAxis]], | ||
| version: String = "0.5"): NgffMetadataV0_5 = { | ||
| val datasets = mags.map( | ||
| mag => | ||
| NgffDataset( | ||
| path = mag.toMagLiteral(allowScalar = true), | ||
| List(NgffCoordinateTransformation( | ||
| scale = Some(List[Double](1.0) ++ (dataSourceVoxelSize.factor * Vec3Double(mag)).toList))) | ||
| )) | ||
| val lengthUnitStr = dataSourceVoxelSize.unit.toString | ||
| val axes = List(NgffAxis(name = "c", `type` = "channel")) ++ additionalAxes | ||
| .getOrElse(List.empty) | ||
| .zipWithIndex | ||
| .map(axisAndIndex => NgffAxis(name = s"t${axisAndIndex._2}", `type` = "space", unit = Some(lengthUnitStr))) ++ List( | ||
| NgffAxis(name = "x", `type` = "space", unit = Some(lengthUnitStr)), | ||
| NgffAxis(name = "y", `type` = "space", unit = Some(lengthUnitStr)), | ||
| NgffAxis(name = "z", `type` = "space", unit = Some(lengthUnitStr)), | ||
| ) | ||
| NgffMetadataV0_5(version, | ||
| multiscales = | ||
| List(NgffMultiscalesItemV0_5(name = Some(dataLayerName), datasets = datasets, axes = axes)), | ||
| None) | ||
| } | ||
|
|
||
| implicit val jsonFormat: OFormat[NgffMetadataV0_5] = Json.format[NgffMetadataV0_5] | ||
| } |
54 changes: 54 additions & 0 deletions
54
...om/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| package com.scalableminds.webknossos.datastore.datareaders.zarr | ||
|
|
||
| import com.scalableminds.webknossos.datastore.models | ||
| import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize} | ||
| import net.liftweb.common.{Box, Failure, Full} | ||
| import play.api.libs.json.{Json, OFormat} | ||
|
|
||
| case class NgffCoordinateTransformation(`type`: String = "scale", scale: Option[List[Double]]) | ||
|
|
||
| object NgffCoordinateTransformation { | ||
| implicit val jsonFormat: OFormat[NgffCoordinateTransformation] = Json.format[NgffCoordinateTransformation] | ||
| } | ||
|
|
||
| case class NgffDataset(path: String, coordinateTransformations: List[NgffCoordinateTransformation]) | ||
|
|
||
| object NgffDataset { | ||
| implicit val jsonFormat: OFormat[NgffDataset] = Json.format[NgffDataset] | ||
| } | ||
|
|
||
| case class NgffAxis(name: String, `type`: String, unit: Option[String] = None) { | ||
|
|
||
| def lengthUnit: Box[models.LengthUnit.Value] = | ||
| if (`type` != "space") | ||
| Failure(f"Could not convert NGFF unit $name of type ${`type`} to LengthUnit") | ||
| else { | ||
| unit match { | ||
| case None | Some("") => Full(VoxelSize.DEFAULT_UNIT) | ||
| case Some(someUnit) => LengthUnit.fromString(someUnit) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| object NgffAxis { | ||
| implicit val jsonFormat: OFormat[NgffAxis] = Json.format[NgffAxis] | ||
| } | ||
|
|
||
| case class NgffOmeroMetadata(channels: List[NgffChannelAttributes]) | ||
| object NgffOmeroMetadata { | ||
| implicit val jsonFormat: OFormat[NgffOmeroMetadata] = Json.format[NgffOmeroMetadata] | ||
| } | ||
|
|
||
| case class NgffChannelWindow(min: Double, max: Double, start: Double, end: Double) | ||
| object NgffChannelWindow { | ||
| implicit val jsonFormat: OFormat[NgffChannelWindow] = Json.format[NgffChannelWindow] | ||
| } | ||
|
|
||
| case class NgffChannelAttributes(color: Option[String], | ||
| label: Option[String], | ||
| window: Option[NgffChannelWindow], | ||
| inverted: Option[Boolean], | ||
| active: Option[Boolean]) | ||
| object NgffChannelAttributes { | ||
| implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
...store/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package com.scalableminds.webknossos.datastore.datareaders.zarr3 | ||
|
|
||
| import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV0_5 | ||
| import play.api.libs.json._ | ||
|
|
||
| case class Zarr3GroupHeader( | ||
| zarr_format: Int, // must be 3 | ||
| node_type: String, // must be "group" | ||
| ngffMetadata: Option[NgffMetadataV0_5], | ||
| ) | ||
|
|
||
| object Zarr3GroupHeader { | ||
| implicit object Zarr3GroupHeaderFormat extends Format[Zarr3GroupHeader] { | ||
| override def reads(json: JsValue): JsResult[Zarr3GroupHeader] = | ||
| for { | ||
| zarr_format <- (json \ "zarr_format").validate[Int] | ||
| node_type <- (json \ "node_type").validate[String] | ||
| // Read the metadata from the correct json path. | ||
| ngffMetadata <- (json \ "attributes" \ "ome").validateOpt[NgffMetadataV0_5] | ||
| } yield | ||
| Zarr3GroupHeader( | ||
| zarr_format, | ||
| node_type, | ||
| ngffMetadata, | ||
| ) | ||
|
|
||
| override def writes(zarrArrayGroup: Zarr3GroupHeader): JsValue = | ||
| Json.obj( | ||
| "zarr_format" -> zarrArrayGroup.zarr_format, | ||
| "node_type" -> zarrArrayGroup.node_type, | ||
| // Enforce correct path for ngffMetadata in the json. | ||
| "attributes" -> Json.obj("ome" -> zarrArrayGroup.ngffMetadata), | ||
| ) | ||
| } | ||
| } |
Oops, something went wrong.