-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* WIP add zarr3 streaming - Added mag/zarr.json & mag/coords route for datasets (not annotations) * manage minimal zarr3 support for viewing a dataset and viewing an annotation * add basic zarr 3 ngff v2 group header route * add zarr 3 ngff v2 group header route for annotations * merge Zarr3StreamController into ZarrStreamingController * ensure full match when parsing zarr coordinates to avoid parsing any non-numerical characters * make NgffMetadataV2 nd compatible * refactor code * remove Zarr3StreamingController.scala * minor code fixes; mostly remove unused imports * remove fixed full length match of zarr coordinate regex parsing; - wk seems to send c.0.x.y.z requests and the c at the start does not match with the regex resulting in invalid coordinate parsing * add c. as a prefix to the coordinate parsing regex as defined by the zarr spec * exclude leading c from the zarr3 coordinate parsing * apply pr feedback * remove unused imports and format code * add datasource-properties.json and dir listing routes to zarr 3 streaming * format code * add changelog entry * fix Zarr3GroupHeader json serialization format * fix backend format * Update webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala * add comments about why manual json serializer is needed --------- Co-authored-by: Norman Rzepka <[email protected]>
- Loading branch information
1 parent
81d2b29
commit cb1331b
Showing
17 changed files
with
732 additions
and
284 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
410 changes: 271 additions & 139 deletions
410
...tore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
38 changes: 37 additions & 1 deletion
38
...e/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,15 +1,51 @@ | ||
| package com.scalableminds.webknossos.datastore.dataformats.zarr | ||
|
|
||
| import com.scalableminds.util.tools.Fox | ||
| import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox} | ||
| import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate | ||
| import play.api.http.Status.NOT_FOUND | ||
| import play.api.i18n.{Messages, MessagesProvider} | ||
|
|
||
| import scala.concurrent.ExecutionContext | ||
|
|
||
| object ZarrCoordinatesParser { | ||
| def parseDotCoordinates( | ||
| cxyz: String, | ||
| ): Option[(Int, Int, Int, Int)] = { | ||
| val singleRx = "\\s*([0-9]+).([0-9]+).([0-9]+).([0-9]+)\\s*".r | ||
| val singleRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\s*$".r | ||
|
|
||
| cxyz match { | ||
| case singleRx(c, x, y, z) => | ||
| Some(Integer.parseInt(c), Integer.parseInt(x), Integer.parseInt(y), Integer.parseInt(z)) | ||
| case _ => None | ||
| } | ||
| } | ||
|
|
||
| def parseNDimensionalDotCoordinates( | ||
| coordinates: String, | ||
| )(implicit ec: ExecutionContext, m: MessagesProvider): Fox[(Int, Int, Int, Option[List[AdditionalCoordinate]])] = { | ||
| val ndCoordinatesRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r | ||
| // The tail cuts off the leading "c" form the "c." at the beginning of coordinates. | ||
|
|
||
| for { | ||
| parsedCoordinates <- ndCoordinatesRx | ||
| .findFirstIn(coordinates) | ||
| .map(m => m.split('.').tail.map(coord => Integer.parseInt(coord))) ?~> | ||
| Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND | ||
| channelCoordinate <- parsedCoordinates.headOption ~> NOT_FOUND | ||
| _ <- bool2Fox(channelCoordinate == 0) ?~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND | ||
| _ <- bool2Fox(parsedCoordinates.length >= 4) ?~> "zarr.notEnoughCoordinates" ~> NOT_FOUND | ||
| (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), | ||
| parsedCoordinates(parsedCoordinates.length - 2), | ||
| parsedCoordinates(parsedCoordinates.length - 1)) | ||
| additionalCoordinates = if (parsedCoordinates.length > 4) | ||
| Some( | ||
| parsedCoordinates | ||
| .slice(1, parsedCoordinates.length - 3) | ||
| .zipWithIndex | ||
| .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) | ||
| .toList) | ||
| else None | ||
| } yield (x, y, z, additionalCoordinates) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
...astore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package com.scalableminds.webknossos.datastore.datareaders.zarr | ||
|
|
||
| import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} | ||
| import com.scalableminds.webknossos.datastore.models.VoxelSize | ||
| import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis | ||
| import play.api.libs.json.{Json, OFormat} | ||
|
|
||
| // See suggested changes to version v0.5 here together with an example: https://ngff.openmicroscopy.org/rfc/2/index.html#examples | ||
| case class NgffMultiscalesItemV0_5( | ||
| // Ngff V0.5 no longer has the version inside the multiscale field. | ||
| name: Option[String], | ||
| axes: List[NgffAxis] = List( | ||
| NgffAxis(name = "c", `type` = "channel"), | ||
| NgffAxis(name = "x", `type` = "space", unit = Some("nanometer")), | ||
| NgffAxis(name = "y", `type` = "space", unit = Some("nanometer")), | ||
| NgffAxis(name = "z", `type` = "space", unit = Some("nanometer")), | ||
| ), | ||
| datasets: List[NgffDataset] | ||
| ) | ||
|
|
||
| object NgffMultiscalesItemV0_5 { | ||
| implicit val jsonFormat: OFormat[NgffMultiscalesItemV0_5] = Json.format[NgffMultiscalesItemV0_5] | ||
| } | ||
|
|
||
| case class NgffMetadataV0_5(version: String, | ||
| multiscales: List[NgffMultiscalesItemV0_5], | ||
| omero: Option[NgffOmeroMetadata]) | ||
|
|
||
| object NgffMetadataV0_5 { | ||
| def fromNameVoxelSizeAndMags(dataLayerName: String, | ||
| dataSourceVoxelSize: VoxelSize, | ||
| mags: List[Vec3Int], | ||
| additionalAxes: Option[Seq[AdditionalAxis]], | ||
| version: String = "0.5"): NgffMetadataV0_5 = { | ||
| val datasets = mags.map( | ||
| mag => | ||
| NgffDataset( | ||
| path = mag.toMagLiteral(allowScalar = true), | ||
| List(NgffCoordinateTransformation( | ||
| scale = Some(List[Double](1.0) ++ (dataSourceVoxelSize.factor * Vec3Double(mag)).toList))) | ||
| )) | ||
| val lengthUnitStr = dataSourceVoxelSize.unit.toString | ||
| val axes = List(NgffAxis(name = "c", `type` = "channel")) ++ additionalAxes | ||
| .getOrElse(List.empty) | ||
| .zipWithIndex | ||
| .map(axisAndIndex => NgffAxis(name = s"t${axisAndIndex._2}", `type` = "space", unit = Some(lengthUnitStr))) ++ List( | ||
| NgffAxis(name = "x", `type` = "space", unit = Some(lengthUnitStr)), | ||
| NgffAxis(name = "y", `type` = "space", unit = Some(lengthUnitStr)), | ||
| NgffAxis(name = "z", `type` = "space", unit = Some(lengthUnitStr)), | ||
| ) | ||
| NgffMetadataV0_5(version, | ||
| multiscales = | ||
| List(NgffMultiscalesItemV0_5(name = Some(dataLayerName), datasets = datasets, axes = axes)), | ||
| None) | ||
| } | ||
|
|
||
| implicit val jsonFormat: OFormat[NgffMetadataV0_5] = Json.format[NgffMetadataV0_5] | ||
| } |
54 changes: 54 additions & 0 deletions
54
...om/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| package com.scalableminds.webknossos.datastore.datareaders.zarr | ||
|
|
||
| import com.scalableminds.webknossos.datastore.models | ||
| import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize} | ||
| import net.liftweb.common.{Box, Failure, Full} | ||
| import play.api.libs.json.{Json, OFormat} | ||
|
|
||
| case class NgffCoordinateTransformation(`type`: String = "scale", scale: Option[List[Double]]) | ||
|
|
||
| object NgffCoordinateTransformation { | ||
| implicit val jsonFormat: OFormat[NgffCoordinateTransformation] = Json.format[NgffCoordinateTransformation] | ||
| } | ||
|
|
||
| case class NgffDataset(path: String, coordinateTransformations: List[NgffCoordinateTransformation]) | ||
|
|
||
| object NgffDataset { | ||
| implicit val jsonFormat: OFormat[NgffDataset] = Json.format[NgffDataset] | ||
| } | ||
|
|
||
| case class NgffAxis(name: String, `type`: String, unit: Option[String] = None) { | ||
|
|
||
| def lengthUnit: Box[models.LengthUnit.Value] = | ||
| if (`type` != "space") | ||
| Failure(f"Could not convert NGFF unit $name of type ${`type`} to LengthUnit") | ||
| else { | ||
| unit match { | ||
| case None | Some("") => Full(VoxelSize.DEFAULT_UNIT) | ||
| case Some(someUnit) => LengthUnit.fromString(someUnit) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| object NgffAxis { | ||
| implicit val jsonFormat: OFormat[NgffAxis] = Json.format[NgffAxis] | ||
| } | ||
|
|
||
| case class NgffOmeroMetadata(channels: List[NgffChannelAttributes]) | ||
| object NgffOmeroMetadata { | ||
| implicit val jsonFormat: OFormat[NgffOmeroMetadata] = Json.format[NgffOmeroMetadata] | ||
| } | ||
|
|
||
| case class NgffChannelWindow(min: Double, max: Double, start: Double, end: Double) | ||
| object NgffChannelWindow { | ||
| implicit val jsonFormat: OFormat[NgffChannelWindow] = Json.format[NgffChannelWindow] | ||
| } | ||
|
|
||
| case class NgffChannelAttributes(color: Option[String], | ||
| label: Option[String], | ||
| window: Option[NgffChannelWindow], | ||
| inverted: Option[Boolean], | ||
| active: Option[Boolean]) | ||
| object NgffChannelAttributes { | ||
| implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
...store/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package com.scalableminds.webknossos.datastore.datareaders.zarr3 | ||
|
|
||
| import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV0_5 | ||
| import play.api.libs.json._ | ||
|
|
||
| case class Zarr3GroupHeader( | ||
| zarr_format: Int, // must be 3 | ||
| node_type: String, // must be "group" | ||
| ngffMetadata: Option[NgffMetadataV0_5], | ||
| ) | ||
|
|
||
| object Zarr3GroupHeader { | ||
| implicit object Zarr3GroupHeaderFormat extends Format[Zarr3GroupHeader] { | ||
| override def reads(json: JsValue): JsResult[Zarr3GroupHeader] = | ||
| for { | ||
| zarr_format <- (json \ "zarr_format").validate[Int] | ||
| node_type <- (json \ "node_type").validate[String] | ||
| // Read the metadata from the correct json path. | ||
| ngffMetadata <- (json \ "attributes" \ "ome").validateOpt[NgffMetadataV0_5] | ||
| } yield | ||
| Zarr3GroupHeader( | ||
| zarr_format, | ||
| node_type, | ||
| ngffMetadata, | ||
| ) | ||
|
|
||
| override def writes(zarrArrayGroup: Zarr3GroupHeader): JsValue = | ||
| Json.obj( | ||
| "zarr_format" -> zarrArrayGroup.zarr_format, | ||
| "node_type" -> zarrArrayGroup.node_type, | ||
| // Enforce correct path for ngffMetadata in the json. | ||
| "attributes" -> Json.obj("ome" -> zarrArrayGroup.ngffMetadata), | ||
| ) | ||
| } | ||
| } |
Oops, something went wrong.