Make mappings field in datasource optional (#4532)

* make mappings field in datasource optional * update validation schema * update snapshots
scalableminds · Apr 9, 2020 · b119728 · b119728
1 parent e1a0c7e
commit b119728
Show file tree

Hide file tree

Showing 13 changed files with 20 additions and 15 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 - Users can now input floating point numbers into the rotation field in flight and oblique mode. These values will get rounded internally. [#4507](https://github.com/scalableminds/webknossos/pull/4507)
 - Deleting an empty tree group in the `Trees` tab no longer prompts for user confirmation. [#4506](https://github.com/scalableminds/webknossos/pull/4506)
 - Toggling the "Render missing data black" option now automatically reloads all layers making it unnecessary to reload the whole page. [#4516](https://github.com/scalableminds/webknossos/pull/4516)
+- The "mappings" attribute of segmentation layers in datasource jsons can now be omitted. [#4532](https://github.com/scalableminds/webknossos/pull/4532)
 
  ### Fixed
 - Users only get tasks of datasets that they can access. [#4488](https://github.com/scalableminds/webknossos/pull/4488)

diff --git a/app/models/binary/DataSet.scala b/app/models/binary/DataSet.scala
@@ -369,6 +369,7 @@ class DataSetDataLayerDAO @Inject()(sqlClient: SQLClient, dataSetResolutionsDAO:
     } yield {
       (row.largestsegmentid, row.mappings) match {
         case (Some(segmentId), Some(mappings)) =>
+          val mappingsAsSet = parseArrayTuple(mappings).toSet
           Fox.successful(
             AbstractSegmentationLayer(
               row.name,
@@ -377,7 +378,7 @@ class DataSetDataLayerDAO @Inject()(sqlClient: SQLClient, dataSetResolutionsDAO:
               resolutions.sortBy(_.maxDim),
               elementClass,
               segmentId,
-              parseArrayTuple(mappings).toSet,
+              if (mappingsAsSet.isEmpty) None else Some(mappingsAsSet),
               defaultViewConfigurationOpt.map(SegmentationLayerViewConfiguration.from)
             ))
         case (None, None) =>
@@ -419,10 +420,11 @@ class DataSetDataLayerDAO @Inject()(sqlClient: SQLClient, dataSetResolutionsDAO:
   def insertLayerQuery(_dataSet: ObjectId, layer: DataLayer) =
     layer match {
       case s: AbstractSegmentationLayer => {
+        val mappings = s.mappings.getOrElse(Set())
         sqlu"""insert into webknossos.dataset_layers(_dataSet, name, category, elementClass, boundingBox, largestSegmentId, mappings, defaultViewConfiguration)
                     values(${_dataSet.id}, ${s.name}, '#${s.category.toString}', '#${s.elementClass.toString}',
                      '#${writeStructTuple(s.boundingBox.toSql.map(_.toString))}', ${s.largestSegmentId}, '#${writeArrayTuple(
-          s.mappings.map(sanitize(_)).toList)}', #${optionLiteral(
+          mappings.map(sanitize(_)).toList)}', #${optionLiteral(
           s.defaultViewConfiguration.map(d => Json.toJson(d).toString))})"""
       }
       case d: AbstractDataLayer => {

diff --git a/frontend/javascripts/libs/datasource.schema.json b/frontend/javascripts/libs/datasource.schema.json
@@ -109,7 +109,7 @@
                 "largestSegmentId": { "type": "number", "minimum": 1 },
                 "mappings": { "type": "array", "items": { "type": "string" } }
               },
-              "required": ["category", "largestSegmentId", "mappings"]
+              "required": ["category", "largestSegmentId"]
             }
           ]
         },

diff --git a/...est/snapshots/public/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md b/...est/snapshots/public/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md
@@ -206,7 +206,6 @@ Generated by [AVA](https://ava.li).
               category: 'segmentation',
               elementClass: 'uint16',
               largestSegmentId: 10000,
-              mappings: [],
               name: 'segmentation',
               resolutions: [],
             },
@@ -495,7 +494,6 @@ Generated by [AVA](https://ava.li).
               category: 'segmentation',
               elementClass: 'uint16',
               largestSegmentId: 10000,
-              mappings: [],
               name: 'segmentation',
               resolutions: [],
             },

diff --git a/...cripts/test/snapshots/public/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap b/...cripts/test/snapshots/public/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap
diff --git a/...os-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MappingProvider.scala b/...os-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MappingProvider.scala
@@ -29,13 +29,15 @@ object MappingProvider {
 
   val mappingFileExtension = "json"
 
-  def exploreMappings(layerDir: Path): Set[String] =
-    PathUtils
+  def exploreMappings(layerDir: Path): Option[Set[String]] = {
+    val mappingSet = PathUtils
       .listFiles(layerDir.resolve(MappingProvider.mappingsDir),
                  PathUtils.fileExtensionFilter(MappingProvider.mappingFileExtension))
       .map { paths =>
         paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
       }
       .getOrElse(Nil)
       .toSet
+    if (mappingSet.isEmpty) None else Some(mappingSet)
+  }
 }
diff --git a/...re/app/com/scalableminds/webknossos/datastore/dataformats/knossos/KnossosDataLayers.scala b/...re/app/com/scalableminds/webknossos/datastore/dataformats/knossos/KnossosDataLayers.scala
@@ -54,7 +54,7 @@ case class KnossosSegmentationLayer(
     name: String,
     sections: List[KnossosSection],
     elementClass: ElementClass.Value,
-    mappings: Set[String],
+    mappings: Option[Set[String]],
     largestSegmentId: Long,
     defaultViewConfiguration: Option[SegmentationLayerViewConfiguration] = None
 ) extends SegmentationLayer

diff --git a/...-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWDataLayers.scala b/...-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWDataLayers.scala
@@ -50,7 +50,7 @@ case class WKWSegmentationLayer(
     boundingBox: BoundingBox,
     wkwResolutions: List[WKWResolution],
     elementClass: ElementClass.Value,
-    mappings: Set[String],
+    mappings: Option[Set[String]],
     largestSegmentId: Long,
     defaultViewConfiguration: Option[SegmentationLayerViewConfiguration] = None
 ) extends SegmentationLayer

diff --git a/...os-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala b/...os-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
@@ -160,7 +160,7 @@ trait SegmentationLayerLike extends DataLayerLike {
 
   def largestSegmentId: Long
 
-  def mappings: Set[String]
+  def mappings: Option[Set[String]]
 
   def defaultViewConfiguration: Option[SegmentationLayerViewConfiguration]
 }
@@ -261,7 +261,7 @@ case class AbstractSegmentationLayer(
     resolutions: List[Point3D],
     elementClass: ElementClass.Value,
     largestSegmentId: Long,
-    mappings: Set[String],
+    mappings: Option[Set[String]],
     defaultViewConfiguration: Option[SegmentationLayerViewConfiguration] = None
 ) extends SegmentationLayerLike
 

diff --git a/...os-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceImporter.scala b/...os-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceImporter.scala
@@ -86,6 +86,6 @@ trait DataSourceImporter {
       case Right(point) => point.maxDim
     }
 
-  protected def exploreMappings(baseDir: Path): Set[String] = MappingProvider.exploreMappings(baseDir)
+  protected def exploreMappings(baseDir: Path): Option[Set[String]] = MappingProvider.exploreMappings(baseDir)
 
 }
diff --git a/...sos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala b/...sos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala
@@ -109,7 +109,9 @@ class DataSourceService @Inject()(
   }
 
   def exploreMappings(organizationName: String, dataSetName: String, dataLayerName: String): Set[String] =
-    MappingProvider.exploreMappings(dataBaseDir.resolve(organizationName).resolve(dataSetName).resolve(dataLayerName))
+    MappingProvider
+      .exploreMappings(dataBaseDir.resolve(organizationName).resolve(dataSetName).resolve(dataLayerName))
+      .getOrElse(Set())
 
   private def validateDataSource(dataSource: DataSource): Box[Unit] = {
     def Check(expression: Boolean, msg: String): Option[String] = if (!expression) Some(msg) else None

diff --git a/.../app/com/scalableminds/webknossos/tracingstore/tracings/volume/FallbackLayerAdapter.scala b/.../app/com/scalableminds/webknossos/tracingstore/tracings/volume/FallbackLayerAdapter.scala
@@ -44,7 +44,7 @@ class FallbackLayerAdapter(primary: SegmentationLayer, fallback: SegmentationLay
 
   val largestSegmentId: Long = math.max(primary.largestSegmentId, fallback.largestSegmentId)
 
-  val mappings: Set[String] = primary.mappings
+  val mappings: Option[Set[String]] = primary.mappings
 
   lazy val bucketProvider: BucketProvider = new FallbackBucketProvider(primary, fallback)
 

diff --git a/...re/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala b/...re/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala
@@ -48,7 +48,7 @@ case class VolumeTracingLayer(
 
   val bucketProvider: BucketProvider = volumeBucketProvider
 
-  val mappings: Set[String] = Set.empty
+  val mappings: Option[Set[String]] = None
 
   val resolutions: List[Point3D] = List(Point3D(1, 1, 1))
 }