From 7c570891df8fcfa99ef9694627956b19b0203acf Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 18 Mar 2024 17:30:56 +0100 Subject: [PATCH 1/5] Assert correct dtype for segment index file --- .../datastore/services/SegmentIndexFileService.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala index 6091593a581..97fac6ac4a2 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala @@ -106,8 +106,10 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig, _ <- Fox.successful(()) topLeftStart = buckets(bucketLocalOffset)(1) topLeftEnd = buckets(bucketLocalOffset)(2) + _ <- Fox + .bool2Fox(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries") == "uint16") ~> ("dtype_bucket_entries not supported") topLefts = segmentIndex.reader - .uint16() // Read datatype from attributes? + .uint16() .readMatrixBlockWithOffset("top_lefts", (topLeftEnd - topLeftStart).toInt, 3, topLeftStart, 0) } yield topLefts) } yield topLeftOpts From 8f87d734efb1840ace556bc055ba6984a8dd1143 Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 18 Mar 2024 17:35:49 +0100 Subject: [PATCH 2/5] Update Changelog --- CHANGELOG.unreleased.md | 3 ++- .../datastore/services/SegmentIndexFileService.scala | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index d9a328e829d..3432562dc8d 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -34,7 +34,8 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added the config key `webKnossos.user.timeTrackingOnlyWithSignificantChanges`, which when set to `true` will only track time if the user has made significant changes to the annotation. [#7627](https://github.com/scalableminds/webknossos/pull/7627) - Only display UI elements to launch background jobs if the (worker) backend actually supports them. [#7591](https://github.com/scalableminds/webknossos/pull/7591) - If the current dataset folder in the dashboard cannot be found (e.g., because somebody else deleted it), the page navigates to the root folder automatically. [#7669](https://github.com/scalableminds/webknossos/pull/7669) -- Voxelytics logs are now stored by organization name, rather than id, in Loki. This is inpreparation of the unificaction of these two concepts. [#7687](https://github.com/scalableminds/webknossos/pull/7687) +- Voxelytics logs are now stored by organization name, rather than id, in Loki. This is in preparation of the unification of these two concepts. [#7687](https://github.com/scalableminds/webknossos/pull/7687) +- Using a segment index file with a different data type than uint16 will now result in an error. [#7698](https://github.com/scalableminds/webknossos/pull/7698) ### Fixed - Fixed rare SIGBUS crashes of the datastore module that were caused by memory mapping on unstable file systems. [#7528](https://github.com/scalableminds/webknossos/pull/7528) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala index 97fac6ac4a2..272e2867647 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala @@ -107,7 +107,7 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig, topLeftStart = buckets(bucketLocalOffset)(1) topLeftEnd = buckets(bucketLocalOffset)(2) _ <- Fox - .bool2Fox(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries") == "uint16") ~> ("dtype_bucket_entries not supported") + .bool2Fox(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries") == "uint16") ~> "dtype_bucket_entries not supported" topLefts = segmentIndex.reader .uint16() .readMatrixBlockWithOffset("top_lefts", (topLeftEnd - topLeftStart).toInt, 3, topLeftStart, 0) From 5f10c529c6caf439e04d86698b8295876a1a9125 Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 18 Mar 2024 17:50:20 +0100 Subject: [PATCH 3/5] Fix typo preventing returning error --- .../webknossos/datastore/services/SegmentIndexFileService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala index 272e2867647..8c7c092f375 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala @@ -107,7 +107,7 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig, topLeftStart = buckets(bucketLocalOffset)(1) topLeftEnd = buckets(bucketLocalOffset)(2) _ <- Fox - .bool2Fox(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries") == "uint16") ~> "dtype_bucket_entries not supported" + .bool2Fox(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries") == "uint16") ?~> "dtype_bucket_entries not supported" topLefts = segmentIndex.reader .uint16() .readMatrixBlockWithOffset("top_lefts", (topLeftEnd - topLeftStart).toInt, 3, topLeftStart, 0) From 863e44cff945a17bf4893a8ed32a3e9bd8d1edf2 Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 18 Mar 2024 17:51:15 +0100 Subject: [PATCH 4/5] Make error more verbose --- .../webknossos/datastore/services/SegmentIndexFileService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala index 8c7c092f375..1714632746f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala @@ -107,7 +107,7 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig, topLeftStart = buckets(bucketLocalOffset)(1) topLeftEnd = buckets(bucketLocalOffset)(2) _ <- Fox - .bool2Fox(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries") == "uint16") ?~> "dtype_bucket_entries not supported" + .bool2Fox(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries") == "uint16") ?~> "value for dtype_bucket_entries in segment index file is not supported, only uint16 is supported" topLefts = segmentIndex.reader .uint16() .readMatrixBlockWithOffset("top_lefts", (topLeftEnd - topLeftStart).toInt, 3, topLeftStart, 0) From 7458250eb5a2f5c04bec0a8604a410bb780d6e36 Mon Sep 17 00:00:00 2001 From: frcroth Date: Wed, 20 Mar 2024 09:33:14 +0100 Subject: [PATCH 5/5] Add tryo before assertion --- .../datastore/services/SegmentIndexFileService.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala index 1714632746f..c85216d8e30 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/SegmentIndexFileService.scala @@ -20,6 +20,7 @@ import com.scalableminds.webknossos.datastore.models.{ datasource } import com.scalableminds.webknossos.datastore.storage.{AgglomerateFileKey, CachedHdf5File, Hdf5FileCache} +import net.liftweb.common.Box.tryo import net.liftweb.common.{Box, Full} import play.api.i18n.MessagesProvider @@ -106,8 +107,9 @@ class SegmentIndexFileService @Inject()(config: DataStoreConfig, _ <- Fox.successful(()) topLeftStart = buckets(bucketLocalOffset)(1) topLeftEnd = buckets(bucketLocalOffset)(2) + bucketEntriesDtype <- tryo(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries")).toFox _ <- Fox - .bool2Fox(segmentIndex.reader.string().getAttr("/", "dtype_bucket_entries") == "uint16") ?~> "value for dtype_bucket_entries in segment index file is not supported, only uint16 is supported" + .bool2Fox(bucketEntriesDtype == "uint16") ?~> "value for dtype_bucket_entries in segment index file is not supported, only uint16 is supported" topLefts = segmentIndex.reader .uint16() .readMatrixBlockWithOffset("top_lefts", (topLeftEnd - topLeftStart).toInt, 3, topLeftStart, 0)