Use chunkContentsCache for sharded data

scalableminds · Mar 20, 2023 · 1b67ef9 · 1b67ef9
1 parent ac38770
commit 1b67ef9
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 5 deletions.
diff --git a/...ossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala b/...ossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala
@@ -26,7 +26,7 @@ class DatasetArray(relativePath: DatasetPath,
     ChunkReader.create(vaultPath, header)
 
   // cache currently limited to 1 GB per array
-  private lazy val chunkContentsCache: Cache[String, MultiArray] = {
+  private lazy val chunkContentsCache: Cache[Array[Int], MultiArray] = {
     val maxSizeBytes = 1000L * 1000 * 1000
     val maxEntries = maxSizeBytes / header.bytesPerChunk
     AlfuCache(maxEntries.toInt)
@@ -88,10 +88,12 @@ class DatasetArray(relativePath: DatasetPath,
       } yield targetBuffer
     }
   }
-
   protected def readShardedChunk(chunkIndex: Array[Int])(implicit ec: ExecutionContext): Future[Array[Byte]] = ???
 
   private def getSourceChunkDataWithCache(chunkIndex: Array[Int])(implicit ec: ExecutionContext): Future[MultiArray] =
+    chunkContentsCache.getOrLoad(chunkIndex, index => readSourceChunkData(index))
+
+  private def readSourceChunkData(chunkIndex: Array[Int])(implicit ec: ExecutionContext): Future[MultiArray] =
     if (header.isSharded) {
       for {
         chunkData: Array[Byte] <- readShardedChunk(chunkIndex)
@@ -104,7 +106,7 @@ class DatasetArray(relativePath: DatasetPath,
       val storeKey = chunkFilePath.storeKey
       val chunkShape = header.chunkSizeAtIndex(chunkIndex)
 
-      chunkContentsCache.getOrLoad(storeKey, key => chunkReader.read(key, chunkShape))
+      chunkReader.read(storeKey, chunkShape)
     }
 
   protected def getChunkFilename(chunkIndex: Array[Int]): String =

diff --git a/...app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedArray.scala b/...app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedArray.scala
@@ -85,8 +85,8 @@ class PrecomputedArray(relativePath: DatasetPath,
   private val minishardIndexCache: AlfuFoxCache[(VaultPath, Int), Seq[(Long, Long, Long)]] =
     AlfuFoxCache(maxEntries = 100)
 
-  private def getHashForChunk(index: Array[Int]): Long =
-    CompressedMortonCode.encode(index, header.gridSize)
+  private def getHashForChunk(chunkIndex: Array[Int]): Long =
+    CompressedMortonCode.encode(chunkIndex, header.gridSize)
 
   private lazy val minishardMask = {
     header.precomputedScale.sharding match {