Add route to load full mesh stl of a segment/agglomerate

CHANGELOG.unreleased.md

@@ -17,6 +17,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Webknossos can now open ND Zarr datasets with arbitrary axis orders (not limited to `**xyz` anymore). [#7592](https://github.com/scalableminds/webknossos/pull/7592)
 - Added a new "Split from all neighboring segments" feature for the proofreading mode. [#7611](https://github.com/scalableminds/webknossos/pull/7611)
 - If storage scan is enabled, the measured used storage is now displayed in the dashboard’s dataset detail view. [#7677](https://github.com/scalableminds/webknossos/pull/7677)
+- Prepared support to download full stl meshes via the HTTP api. [#7587](https://github.com/scalableminds/webknossos/pull/7587)
 
 ### Changed
 - Datasets stored in WKW format are no longer loaded with memory mapping, reducing memory demands. [#7528](https://github.com/scalableminds/webknossos/pull/7528)

DEV_INSTALL.md

@@ -26,6 +26,7 @@ For non-localhost deployments, check out the [installation guide in the document
 * [Redis 5+](https://redis.io/)
 * [Blosc](https://github.com/Blosc/c-blosc)
 * [Brotli](https://github.com/google/brotli)
+* [Draco](https://github.com/google/draco)
 * [node.js 18](http://nodejs.org/download/)
 * [yarn package manager](https://yarnpkg.com/)
 * [git](http://git-scm.com/downloads)
@@ -41,13 +42,14 @@ arch -x86_64 /bin/zsh
 # Install Homebrew package manager
 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
 
-# Install git, node.js, postgres, sbt, gfind, gsed
-brew install openjdk@21 openssl git node postgresql sbt findutils coreutils gnu-sed redis yarn c-blosc brotli wget
+# Install git, node.js, postgres, sbt, gfind, gsed, draco
+brew install openjdk draco openssl git node postgresql sbt findutils coreutils gnu-sed redis yarn c-blosc brotli wget
 
 # Set env variables for openjdk and openssl
 # You probably want to add these lines manually to avoid conflicts in your zshrc
 echo 'if [ $(arch) = "i386" ]; then' >> ~/.zshrc
-echo '  export PATH="/usr/local/opt/openjdk@14/bin:$PATH"' >> ~/.zshrc
+echo '  export JAVA_HOME=/opt/homebrew/opt/openjdk/libexec/openjdk.jdk/Contents/Home' >> ~/.zshrc
+echo '  export PATH="/usr/local/opt/openjdk/bin:$PATH"' >> ~/.zshrc
 echo '  export PATH="/usr/local/opt/openssl/bin:$PATH"' >> ~/.zshrc
 echo '  export LDFLAGS="-L/usr/local/opt/openssl/lib"' >> ~/.zshrc
 echo '  export CPPFLAGS="-I/usr/local/opt/openssl/include"' >> ~/.zshrc
@@ -80,7 +82,7 @@ curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
 echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
 
 sudo apt update
-sudo apt install -y nodejs git postgresql postgresql-client unzip zip yarn redis-server build-essential libblosc1 libbrotli1
+sudo apt install -y nodejs git postgresql postgresql-client unzip zip yarn redis-server build-essential libblosc1 libbrotli1 libdraco-dev
 
  # Install sdkman, java, scala and sbt
 curl -s "https://get.sdkman.io" | bash

Dockerfile

@@ -2,7 +2,7 @@ FROM eclipse-temurin:21
 ARG VERSION_NODE="18.x"
 
 RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \
-  && apt-get -y install libblosc1 libbrotli1 postgresql-client git nodejs \
+  && apt-get -y install libblosc1 libbrotli1 postgresql-client libdraco4 git nodejs \
   && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /webknossos

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM scalableminds/sbt:master__7785473656
+FROM scalableminds/sbt:master__7830403826
 ARG VERSION_NODE="18.x"
 
 ENV DEBIAN_FRONTEND noninteractive
@@ -24,6 +24,7 @@ RUN apt-get update \
   postgresql-client \
   yarn \
   git \
+  libdraco-dev \
   libblosc1 \
   # The following packages are necessary to run headless-gl
   && apt-get install -y \

MIGRATIONS.unreleased.md

@@ -17,6 +17,7 @@ UPDATE webknossos.annotations_ SET state = 'Finished' WHERE _id IN  (SELECT DIST
 - WEBKNOSSOS now uses Java 21 (up from Java 11). [#7599](https://github.com/scalableminds/webknossos/pull/7599)
 - NodeJS version 18+ is required for snapshot tests with ShadowDOM elements from Antd v5. [#7522](https://github.com/scalableminds/webknossos/pull/7522)
 - Email verification is disabled by default. To enable it, set `webKnossos.user.emailVerification.activated` to `true` in your `application.conf`. [#7620](https://github.com/scalableminds/webknossos/pull/7620) [#7621](https://github.com/scalableminds/webknossos/pull/7621)
+- New dependency draco/libdraco-dev needs to be installed when deploying without docker and for local development.
 
 ### Postgres Evolutions:
 

app/controllers/UserTokenController.scala

@@ -30,7 +30,7 @@ import scala.concurrent.ExecutionContext
 object RpcTokenHolder {
   /*
    * This token is used to tell the datastore or tracing store “I am WEBKNOSSOS”.
-   * The respective module asks the remote webKnossos to validate that.
+   * The respective module asks the remote WEBKNOSSOS to validate that.
    * The token is refreshed on every wK restart.
    * Keep it secret!
    */
@@ -81,7 +81,7 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
     }
 
   /* token may be
-       - the global webKnossosToken (allow everything)
+       - the global webknossosToken (allow everything)
        - a user token (allow what that user may do)
        - a dataset sharing token (allow seeing dataset / annotations that token belongs to)
    */

app/models/job/Job.scala

@@ -80,16 +80,16 @@ case class Job(
       }
     }
 
-  def resultLinkPublic(organizationName: String, webKnossosPublicUrl: String): Option[String] =
+  def resultLinkPublic(organizationName: String, webknossosPublicUrl: String): Option[String] =
     for {
       resultLink <- resultLink(organizationName)
-      resultLinkPublic = if (resultLink.startsWith("/")) s"$webKnossosPublicUrl$resultLink"
+      resultLinkPublic = if (resultLink.startsWith("/")) s"$webknossosPublicUrl$resultLink"
       else s"$resultLink"
     } yield resultLinkPublic
 
-  def resultLinkSlackFormatted(organizationName: String, webKnossosPublicUrl: String): Option[String] =
+  def resultLinkSlackFormatted(organizationName: String, webknossosPublicUrl: String): Option[String] =
     for {
-      resultLink <- resultLinkPublic(organizationName, webKnossosPublicUrl)
+      resultLink <- resultLinkPublic(organizationName, webknossosPublicUrl)
       resultLinkFormatted = s" <$resultLink|Result>"
     } yield resultLinkFormatted
 }

build.sbt

@@ -57,11 +57,17 @@ lazy val util = (project in file("util")).settings(
   dependencyOverrides ++= Dependencies.dependencyOverrides
 )
 
+lazy val webknossosJni = (project in file("webknossos-jni"))
+  .settings(nativeCompile / sourceDirectory := sourceDirectory.value)
+  .enablePlugins(JniNative)
+
 lazy val webknossosDatastore = (project in file("webknossos-datastore"))
   .dependsOn(util)
+  .dependsOn(webknossosJni)
   .enablePlugins(play.sbt.PlayScala)
   .enablePlugins(BuildInfoPlugin)
   .enablePlugins(ProtocPlugin)
+  .settings(javah / target := (webknossosJni / nativeCompile / sourceDirectory).value / "include")
   .settings(
     name := "webknossos-datastore",
     commonSettings,

clean

@@ -11,6 +11,7 @@ rm -rf webknossos-datastore/project/project/target
 rm -rf webknossos-tracingstore/target
 rm -rf webknossos-tracingstore/project/target
 rm -rf webknossos-tracingstore/project/project/target
+rm -rf webknossos-jni/target
 rm -rf node_modules
 rm -rf .eslintcache
 mkdir target

frontend/javascripts/admin/admin_rest_api.ts

@@ -2191,7 +2191,6 @@ type MeshRequest = {
   additionalCoordinates: AdditionalCoordinate[] | undefined;
   mag: Vector3;
   segmentId: number; // Segment to build mesh for
-  subsamplingStrides: Vector3;
   // The cubeSize is in voxels in mag <mag>
   cubeSize: Vector3;
   scale: Vector3;
@@ -2212,7 +2211,6 @@ export function computeAdHocMesh(
     additionalCoordinates,
     cubeSize,
     mappingName,
-    subsamplingStrides,
 
     ...rest
   } = meshRequest;
@@ -2228,13 +2226,11 @@ export function computeAdHocMesh(
           // The back-end needs a small padding at the border of the
           // bounding box to calculate the mesh. This padding
           // is added here to the position and bbox size.
-          position: V3.toArray(V3.sub(position, subsamplingStrides)),
+          position: V3.toArray(V3.sub(position, [1, 1, 1])),
           additionalCoordinates,
-          cubeSize: V3.toArray(V3.add(cubeSize, subsamplingStrides)),
+          cubeSize: V3.toArray(V3.add(cubeSize, [1, 1, 1])),
           // Name and type of mapping to apply before building mesh (optional)
           mapping: mappingName,
-          // "size" of each voxel (i.e., only every nth voxel is considered in each dimension)
-          subsamplingStrides,
           ...rest,
         },
       },

frontend/javascripts/oxalis/model/sagas/mesh_saga.ts

@@ -451,11 +451,6 @@ function* maybeLoadMeshChunk(
 
   batchCounterPerSegment[segmentId]++;
   threeDMap.set(clippedPosition, true);
-  // In general, it is more performant to compute meshes in a more coarse resolution instead of using subsampling strides
-  // since in the coarse resolution less data needs to be loaded. Another possibility to increase performance is
-  // window.__marchingCubeSizeInMag1 which affects the cube size the marching cube algorithm will work on. If the cube is significantly larger than the
-  // segments, computations are wasted.
-  const subsamplingStrides = (window as any).__meshSubsamplingStrides || [1, 1, 1];
   const scale = yield* select((state) => state.dataset.dataSource.scale);
   const dataStoreHost = yield* select((state) => state.dataset.dataStore.url);
   const owningOrganization = yield* select((state) => state.dataset.owningOrganization);
@@ -493,7 +488,6 @@ function* maybeLoadMeshChunk(
           additionalCoordinates: additionalCoordinates || undefined,
           mag,
           segmentId,
-          subsamplingStrides,
           cubeSize,
           scale,
           findNeighbors,

project/BuildInfoSettings.scala

@@ -5,56 +5,63 @@ import scala.language.postfixOps
 
 object BuildInfoSettings {
 
-  def getStdoutFromCommand(command: String, failureMsg: String): String = {
+  def getStdoutFromCommand(command: String, failureMsg: String): String =
     try {
       (command !!).trim
     } catch {
       case _: Throwable => failureMsg
     }
-  }
 
-  val ciBuild: String = if (System.getenv().containsKey("CIRCLE_BUILD_NUM")) System.getenv().get("CIRCLE_BUILD_NUM") else ""
+  val ciBuild: String =
+    if (System.getenv().containsKey("CIRCLE_BUILD_NUM")) System.getenv().get("CIRCLE_BUILD_NUM") else ""
   val ciTag: String = if (System.getenv().containsKey("CIRCLE_TAG")) System.getenv().get("CIRCLE_TAG") else ""
 
   def commitHash: String = getStdoutFromCommand("git rev-parse HEAD", "<getting commit hash failed>")
   def commitDate: String = getStdoutFromCommand("git log -1 --format=%cd ", "<getting git date failed>")
 
-  def webKnossosVersion: String = if (ciTag != "") ciTag else (if (ciBuild != "") ciBuild else "dev")
+  def webknossosVersion: String = if (ciTag != "") ciTag else (if (ciBuild != "") ciBuild else "dev")
 
   lazy val webknossosBuildInfoSettings = Seq(
-    buildInfoKeys := Seq[BuildInfoKey](name, scalaVersion, sbtVersion,
+    buildInfoKeys := Seq[BuildInfoKey](
+      name,
+      scalaVersion,
+      sbtVersion,
       "commitHash" -> commitHash,
       "commitDate" -> commitDate,
       "ciBuild" -> ciBuild,
       "ciTag" -> ciTag,
-      "version" -> webKnossosVersion,
+      "version" -> webknossosVersion,
       "datastoreApiVersion" -> "2.0"
     ),
     buildInfoPackage := "webknossos",
     buildInfoOptions := Seq(BuildInfoOption.ToJson)
   )
 
   lazy val webknossosDatastoreBuildInfoSettings = Seq(
-    buildInfoKeys := Seq[BuildInfoKey](name, scalaVersion, sbtVersion,
+    buildInfoKeys := Seq[BuildInfoKey](
+      name,
+      scalaVersion,
+      sbtVersion,
       "commitHash" -> commitHash,
       "commitDate" -> commitDate,
       "ciBuild" -> ciBuild,
       "ciTag" -> ciTag,
-      "version" -> webKnossosVersion,
+      "version" -> webknossosVersion,
       "datastoreApiVersion" -> "2.0"
     ),
     buildInfoPackage := "webknossosDatastore",
     buildInfoOptions := Seq(BuildInfoOption.ToJson)
   )
 
   lazy val webknossosTracingstoreBuildInfoSettings = Seq(
-    buildInfoKeys := Seq[BuildInfoKey](name, scalaVersion, sbtVersion,
-      "commitHash" -> commitHash,
-      "commitDate" -> commitDate,
-      "ciBuild" -> ciBuild,
-      "ciTag" -> ciTag,
-      "version" -> webKnossosVersion
-    ),
+    buildInfoKeys := Seq[BuildInfoKey](name,
+                                       scalaVersion,
+                                       sbtVersion,
+                                       "commitHash" -> commitHash,
+                                       "commitDate" -> commitDate,
+                                       "ciBuild" -> ciBuild,
+                                       "ciTag" -> ciTag,
+                                       "version" -> webknossosVersion),
     buildInfoPackage := "webknossosTracingstore",
     buildInfoOptions := Seq(BuildInfoOption.ToJson)
   )

project/plugins.sbt

@@ -16,5 +16,8 @@ addSbtPlugin("com.sksamuel.scapegoat" %% "sbt-scapegoat" % "1.2.1")
 // check dependencies against published vulnerabilities with sbt dependencyCheck
 addSbtPlugin("net.vonbuchholtz" % "sbt-dependency-check" % "5.1.0")
 
-//protocol buffers
+// protocol buffers
 libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.13"
+
+// java native interface
+addSbtPlugin("com.github.sbt" % "sbt-jni" % "1.7.0")

util/src/main/scala/com/scalableminds/util/geometry/Vec3Float.scala

@@ -7,11 +7,21 @@ case class Vec3Float(x: Float, y: Float, z: Float) {
   def scale(s: Float): Vec3Float = Vec3Float(x * s, y * s, z * s)
 
   def *(s: Float): Vec3Float = scale(s)
+
   def *(s: Double): Vec3Float = scale(s.toFloat)
+
   def *(that: Vec3Float): Vec3Float = Vec3Float(x * that.x, y * that.y, z * that.z)
+
   def +(that: Vec3Float): Vec3Float = Vec3Float(x + that.x, y + that.y, z + that.z)
 
+  def -(that: Vec3Float): Vec3Float = Vec3Float(x - that.x, y - that.y, z - that.z)
+
   def toList: List[Float] = List(x, y, z)
+
+  def normalize: Vec3Float = {
+    val length = Math.sqrt(x * x + y * y + z * z)
+    scale(1 / length.toFloat)
+  }
 }
 
 object Vec3Float {
@@ -34,4 +44,7 @@ object Vec3Float {
       JsArray(l.map(toJson(_)))
     }
   }
+
+  def crossProduct(a: Vec3Float, b: Vec3Float): Vec3Float =
+    Vec3Float(a.y * b.z - a.z * b.y, a.z * b.x - a.x * b.z, a.x * b.y - a.y * b.x)
 }

util/src/main/scala/com/scalableminds/util/geometry/Vec3Int.scala

@@ -17,6 +17,9 @@ case class Vec3Int(x: Int, y: Int, z: Int) {
   def *(that: Int): Vec3Int =
     Vec3Int(x * that, y * that, z * that)
 
+  def -(that: Vec3Int): Vec3Int =
+    Vec3Int(x - that.x, y - that.y, z - that.z)
+
   def /(that: Vec3Int): Vec3Int =
     Vec3Int(x / that.x, y / that.y, z / that.z)
 

util/src/main/scala/com/scalableminds/util/requestlogging/RequestLogging.scala

@@ -49,7 +49,7 @@ trait AbstractRequestLogging extends LazyLogging {
 }
 
 trait RequestLogging extends AbstractRequestLogging {
-  // Hint: within webKnossos itself, UserAwareRequestLogging is available, which additionally logs the requester user id
+  // Hint: within webknossos itself, UserAwareRequestLogging is available, which additionally logs the requester user id
 
   def log(notifier: Option[String => Unit] = None)(block: => Future[Result])(implicit request: Request[_],
                                                                              ec: ExecutionContext): Future[Result] =

util/src/main/scala/com/scalableminds/util/time/Instant.scala

@@ -1,6 +1,7 @@
 package com.scalableminds.util.time
 
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.Box.tryo
 import play.api.libs.json._
 
@@ -44,7 +45,7 @@ case class Instant(epochMillis: Long) extends Ordered[Instant] {
   def weekyear: Int = toJodaDateTime.getWeekyear
 }
 
-object Instant extends FoxImplicits {
+object Instant extends FoxImplicits with LazyLogging {
   def now: Instant = Instant(System.currentTimeMillis())
 
   def max: Instant = Instant(253370761200000L)
@@ -74,6 +75,8 @@ object Instant extends FoxImplicits {
 
   def since(before: Instant): FiniteDuration = now - before
 
+  def logSince(before: Instant, label: String): Unit = logger.info(f"$label took ${Instant.since(before)}")
+
   private def fromStringSync(instantLiteral: String): Option[Instant] =
     fromIsoString(instantLiteral).orElse(fromEpochMillisString(instantLiteral))
 

webknossos-datastore/Dockerfile

@@ -1,6 +1,6 @@
 FROM eclipse-temurin:21
 RUN apt-get update \
-  && apt-get -y install libblosc1 libbrotli1 \
+  && apt-get -y install libblosc1 libbrotli1 libdraco4 \
   && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /webknossos-datastore \

webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala

@@ -19,11 +19,11 @@ class DataStoreModule extends AbstractModule {
     bind(classOf[UploadService]).asEagerSingleton()
     bind(classOf[DataSourceService]).asEagerSingleton()
     bind(classOf[DataVaultService]).asEagerSingleton()
-    bind(classOf[DSRemoteWebKnossosClient]).asEagerSingleton()
+    bind(classOf[DSRemoteWebknossosClient]).asEagerSingleton()
     bind(classOf[BinaryDataServiceHolder]).asEagerSingleton()
     bind(classOf[MappingService]).asEagerSingleton()
     bind(classOf[AgglomerateService]).asEagerSingleton()
-    bind(classOf[AdHocMeshingServiceHolder]).asEagerSingleton()
+    bind(classOf[AdHocMeshServiceHolder]).asEagerSingleton()
     bind(classOf[ApplicationHealthService]).asEagerSingleton()
     bind(classOf[DatasetErrorLoggingService]).asEagerSingleton()
   }