Use LZ4 with WASM for volume saving/undo/redo (#6652)

* fix conversion of NaN to big int

* use lz4-wasm for compression/decompression of buckets (for undo/redo)

* also use lz4-wasm for compression of buckets (for saving)

* clean up lz-wasm usage

* fix tests by using different lz4 library there via mocking

* more mocks

* update changelog

* more mocks

* add missing file

* fix accidental mock of non-spec files

* fix accidental mock of non-spec files

* also mock in e2e tests

* add wasm mime type

* extract slick conf

* extract lz4 mock in e2e specs away using e2e-setup

* re-add slow lz4js library to conduct performance comparison on dev instance

* Revert "re-add slow lz4js library to conduct performance comparison on dev instance"

This reverts commit c86c647.

* tune down changelog entry

* bump pako version (used for gzip compression when sending buckets to server)

Co-authored-by: Florian M <[email protected]>

CHANGELOG.unreleased.md

@@ -28,6 +28,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Re-phrased some backend (error) messages to improve clarity and provide helping hints. [#6616](https://github.com/scalableminds/webknossos/pull/6616)
 - The layer visibility is now encoded in the sharing link. The user opening the link will see the same layers that were visible when copying the link. [#6634](https://github.com/scalableminds/webknossos/pull/6634)
 - Voxelytics workflows can now be viewed by anyone with the link who is in the right organization. [#6622](https://github.com/scalableminds/webknossos/pull/6622)
+- Improve performance for handling of volume annotation data (saving/undo/redo). [#6652](https://github.com/scalableminds/webknossos/pull/6652)
 - When importing an annotation into an existing annotation, webKnossos ensures that bounding boxes are not duplicated in case they exist in the current *and* imported annotation. [#6648](https://github.com/scalableminds/webknossos/pull/6648)
 - Reworked the proofreading mode so that agglomerate skeletons are no longer needed (nor automatically loaded). Instead, segments can be selected by left-clicking onto them, indicated by a small white cross. To merge or split agglomerates, then either use the shortcuts `Shift + Leftclick`/`Ctrl + Leftclick` or use the context menu. [#6625](https://github.com/scalableminds/webknossos/pull/6625)
 

conf/application.conf

@@ -18,6 +18,9 @@ play {
       maxDiskBuffer = 1G
     }
     filters = "com.scalableminds.webknossos.datastore.Filters"
+    fileMimeTypes = ${play.http.fileMimeTypes} """
+      wasm=application/wasm
+    """
   }
   filters.headers {
     # Unset some of the security filters enabled in datastore.Filters
@@ -175,20 +178,8 @@ mail {
   }
 }
 
-# SQL database connection
-slick = {
-  profile = "slick.jdbc.PostgresProfile$"
-  codegen.package = "com.scalableminds.webknossos.schema"
-  db = {
-    url = "jdbc:postgresql://localhost/webknossos"
-    url = ${?POSTGRES_URL}
-    driver = org.postgresql.Driver
-    keepAliveConnection = true
-    user = "postgres"
-    password = "postgres"
-    queueSize = 5000
-  }
-}
+# SQL Slick Database Connection Config in Subfile to provide it also to the AssetGeneration task
+include "slick.conf"
 
 # Authentication via cookies and tokens
 silhouette {

conf/slick.conf

@@ -0,0 +1,14 @@
+# SQL database connection
+slick = {
+  profile = "slick.jdbc.PostgresProfile$"
+  codegen.package = "com.scalableminds.webknossos.schema"
+  db = {
+    url = "jdbc:postgresql://localhost/webknossos"
+    url = ${?POSTGRES_URL}
+    driver = org.postgresql.Driver
+    keepAliveConnection = true
+    user = "postgres"
+    password = "postgres"
+    queueSize = 5000
+  }
+}

frontend/javascripts/libs/utils.ts

@@ -886,7 +886,7 @@ export function castForArrayType(uncastNumber: number, data: TypedArray): number
 }
 
 export function convertNumberTo64Bit(num: number | null): [Vector4, Vector4] {
-  if (num == null) {
+  if (num == null || isNaN(num)) {
     return [
       [0, 0, 0, 0],
       [0, 0, 0, 0],

frontend/javascripts/oxalis/workers/byte_array_lz4_compression.worker.ts

@@ -1,6 +1,5 @@
-// @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'lz4j... Remove this comment to see the full error message
-import lz4 from "lz4js";
 import { expose } from "./comlink_wrapper";
+import * as lz4 from "lz4-wasm";
 
 function compressLz4Block(data: Uint8Array, compress: boolean): Uint8Array {
   if (compress) {

frontend/javascripts/oxalis/workers/byte_array_to_lz4_base64.worker.ts

@@ -1,16 +1,13 @@
 import Base64 from "base64-js";
-// @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'lz4j... Remove this comment to see the full error message
-import lz4 from "lz4js";
+import * as lz4wasm from "lz4-wasm";
 import { expose } from "./comlink_wrapper";
 
 function compressLz4Block(data: Uint8Array): Uint8Array {
-  // Backend expects the frame-less version of lz4,
-  // so we need to call lz4.compressBlock rather than compress
-  const hashSize = 1 << 16;
-  const hashTable = new Uint32Array(hashSize);
-  const compressedBuffer = new Uint8Array(data.length);
-  const compressedSize = lz4.compressBlock(data, compressedBuffer, 0, data.length, hashTable);
-  return compressedBuffer.slice(0, compressedSize);
+  // The backend expects the block (frame-less) version of lz4.
+  // lz4-wasm uses the block version, but prepends the size of the
+  // compressed data. Therefore, we strip the first 4 bytes.
+  const newCompressed = lz4wasm.compress(data);
+  return newCompressed.slice(4);
 }
 
 export function byteArrayToLz4Base64(byteArray: Uint8Array): string {

frontend/javascripts/oxalis/workers/compress.worker.ts

@@ -1,8 +1,7 @@
-// @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'pako... Remove this comment to see the full error message
 import pako from "pako";
 import { expose } from "./comlink_wrapper";
 
-function compress(data: ArrayBuffer | string): Promise<ArrayBuffer> {
+function compress(data: ArrayBuffer | string): ArrayBuffer {
   return pako.gzip(data);
 }
 

frontend/javascripts/test/api/api_skeleton_latest.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { __setupOxalis, KeyboardJS } from "test/helpers/apiHelpers";
 import { makeBasicGroupObject } from "oxalis/view/right-border-tabs/tree_hierarchy_view_helpers";
 import { setMappingEnabledAction } from "oxalis/model/actions/settings_actions";

frontend/javascripts/test/api/api_v2.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { __setupOxalis, KeyboardJS } from "test/helpers/apiHelpers";
 import { setMappingEnabledAction } from "oxalis/model/actions/settings_actions";
 import Store from "oxalis/store";

frontend/javascripts/test/api/api_volume_latest.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { AnnotationToolEnum } from "oxalis/constants";
 import { __setupOxalis } from "test/helpers/apiHelpers";
 import test from "ava";

frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts

@@ -1,3 +1,10 @@
+import {
+  resetDatabase,
+  replaceVolatileValues,
+  setCurrToken,
+  tokenUserA,
+  writeTypeCheckingFile,
+} from "test/enzyme/e2e-setup";
 import type { APIAnnotation } from "types/api_flow_types";
 import { APIAnnotationTypeEnum } from "types/api_flow_types";
 import { createTreeMapFromTreeArray } from "oxalis/model/reducers/skeletontracing_reducer_helpers";
@@ -7,13 +14,6 @@ import {
   getSkeletonDescriptor,
 } from "oxalis/model/accessors/skeletontracing_accessor";
 import { getServerVolumeTracings } from "oxalis/model/accessors/volumetracing_accessor";
-import {
-  resetDatabase,
-  replaceVolatileValues,
-  setCurrToken,
-  tokenUserA,
-  writeTypeCheckingFile,
-} from "test/enzyme/e2e-setup";
 import { sendRequestWithToken, addVersionNumbers } from "oxalis/model/sagas/save_saga";
 import * as UpdateActions from "oxalis/model/sagas/update_actions";
 import * as api from "admin/admin_rest_api";

frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts

@@ -1,11 +1,11 @@
 import _ from "lodash";
-import type { APIDataset } from "types/api_flow_types";
 import {
   tokenUserA,
   setCurrToken,
   resetDatabase,
   writeTypeCheckingFile,
 } from "test/enzyme/e2e-setup";
+import type { APIDataset } from "types/api_flow_types";
 import * as api from "admin/admin_rest_api";
 import test from "ava";
 

frontend/javascripts/test/backend-snapshot-tests/projects.e2e.ts

@@ -1,5 +1,4 @@
 import _ from "lodash";
-import type { APIProject, APIProjectUpdater } from "types/api_flow_types";
 import {
   tokenUserA,
   tokenUserD,
@@ -8,6 +7,7 @@ import {
   resetDatabase,
   writeTypeCheckingFile,
 } from "test/enzyme/e2e-setup";
+import type { APIProject, APIProjectUpdater } from "types/api_flow_types";
 import * as api from "admin/admin_rest_api";
 import test from "ava";
 test.before("Reset database", async () => {

frontend/javascripts/test/controller/url_manager.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import test from "ava";
 import UrlManager, { updateTypeAndId, encodeUrlHash } from "oxalis/controller/url_manager";
 import { location } from "libs/window";

frontend/javascripts/test/enzyme/e2e-setup.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 // @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'deep... Remove this comment to see the full error message
 import deepForEach from "deep-for-each";

frontend/javascripts/test/geometries/skeleton.spec.ts

@@ -1,4 +1,5 @@
 // Integration tests for skeleton.js
+import "test/mocks/lz4";
 import _ from "lodash";
 import { getSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor";
 import * as Utils from "libs/utils";
@@ -7,7 +8,7 @@ import test from "ava";
 import { Vector3 } from "oxalis/constants";
 import { OxalisState } from "oxalis/store";
 import { tracing, annotation } from "../fixtures/skeletontracing_server_objects";
-mockRequire.stopAll();
+
 mockRequire("app", {
   currentUser: {
     firstName: "SCM",

frontend/javascripts/test/libs/nml.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import _ from "lodash";
 import update from "immutability-helper";
 import sinon from "sinon";

frontend/javascripts/test/mocks/lz4.ts

@@ -0,0 +1,4 @@
+import mockRequire from "mock-require";
+import * as lz4 from "lz4-wasm-nodejs";
+
+mockRequire("lz4-wasm", lz4);

frontend/javascripts/test/model/binary/cube.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import _ from "lodash";
 import { ResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { tracing as skeletontracingServerObject } from "test/fixtures/skeletontracing_server_objects";
@@ -9,7 +10,7 @@ import datasetServerObject from "test/fixtures/dataset_server_object";
 import mockRequire from "mock-require";
 import runAsync from "test/helpers/run-async";
 import sinon from "sinon";
-mockRequire.stopAll();
+
 const StoreMock = {
   getState: () => ({
     dataset: datasetServerObject,

frontend/javascripts/test/model/binary/layers/wkstore_adapter.mock.ts

@@ -1,4 +1,6 @@
+import "test/mocks/lz4";
 import mockRequire from "mock-require";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

frontend/javascripts/test/model/binary/temporal_bucket_manager.spec.ts

@@ -3,6 +3,8 @@ import runAsync from "test/helpers/run-async";
 import sinon from "sinon";
 import type { TestInterface } from "ava";
 import anyTest from "ava";
+import "test/mocks/lz4";
+
 mockRequire("oxalis/model/sagas/root_saga", function* () {
   yield;
 });

frontend/javascripts/test/model/model.spec.ts

@@ -1,5 +1,6 @@
 // @ts-nocheck
 import _ from "lodash";
+import "test/mocks/lz4";
 import mockRequire from "mock-require";
 import sinon from "sinon";
 import test from "ava";
@@ -9,7 +10,6 @@ import {
   annotation as ANNOTATION,
 } from "../fixtures/skeletontracing_server_objects";
 import DATASET from "../fixtures/dataset_server_object";
-mockRequire.stopAll();
 
 function makeModelMock() {
   class ModelMock {}

frontend/javascripts/test/model/model_resolutions.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import test from "ava";
 import mockRequire from "mock-require";
 import {

frontend/javascripts/test/model/texture_bucket_manager.spec.ts

@@ -1,4 +1,5 @@
 import mock from "mock-require";
+import "test/mocks/lz4";
 import test, { ExecutionContext } from "ava";
 import { Vector4 } from "oxalis/constants";
 

frontend/javascripts/test/model/volumetracing/volume_annotation_sampling.spec.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { ResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { tracing as skeletontracingServerObject } from "test/fixtures/skeletontracing_server_objects";
 import { tracing as volumetracingServerObject } from "test/fixtures/volumetracing_server_objects";
@@ -8,7 +9,7 @@ import anyTest from "ava";
 import datasetServerObject from "test/fixtures/dataset_server_object";
 import mockRequire from "mock-require";
 import sinon from "sinon";
-mockRequire.stopAll();
+
 const StoreMock = {
   getState: () => ({
     dataset: datasetServerObject,

frontend/javascripts/test/reducers/skeletontracing_reducer.spec.ts

@@ -1,5 +1,6 @@
 // @ts-nocheck
 /* eslint-disable no-useless-computed-key */
+import "test/mocks/lz4";
 import _ from "lodash";
 import update from "immutability-helper";
 import { rgbs as colors } from "libs/color_generator";
@@ -8,7 +9,7 @@ import DiffableMap from "libs/diffable_map";
 import EdgeCollection from "oxalis/model/edge_collection";
 import mock from "mock-require";
 import test from "ava";
-mock.stopAll();
+
 mock("app", {
   currentUser: {
     firstName: "SCM",

frontend/javascripts/test/reducers/volumetracing_reducer.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import update from "immutability-helper";
 import { getFirstVolumeTracingOrFail } from "test/helpers/apiHelpers";
 import { AnnotationToolEnum } from "oxalis/constants";

frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import test from "ava";
 import { AnnotationToolEnum, AnnotationTool } from "oxalis/constants";
 import mockRequire from "mock-require";

frontend/javascripts/test/sagas/compact_toggle_actions.spec.ts

@@ -1,5 +1,6 @@
 // @ts-nocheck
 import _ from "lodash";
+import "test/mocks/lz4";
 import type { Flycam, OxalisState, Tree, TreeMap } from "oxalis/store";
 import { diffSkeletonTracing } from "oxalis/model/sagas/skeletontracing_saga";
 import { enforceSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor";

frontend/javascripts/test/sagas/prefetch_saga.spec.ts

@@ -1,13 +1,15 @@
+import "test/mocks/lz4";
 import Model from "oxalis/model";
 import constants from "oxalis/constants";
 import mockRequire from "mock-require";
 import test from "ava";
 import { expectValueDeepEqual, execCall } from "../helpers/sagaHelpers";
 import DATASET from "../fixtures/dataset_server_object";
-mockRequire.stopAll();
+
 mockRequire("oxalis/model/sagas/root_saga", function* () {
   yield;
 });
+
 const { call } = mockRequire.reRequire("redux-saga/effects");
 const { triggerDataPrefetching, prefetchForArbitraryMode, prefetchForPlaneMode } =
   mockRequire.reRequire("oxalis/model/sagas/prefetch_saga");

frontend/javascripts/test/sagas/saga_integration.mock.ts

@@ -1,6 +1,8 @@
 import mockRequire from "mock-require";
 import * as antd from "antd";
 import _ from "lodash";
+import "test/mocks/lz4";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

frontend/javascripts/test/sagas/save_saga.spec.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { alert } from "libs/window";
 import { setSaveBusyAction } from "oxalis/model/actions/save_actions";
 import DiffableMap from "libs/diffable_map";
@@ -7,7 +8,7 @@ import mockRequire from "mock-require";
 import test from "ava";
 import { createSaveQueueFromUpdateActions } from "../helpers/saveHelpers";
 import { expectValueDeepEqual } from "../helpers/sagaHelpers";
-mockRequire.stopAll();
+
 const TIMESTAMP = 1494695001688;
 const DateMock = {
   now: () => TIMESTAMP,

frontend/javascripts/test/sagas/skeletontracing_saga.mock.ts

@@ -1,4 +1,6 @@
+import "test/mocks/lz4";
 import mockRequire from "mock-require";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

frontend/javascripts/test/sagas/volumetracing/bucket_eviction_with_saving.spec.ts

@@ -1,6 +1,7 @@
 import test from "ava";
 import mockRequire from "mock-require";
 import { waitForCondition } from "libs/utils";
+import "test/mocks/lz4";
 import "test/sagas/saga_integration.mock";
 import { __setupOxalis, createBucketResponseFunction } from "test/helpers/apiHelpers";
 import { restartSagaAction, wkReadyAction } from "oxalis/model/actions/actions";

frontend/javascripts/test/sagas/volumetracing/bucket_eviction_without_saving.spec.ts

@@ -1,6 +1,7 @@
 import test from "ava";
 import mockRequire from "mock-require";
 import { waitForCondition } from "libs/utils";
+import "test/mocks/lz4";
 import "test/sagas/saga_integration.mock";
 import { __setupOxalis, createBucketResponseFunction } from "test/helpers/apiHelpers";
 import { restartSagaAction, wkReadyAction } from "oxalis/model/actions/actions";

frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.mock.ts

@@ -1,4 +1,6 @@
 import mockRequire from "mock-require";
+import "test/mocks/lz4";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

frontend/javascripts/test/shaders/shader_syntax.spec.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { getLookupBufferSize } from "oxalis/model/bucket_data_handling/data_rendering_logic";
 import constants from "oxalis/constants";
 import getMainFragmentShader from "oxalis/shaders/main_data_fragment.glsl";