Use LZ4 with WASM for volume saving/undo/redo

CHANGELOG.unreleased.md

@@ -28,7 +28,8 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Re-phrased some backend (error) messages to improve clarity and provide helping hints. [#6616](https://github.com/scalableminds/webknossos/pull/6616)
 - The layer visibility is now encoded in the sharing link. The user opening the link will see the same layers that were visible when copying the link. [#6634](https://github.com/scalableminds/webknossos/pull/6634)
 - Voxelytics workflows can now be viewed by anyone with the link who is in the right organization. [#6622](https://github.com/scalableminds/webknossos/pull/6622)
-
+ - Major performance improvements to volume saving/undo/redo. [#6652](https://github.com/scalableminds/webknossos/pull/6652)
+
 ### Fixed
 - Fixed a bug in the dataset import view, where the layer name text field would lose focus after each key press. [#6615](https://github.com/scalableminds/webknossos/pull/6615)
 - Fixed importing NGFF Zarr datasets with non-scale transforms. [#6621](https://github.com/scalableminds/webknossos/pull/6621)

conf/application.conf

@@ -18,6 +18,9 @@ play {
       maxDiskBuffer = 1G
     }
     filters = "com.scalableminds.webknossos.datastore.Filters"
+    fileMimeTypes = ${play.http.fileMimeTypes} """
+      wasm=application/wasm
+    """
   }
   filters.headers {
     # Unset some of the security filters enabled in datastore.Filters
@@ -175,20 +178,8 @@ mail {
   }
 }
 
-# SQL database connection
-slick = {
-  profile = "slick.jdbc.PostgresProfile$"
-  codegen.package = "com.scalableminds.webknossos.schema"
-  db = {
-    url = "jdbc:postgresql://localhost/webknossos"
-    url = ${?POSTGRES_URL}
-    driver = org.postgresql.Driver
-    keepAliveConnection = true
-    user = "postgres"
-    password = "postgres"
-    queueSize = 5000
-  }
-}
+# SQL Slick Database Connection Config in Subfile to provide it also to the AssetGeneration task
+include "slick.conf"
 
 # Authentication via cookies and tokens
 silhouette {

conf/slick.conf

@@ -0,0 +1,14 @@
+# SQL database connection
+slick = {
+  profile = "slick.jdbc.PostgresProfile$"
+  codegen.package = "com.scalableminds.webknossos.schema"
+  db = {
+    url = "jdbc:postgresql://localhost/webknossos"
+    url = ${?POSTGRES_URL}
+    driver = org.postgresql.Driver
+    keepAliveConnection = true
+    user = "postgres"
+    password = "postgres"
+    queueSize = 5000
+  }
+}

frontend/javascripts/libs/utils.ts

@@ -886,7 +886,7 @@ export function castForArrayType(uncastNumber: number, data: TypedArray): number
 }
 
 export function convertNumberTo64Bit(num: number | null): [Vector4, Vector4] {
-  if (num == null) {
+  if (num == null || isNaN(num)) {
     return [
       [0, 0, 0, 0],
       [0, 0, 0, 0],

frontend/javascripts/oxalis/workers/byte_array_lz4_compression.worker.ts

@@ -1,6 +1,5 @@
-// @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'lz4j... Remove this comment to see the full error message
-import lz4 from "lz4js";
 import { expose } from "./comlink_wrapper";
+import * as lz4 from "lz4-wasm";
 
 function compressLz4Block(data: Uint8Array, compress: boolean): Uint8Array {
   if (compress) {

frontend/javascripts/oxalis/workers/byte_array_to_lz4_base64.worker.ts

@@ -1,16 +1,13 @@
 import Base64 from "base64-js";
-// @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'lz4j... Remove this comment to see the full error message
-import lz4 from "lz4js";
+import * as lz4wasm from "lz4-wasm";
 import { expose } from "./comlink_wrapper";
 
 function compressLz4Block(data: Uint8Array): Uint8Array {
-  // Backend expects the frame-less version of lz4,
-  // so we need to call lz4.compressBlock rather than compress
-  const hashSize = 1 << 16;
-  const hashTable = new Uint32Array(hashSize);
-  const compressedBuffer = new Uint8Array(data.length);
-  const compressedSize = lz4.compressBlock(data, compressedBuffer, 0, data.length, hashTable);
-  return compressedBuffer.slice(0, compressedSize);
+  // The backend expects the block (frame-less) version of lz4.
+  // lz4-wasm uses the block version, but prepends the size of the
+  // compressed data. Therefore, we strip the first 4 bytes.
+  const newCompressed = lz4wasm.compress(data);
+  return newCompressed.slice(4);
 }
 
 export function byteArrayToLz4Base64(byteArray: Uint8Array): string {

frontend/javascripts/test/api/api_skeleton_latest.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { __setupOxalis, KeyboardJS } from "test/helpers/apiHelpers";
 import { makeBasicGroupObject } from "oxalis/view/right-border-tabs/tree_hierarchy_view_helpers";
 import { setMappingEnabledAction } from "oxalis/model/actions/settings_actions";

frontend/javascripts/test/api/api_v2.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { __setupOxalis, KeyboardJS } from "test/helpers/apiHelpers";
 import { setMappingEnabledAction } from "oxalis/model/actions/settings_actions";
 import Store from "oxalis/store";

frontend/javascripts/test/api/api_volume_latest.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { AnnotationToolEnum } from "oxalis/constants";
 import { __setupOxalis } from "test/helpers/apiHelpers";
 import test from "ava";

frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import type { APIAnnotation } from "types/api_flow_types";
 import { APIAnnotationTypeEnum } from "types/api_flow_types";
 import { createTreeMapFromTreeArray } from "oxalis/model/reducers/skeletontracing_reducer_helpers";

frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 import type { APIDataset } from "types/api_flow_types";
 import {

frontend/javascripts/test/backend-snapshot-tests/misc.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import {
   tokenUserA,
   setCurrToken,

frontend/javascripts/test/backend-snapshot-tests/projects.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 import type { APIProject, APIProjectUpdater } from "types/api_flow_types";
 import {

frontend/javascripts/test/backend-snapshot-tests/scripts.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import {
   tokenUserA,
   setCurrToken,

frontend/javascripts/test/backend-snapshot-tests/tasks.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 import { resetDatabase, replaceVolatileValues, writeTypeCheckingFile } from "test/enzyme/e2e-setup";
 import * as api from "admin/admin_rest_api";

frontend/javascripts/test/backend-snapshot-tests/tasktypes.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import {
   tokenUserA,
   setCurrToken,

frontend/javascripts/test/backend-snapshot-tests/teams.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 import {
   tokenUserA,

frontend/javascripts/test/backend-snapshot-tests/teamstructure.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 import {
   tokenUserA,

frontend/javascripts/test/backend-snapshot-tests/timetracking.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 import moment from "moment";
 import {

frontend/javascripts/test/backend-snapshot-tests/users.e2e.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import {
   tokenUserA,
   setCurrToken,

frontend/javascripts/test/controller/url_manager.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import test from "ava";
 import UrlManager, { updateTypeAndId, encodeUrlHash } from "oxalis/controller/url_manager";
 import { location } from "libs/window";

frontend/javascripts/test/geometries/skeleton.spec.ts

@@ -1,4 +1,5 @@
 // Integration tests for skeleton.js
+import "test/mocks/lz4";
 import _ from "lodash";
 import { getSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor";
 import * as Utils from "libs/utils";
@@ -7,7 +8,7 @@ import test from "ava";
 import { Vector3 } from "oxalis/constants";
 import { OxalisState } from "oxalis/store";
 import { tracing, annotation } from "../fixtures/skeletontracing_server_objects";
-mockRequire.stopAll();
+
 mockRequire("app", {
   currentUser: {
     firstName: "SCM",

frontend/javascripts/test/libs/nml.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import _ from "lodash";
 import update from "immutability-helper";
 import sinon from "sinon";

frontend/javascripts/test/mocks/lz4.ts

@@ -0,0 +1,4 @@
+import mockRequire from "mock-require";
+import * as lz4 from "lz4-wasm-nodejs";
+
+mockRequire("lz4-wasm", lz4);

frontend/javascripts/test/model/binary/cube.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import _ from "lodash";
 import { ResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { tracing as skeletontracingServerObject } from "test/fixtures/skeletontracing_server_objects";
@@ -9,7 +10,7 @@ import datasetServerObject from "test/fixtures/dataset_server_object";
 import mockRequire from "mock-require";
 import runAsync from "test/helpers/run-async";
 import sinon from "sinon";
-mockRequire.stopAll();
+
 const StoreMock = {
   getState: () => ({
     dataset: datasetServerObject,

frontend/javascripts/test/model/binary/layers/wkstore_adapter.mock.ts

@@ -1,4 +1,6 @@
+import "test/mocks/lz4";
 import mockRequire from "mock-require";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

frontend/javascripts/test/model/binary/temporal_bucket_manager.spec.ts

@@ -3,6 +3,8 @@ import runAsync from "test/helpers/run-async";
 import sinon from "sinon";
 import type { TestInterface } from "ava";
 import anyTest from "ava";
+import "test/mocks/lz4";
+
 mockRequire("oxalis/model/sagas/root_saga", function* () {
   yield;
 });

frontend/javascripts/test/model/model.spec.ts

@@ -1,5 +1,6 @@
 // @ts-nocheck
 import _ from "lodash";
+import "test/mocks/lz4";
 import mockRequire from "mock-require";
 import sinon from "sinon";
 import test from "ava";
@@ -9,7 +10,6 @@ import {
   annotation as ANNOTATION,
 } from "../fixtures/skeletontracing_server_objects";
 import DATASET from "../fixtures/dataset_server_object";
-mockRequire.stopAll();
 
 function makeModelMock() {
   class ModelMock {}

frontend/javascripts/test/model/model_resolutions.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import test from "ava";
 import mockRequire from "mock-require";
 import {

frontend/javascripts/test/model/texture_bucket_manager.spec.ts

@@ -1,4 +1,5 @@
 import mock from "mock-require";
+import "test/mocks/lz4";
 import test, { ExecutionContext } from "ava";
 import { Vector4 } from "oxalis/constants";
 

frontend/javascripts/test/model/volumetracing/volume_annotation_sampling.spec.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { ResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { tracing as skeletontracingServerObject } from "test/fixtures/skeletontracing_server_objects";
 import { tracing as volumetracingServerObject } from "test/fixtures/volumetracing_server_objects";
@@ -8,7 +9,7 @@ import anyTest from "ava";
 import datasetServerObject from "test/fixtures/dataset_server_object";
 import mockRequire from "mock-require";
 import sinon from "sinon";
-mockRequire.stopAll();
+
 const StoreMock = {
   getState: () => ({
     dataset: datasetServerObject,

frontend/javascripts/test/reducers/skeletontracing_reducer.spec.ts

@@ -1,5 +1,6 @@
 // @ts-nocheck
 /* eslint-disable no-useless-computed-key */
+import "test/mocks/lz4";
 import _ from "lodash";
 import update from "immutability-helper";
 import { rgbs as colors } from "libs/color_generator";
@@ -8,7 +9,7 @@ import DiffableMap from "libs/diffable_map";
 import EdgeCollection from "oxalis/model/edge_collection";
 import mock from "mock-require";
 import test from "ava";
-mock.stopAll();
+
 mock("app", {
   currentUser: {
     firstName: "SCM",

frontend/javascripts/test/reducers/volumetracing_reducer.spec.ts

@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import update from "immutability-helper";
 import { getFirstVolumeTracingOrFail } from "test/helpers/apiHelpers";
 import { AnnotationToolEnum } from "oxalis/constants";

frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import test from "ava";
 import { AnnotationToolEnum, AnnotationTool } from "oxalis/constants";
 import mockRequire from "mock-require";

frontend/javascripts/test/sagas/compact_toggle_actions.spec.ts

@@ -1,5 +1,6 @@
 // @ts-nocheck
 import _ from "lodash";
+import "test/mocks/lz4";
 import type { Flycam, OxalisState, Tree, TreeMap } from "oxalis/store";
 import { diffSkeletonTracing } from "oxalis/model/sagas/skeletontracing_saga";
 import { enforceSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor";

frontend/javascripts/test/sagas/prefetch_saga.spec.ts

@@ -1,13 +1,15 @@
+import "test/mocks/lz4";
 import Model from "oxalis/model";
 import constants from "oxalis/constants";
 import mockRequire from "mock-require";
 import test from "ava";
 import { expectValueDeepEqual, execCall } from "../helpers/sagaHelpers";
 import DATASET from "../fixtures/dataset_server_object";
-mockRequire.stopAll();
+
 mockRequire("oxalis/model/sagas/root_saga", function* () {
   yield;
 });
+
 const { call } = mockRequire.reRequire("redux-saga/effects");
 const { triggerDataPrefetching, prefetchForArbitraryMode, prefetchForPlaneMode } =
   mockRequire.reRequire("oxalis/model/sagas/prefetch_saga");

frontend/javascripts/test/sagas/saga_integration.mock.ts

@@ -1,6 +1,8 @@
 import mockRequire from "mock-require";
 import * as antd from "antd";
 import _ from "lodash";
+import "test/mocks/lz4";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

frontend/javascripts/test/sagas/save_saga.spec.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { alert } from "libs/window";
 import { setSaveBusyAction } from "oxalis/model/actions/save_actions";
 import DiffableMap from "libs/diffable_map";
@@ -7,7 +8,7 @@ import mockRequire from "mock-require";
 import test from "ava";
 import { createSaveQueueFromUpdateActions } from "../helpers/saveHelpers";
 import { expectValueDeepEqual } from "../helpers/sagaHelpers";
-mockRequire.stopAll();
+
 const TIMESTAMP = 1494695001688;
 const DateMock = {
   now: () => TIMESTAMP,

frontend/javascripts/test/sagas/skeletontracing_saga.mock.ts

@@ -1,4 +1,6 @@
+import "test/mocks/lz4";
 import mockRequire from "mock-require";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

frontend/javascripts/test/sagas/volumetracing/bucket_eviction_with_saving.spec.ts

@@ -1,6 +1,7 @@
 import test from "ava";
 import mockRequire from "mock-require";
 import { waitForCondition } from "libs/utils";
+import "test/mocks/lz4";
 import "test/sagas/saga_integration.mock";
 import { __setupOxalis, createBucketResponseFunction } from "test/helpers/apiHelpers";
 import { restartSagaAction, wkReadyAction } from "oxalis/model/actions/actions";

frontend/javascripts/test/sagas/volumetracing/bucket_eviction_without_saving.spec.ts

@@ -1,6 +1,7 @@
 import test from "ava";
 import mockRequire from "mock-require";
 import { waitForCondition } from "libs/utils";
+import "test/mocks/lz4";
 import "test/sagas/saga_integration.mock";
 import { __setupOxalis, createBucketResponseFunction } from "test/helpers/apiHelpers";
 import { restartSagaAction, wkReadyAction } from "oxalis/model/actions/actions";

frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.mock.ts

@@ -1,4 +1,6 @@
 import mockRequire from "mock-require";
+import "test/mocks/lz4";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

frontend/javascripts/test/shaders/shader_syntax.spec.ts

@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { getLookupBufferSize } from "oxalis/model/bucket_data_handling/data_rendering_logic";
 import constants from "oxalis/constants";
 import getMainFragmentShader from "oxalis/shaders/main_data_fragment.glsl";