scalableminds · philippotto · Nov 23, 2022 · Nov 21, 2022 · Nov 21, 2022 · Nov 21, 2022
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -28,7 +28,8 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Re-phrased some backend (error) messages to improve clarity and provide helping hints. [#6616](https://github.com/scalableminds/webknossos/pull/6616)
 - The layer visibility is now encoded in the sharing link. The user opening the link will see the same layers that were visible when copying the link. [#6634](https://github.com/scalableminds/webknossos/pull/6634)
 - Voxelytics workflows can now be viewed by anyone with the link who is in the right organization. [#6622](https://github.com/scalableminds/webknossos/pull/6622)
-
+ - Major performance improvements to volume saving/undo/redo. [#6652](https://github.com/scalableminds/webknossos/pull/6652)
+
 ### Fixed
 - Fixed a bug in the dataset import view, where the layer name text field would lose focus after each key press. [#6615](https://github.com/scalableminds/webknossos/pull/6615)
 - Fixed importing NGFF Zarr datasets with non-scale transforms. [#6621](https://github.com/scalableminds/webknossos/pull/6621)

diff --git a/frontend/javascripts/libs/utils.ts b/frontend/javascripts/libs/utils.ts
@@ -886,7 +886,7 @@ export function castForArrayType(uncastNumber: number, data: TypedArray): number
 }
 
 export function convertNumberTo64Bit(num: number | null): [Vector4, Vector4] {
-  if (num == null) {
+  if (num == null || isNaN(num)) {
     return [
       [0, 0, 0, 0],
       [0, 0, 0, 0],

diff --git a/frontend/javascripts/oxalis/api/api_v2.ts b/frontend/javascripts/oxalis/api/api_v2.ts
@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 import { InputKeyboardNoLoop } from "libs/input";
 import Model from "oxalis/model";

diff --git a/frontend/javascripts/oxalis/controller/url_manager.ts b/frontend/javascripts/oxalis/controller/url_manager.ts
@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import _ from "lodash";
 import { V3 } from "libs/mjs";
 import { applyState } from "oxalis/model_initialization";

diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/texture_bucket_manager.ts b/frontend/javascripts/oxalis/model/bucket_data_handling/texture_bucket_manager.ts
@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import * as THREE from "three";
 import _ from "lodash";
 import { DataBucket, bucketDebuggingFlags } from "oxalis/model/bucket_data_handling/bucket";

diff --git a/frontend/javascripts/oxalis/model/volumetracing/volume_annotation_sampling.ts b/frontend/javascripts/oxalis/model/volumetracing/volume_annotation_sampling.ts
@@ -1,4 +1,5 @@
 import _ from "lodash";
+import "test/mocks/lz4";
 import type { Vector3, LabeledVoxelsMap, Vector4 } from "oxalis/constants";
 import constants from "oxalis/constants";
 import { map3 } from "libs/utils";

diff --git a/frontend/javascripts/oxalis/workers/byte_array_lz4_compression.worker.ts b/frontend/javascripts/oxalis/workers/byte_array_lz4_compression.worker.ts
@@ -1,6 +1,5 @@
-// @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'lz4j... Remove this comment to see the full error message
-import lz4 from "lz4js";
 import { expose } from "./comlink_wrapper";
+import * as lz4 from "lz4-wasm";
 
 function compressLz4Block(data: Uint8Array, compress: boolean): Uint8Array {
   if (compress) {

diff --git a/frontend/javascripts/oxalis/workers/byte_array_to_lz4_base64.worker.ts b/frontend/javascripts/oxalis/workers/byte_array_to_lz4_base64.worker.ts
@@ -1,16 +1,13 @@
 import Base64 from "base64-js";
-// @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'lz4j... Remove this comment to see the full error message
-import lz4 from "lz4js";
+import * as lz4wasm from "lz4-wasm";
 import { expose } from "./comlink_wrapper";
 
 function compressLz4Block(data: Uint8Array): Uint8Array {
-  // Backend expects the frame-less version of lz4,
-  // so we need to call lz4.compressBlock rather than compress
-  const hashSize = 1 << 16;
-  const hashTable = new Uint32Array(hashSize);
-  const compressedBuffer = new Uint8Array(data.length);
-  const compressedSize = lz4.compressBlock(data, compressedBuffer, 0, data.length, hashTable);
-  return compressedBuffer.slice(0, compressedSize);
+  // The backend expects the block (frame-less) version of lz4.
+  // lz4-wasm uses the block version, but prepends the size of the
+  // compressed data. Therefore, we strip the first 4 bytes.
+  const newCompressed = lz4wasm.compress(data);
+  return newCompressed.slice(4);
 }
 
 export function byteArrayToLz4Base64(byteArray: Uint8Array): string {

diff --git a/frontend/javascripts/test/api/api_skeleton_latest.spec.ts b/frontend/javascripts/test/api/api_skeleton_latest.spec.ts
@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { __setupOxalis, KeyboardJS } from "test/helpers/apiHelpers";
 import { makeBasicGroupObject } from "oxalis/view/right-border-tabs/tree_hierarchy_view_helpers";
 import { setMappingEnabledAction } from "oxalis/model/actions/settings_actions";

diff --git a/frontend/javascripts/test/api/api_v2.spec.ts b/frontend/javascripts/test/api/api_v2.spec.ts
@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { __setupOxalis, KeyboardJS } from "test/helpers/apiHelpers";
 import { setMappingEnabledAction } from "oxalis/model/actions/settings_actions";
 import Store from "oxalis/store";

diff --git a/frontend/javascripts/test/api/api_volume_latest.spec.ts b/frontend/javascripts/test/api/api_volume_latest.spec.ts
@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import { AnnotationToolEnum } from "oxalis/constants";
 import { __setupOxalis } from "test/helpers/apiHelpers";
 import test from "ava";

diff --git a/frontend/javascripts/test/geometries/skeleton.spec.ts b/frontend/javascripts/test/geometries/skeleton.spec.ts
@@ -1,4 +1,5 @@
 // Integration tests for skeleton.js
+import "test/mocks/lz4";
 import _ from "lodash";
 import { getSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor";
 import * as Utils from "libs/utils";
@@ -7,7 +8,7 @@ import test from "ava";
 import { Vector3 } from "oxalis/constants";
 import { OxalisState } from "oxalis/store";
 import { tracing, annotation } from "../fixtures/skeletontracing_server_objects";
-mockRequire.stopAll();
+
 mockRequire("app", {
   currentUser: {
     firstName: "SCM",

diff --git a/frontend/javascripts/test/libs/nml.spec.ts b/frontend/javascripts/test/libs/nml.spec.ts
@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import _ from "lodash";
 import update from "immutability-helper";
 import sinon from "sinon";

diff --git a/frontend/javascripts/test/model/binary/cube.spec.ts b/frontend/javascripts/test/model/binary/cube.spec.ts
@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import _ from "lodash";
 import { ResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { tracing as skeletontracingServerObject } from "test/fixtures/skeletontracing_server_objects";
@@ -9,7 +10,7 @@ import datasetServerObject from "test/fixtures/dataset_server_object";
 import mockRequire from "mock-require";
 import runAsync from "test/helpers/run-async";
 import sinon from "sinon";
-mockRequire.stopAll();
+
 const StoreMock = {
   getState: () => ({
     dataset: datasetServerObject,

diff --git a/frontend/javascripts/test/model/binary/layers/wkstore_adapter.mock.ts b/frontend/javascripts/test/model/binary/layers/wkstore_adapter.mock.ts
@@ -1,4 +1,6 @@
+import "test/mocks/lz4";
 import mockRequire from "mock-require";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

diff --git a/frontend/javascripts/test/model/binary/temporal_bucket_manager.spec.ts b/frontend/javascripts/test/model/binary/temporal_bucket_manager.spec.ts
@@ -3,6 +3,8 @@ import runAsync from "test/helpers/run-async";
 import sinon from "sinon";
 import type { TestInterface } from "ava";
 import anyTest from "ava";
+import "test/mocks/lz4";
+
 mockRequire("oxalis/model/sagas/root_saga", function* () {
   yield;
 });

diff --git a/frontend/javascripts/test/model/model.spec.ts b/frontend/javascripts/test/model/model.spec.ts
@@ -9,7 +9,6 @@ import {
   annotation as ANNOTATION,
 } from "../fixtures/skeletontracing_server_objects";
 import DATASET from "../fixtures/dataset_server_object";
-mockRequire.stopAll();
 
 function makeModelMock() {
   class ModelMock {}

diff --git a/frontend/javascripts/test/model/model_resolutions.spec.ts b/frontend/javascripts/test/model/model_resolutions.spec.ts
@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import test from "ava";
 import mockRequire from "mock-require";
 import {

diff --git a/frontend/javascripts/test/model/volumetracing/volume_annotation_sampling.spec.ts b/frontend/javascripts/test/model/volumetracing/volume_annotation_sampling.spec.ts
@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { ResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { tracing as skeletontracingServerObject } from "test/fixtures/skeletontracing_server_objects";
 import { tracing as volumetracingServerObject } from "test/fixtures/volumetracing_server_objects";
@@ -8,7 +9,7 @@ import anyTest from "ava";
 import datasetServerObject from "test/fixtures/dataset_server_object";
 import mockRequire from "mock-require";
 import sinon from "sinon";
-mockRequire.stopAll();
+
 const StoreMock = {
   getState: () => ({
     dataset: datasetServerObject,

diff --git a/frontend/javascripts/test/reducers/skeletontracing_reducer.spec.ts b/frontend/javascripts/test/reducers/skeletontracing_reducer.spec.ts
@@ -1,5 +1,6 @@
 // @ts-nocheck
 /* eslint-disable no-useless-computed-key */
+import "test/mocks/lz4";
 import _ from "lodash";
 import update from "immutability-helper";
 import { rgbs as colors } from "libs/color_generator";
@@ -8,7 +9,7 @@ import DiffableMap from "libs/diffable_map";
 import EdgeCollection from "oxalis/model/edge_collection";
 import mock from "mock-require";
 import test from "ava";
-mock.stopAll();
+
 mock("app", {
   currentUser: {
     firstName: "SCM",

diff --git a/frontend/javascripts/test/reducers/volumetracing_reducer.spec.ts b/frontend/javascripts/test/reducers/volumetracing_reducer.spec.ts
@@ -1,4 +1,5 @@
 // @ts-nocheck
+import "test/mocks/lz4";
 import update from "immutability-helper";
 import { getFirstVolumeTracingOrFail } from "test/helpers/apiHelpers";
 import { AnnotationToolEnum } from "oxalis/constants";

diff --git a/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts b/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts
@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import test from "ava";
 import { AnnotationToolEnum, AnnotationTool } from "oxalis/constants";
 import mockRequire from "mock-require";

diff --git a/frontend/javascripts/test/sagas/compact_toggle_actions.spec.ts b/frontend/javascripts/test/sagas/compact_toggle_actions.spec.ts
@@ -1,5 +1,6 @@
 // @ts-nocheck
 import _ from "lodash";
+import "test/mocks/lz4";
 import type { Flycam, OxalisState, Tree, TreeMap } from "oxalis/store";
 import { diffSkeletonTracing } from "oxalis/model/sagas/skeletontracing_saga";
 import { enforceSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor";

diff --git a/frontend/javascripts/test/sagas/prefetch_saga.spec.ts b/frontend/javascripts/test/sagas/prefetch_saga.spec.ts
@@ -1,13 +1,15 @@
+import "test/mocks/lz4";
 import Model from "oxalis/model";
 import constants from "oxalis/constants";
 import mockRequire from "mock-require";
 import test from "ava";
 import { expectValueDeepEqual, execCall } from "../helpers/sagaHelpers";
 import DATASET from "../fixtures/dataset_server_object";
-mockRequire.stopAll();
+
 mockRequire("oxalis/model/sagas/root_saga", function* () {
   yield;
 });
+
 const { call } = mockRequire.reRequire("redux-saga/effects");
 const { triggerDataPrefetching, prefetchForArbitraryMode, prefetchForPlaneMode } =
   mockRequire.reRequire("oxalis/model/sagas/prefetch_saga");

diff --git a/frontend/javascripts/test/sagas/saga_integration.mock.ts b/frontend/javascripts/test/sagas/saga_integration.mock.ts
@@ -1,6 +1,8 @@
 import mockRequire from "mock-require";
 import * as antd from "antd";
 import _ from "lodash";
+import "test/mocks/lz4";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts
@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { alert } from "libs/window";
 import { setSaveBusyAction } from "oxalis/model/actions/save_actions";
 import DiffableMap from "libs/diffable_map";
@@ -7,7 +8,7 @@ import mockRequire from "mock-require";
 import test from "ava";
 import { createSaveQueueFromUpdateActions } from "../helpers/saveHelpers";
 import { expectValueDeepEqual } from "../helpers/sagaHelpers";
-mockRequire.stopAll();
+
 const TIMESTAMP = 1494695001688;
 const DateMock = {
   now: () => TIMESTAMP,

diff --git a/frontend/javascripts/test/sagas/skeletontracing_saga.mock.ts b/frontend/javascripts/test/sagas/skeletontracing_saga.mock.ts
@@ -1,4 +1,6 @@
+import "test/mocks/lz4";
 import mockRequire from "mock-require";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

diff --git a/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_with_saving.spec.ts b/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_with_saving.spec.ts
@@ -1,6 +1,7 @@
 import test from "ava";
 import mockRequire from "mock-require";
 import { waitForCondition } from "libs/utils";
+import "test/mocks/lz4";
 import "test/sagas/saga_integration.mock";
 import { __setupOxalis, createBucketResponseFunction } from "test/helpers/apiHelpers";
 import { restartSagaAction, wkReadyAction } from "oxalis/model/actions/actions";

diff --git a/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_without_saving.spec.ts b/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_without_saving.spec.ts
@@ -1,6 +1,7 @@
 import test from "ava";
 import mockRequire from "mock-require";
 import { waitForCondition } from "libs/utils";
+import "test/mocks/lz4";
 import "test/sagas/saga_integration.mock";
 import { __setupOxalis, createBucketResponseFunction } from "test/helpers/apiHelpers";
 import { restartSagaAction, wkReadyAction } from "oxalis/model/actions/actions";

diff --git a/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.mock.ts b/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.mock.ts
@@ -1,4 +1,6 @@
 import mockRequire from "mock-require";
+import "test/mocks/lz4";
+
 const REQUEST_ID = "dummyRequestId";
 const UidMock = {
   getUid: () => REQUEST_ID,

diff --git a/frontend/javascripts/test/shaders/shader_syntax.spec.ts b/frontend/javascripts/test/shaders/shader_syntax.spec.ts
@@ -1,3 +1,4 @@
+import "test/mocks/lz4";
 import { getLookupBufferSize } from "oxalis/model/bucket_data_handling/data_rendering_logic";
 import constants from "oxalis/constants";
 import getMainFragmentShader from "oxalis/shaders/main_data_fragment.glsl";

diff --git a/package.json b/package.json
@@ -64,6 +64,7 @@
     "less": "^4.0.0",
     "less-loader": "^10.2.0",
     "lint-staged": "^7.3.0",
+    "lz4-wasm-nodejs": "^0.9.2",
     "merge-img": "^2.1.2",
     "mock-require": "^1.2.1",
     "node-fetch": "^2.6.7",
@@ -183,7 +184,7 @@
     "jsonschema": "^1.2.4",
     "jszip": "^3.7.0",
     "lodash": "^4.17.21",
-    "lz4js": "^0.2.0",
+    "lz4-wasm": "^0.9.2",
     "memoize-one": "^6.0.0",
     "mini-css-extract-plugin": "^2.5.2",
     "minisearch": "^5.0.0",

diff --git a/webpack.config.js b/webpack.config.js
@@ -47,6 +47,7 @@ module.exports = function (env = {}) {
   };
 
   return {
+    experiments: { asyncWebAssembly: true },
     entry: {
       main: "main.tsx",
       light: "style_light.ts",
@@ -136,6 +137,13 @@ module.exports = function (env = {}) {
       fallback: {
         // Needed for jsonschema
         url: require.resolve("url/"),
+        // Needed for lz4-wasm-nodejs (which is only used in test context, which is
+        // why we use empty modules)
+        path: false,
+        util: false,
+        fs: false,
+        // Needed for mock-require
+        module: false,
       },
     },
     optimization: {

diff --git a/yarn.lock b/yarn.lock
@@ -8722,10 +8722,15 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
-lz4js@^0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/lz4js/-/lz4js-0.2.0.tgz#09f1a397cb2158f675146c3351dde85058cb322f"
-  integrity sha1-CfGjl8shWPZ1FGwzUd3oUFjLMi8=
+lz4-wasm-nodejs@^0.9.2:
+  version "0.9.2"
+  resolved "https://registry.yarnpkg.com/lz4-wasm-nodejs/-/lz4-wasm-nodejs-0.9.2.tgz#eaa3184ac048e1af9bd0ec82f89de8e8f158c6c7"
+  integrity sha512-hSwgJPS98q/Oe/89Y1OxzeA/UdnASG8GvldRyKa7aZyoAFCC8VPRtViBSava7wWC66WocjUwBpWau2rEmyFPsw==
+
+lz4-wasm@^0.9.2:
+  version "0.9.2"
+  resolved "https://registry.yarnpkg.com/lz4-wasm/-/lz4-wasm-0.9.2.tgz#2047b8a9771ad33b1699c350cda3e86249fbbb46"
+  integrity sha512-8Y/OvG/nTQNWR242NhLmjI7VzLMRKFaxg/+Pxi/vkczgFdrS4qiJ8l6n30XyNJyc0TB5C66Ncdjq345Kx5XsvQ==
 
 magic-string@^0.25.7:
   version "0.25.7"