Move blurhashing into a Worker and use OffscreenCanvas for thumbnailing

__mocks__/workerMock.js

@@ -0,0 +1 @@
+module.exports = jest.fn();

package.json

@@ -187,7 +187,8 @@
       "\\$webapp/i18n/languages.json": "<rootDir>/__mocks__/languages.json",
       "decoderWorker\\.min\\.js": "<rootDir>/__mocks__/empty.js",
       "decoderWorker\\.min\\.wasm": "<rootDir>/__mocks__/empty.js",
-      "waveWorker\\.min\\.js": "<rootDir>/__mocks__/empty.js"
+      "waveWorker\\.min\\.js": "<rootDir>/__mocks__/empty.js",
+      "workers/(.+)\\.worker\\.ts": "<rootDir>/__mocks__/workerMock.js"
     },
     "transformIgnorePatterns": [
       "/node_modules/(?!matrix-js-sdk).+$"

src/BlurhashEncoder.ts

@@ -0,0 +1,60 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { defer, IDeferred } from "matrix-js-sdk/src/utils";
+
+// @ts-ignore - `.ts` is needed here to make TS happy
+import BlurhashWorker from "./workers/blurhash.worker.ts";
+
+interface IBlurhashWorkerResponse {
+    seq: number;
+    blurhash: string;
+}
+
+export class BlurhashEncoder {
+    private static internalInstance = new BlurhashEncoder();
+
+    public static get instance(): BlurhashEncoder {
+        return BlurhashEncoder.internalInstance;
+    }
+
+    private readonly worker: Worker;
+    private seq = 0;
+    private pendingDeferredMap = new Map<number, IDeferred<string>>();
+
+    constructor() {
+        this.worker = new BlurhashWorker();
+        this.worker.onmessage = this.onMessage;
+    }
+
+    private onMessage = (ev: MessageEvent<IBlurhashWorkerResponse>) => {
+        const { seq, blurhash } = ev.data;
+        const deferred = this.pendingDeferredMap.get(seq);
+        if (deferred) {
+            this.pendingDeferredMap.delete(seq);
+            deferred.resolve(blurhash);
+        }
+    };
+
+    public getBlurhash(imageData: ImageData): Promise<string> {
+        const seq = this.seq++;
+        const deferred = defer<string>();
+        this.pendingDeferredMap.set(seq, deferred);
+        this.worker.postMessage({ seq, imageData });
+        return deferred.promise;
+    }
+}
+

src/ContentMessages.tsx

@@ -17,7 +17,6 @@ limitations under the License.
 */
 
 import React from "react";
-import { encode } from "blurhash";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 
 import dis from './dispatcher/dispatcher';
@@ -28,7 +27,6 @@ import RoomViewStore from './stores/RoomViewStore';
 import encrypt from "browser-encrypt-attachment";
 import extractPngChunks from "png-chunks-extract";
 import Spinner from "./components/views/elements/Spinner";
-
 import { Action } from "./dispatcher/actions";
 import CountlyAnalytics from "./CountlyAnalytics";
 import {
@@ -40,6 +38,7 @@ import {
 } from "./dispatcher/payloads/UploadPayload";
 import { IUpload } from "./models/IUpload";
 import { IAbortablePromise, IImageInfo } from "matrix-js-sdk/src/@types/partials";
+import { BlurhashEncoder } from "./BlurhashEncoder";
 
 const MAX_WIDTH = 800;
 const MAX_HEIGHT = 600;
@@ -103,55 +102,62 @@ interface IThumbnail {
  * @return {Promise} A promise that resolves with an object with an info key
  *  and a thumbnail key.
  */
-function createThumbnail(
+async function createThumbnail(
     element: ThumbnailableElement,
     inputWidth: number,
     inputHeight: number,
     mimeType: string,
 ): Promise<IThumbnail> {
-    return new Promise((resolve) => {
-        let targetWidth = inputWidth;
-        let targetHeight = inputHeight;
-        if (targetHeight > MAX_HEIGHT) {
-            targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight));
-            targetHeight = MAX_HEIGHT;
-        }
-        if (targetWidth > MAX_WIDTH) {
-            targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth));
-            targetWidth = MAX_WIDTH;
-        }
+    let targetWidth = inputWidth;
+    let targetHeight = inputHeight;
+    if (targetHeight > MAX_HEIGHT) {
+        targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight));
+        targetHeight = MAX_HEIGHT;
+    }
+    if (targetWidth > MAX_WIDTH) {
+        targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth));
+        targetWidth = MAX_WIDTH;
+    }
 
-        const canvas = document.createElement("canvas");
+    let canvas: HTMLCanvasElement | OffscreenCanvas;
+    if (window.OffscreenCanvas) {
+        canvas = new window.OffscreenCanvas(targetWidth, targetHeight);
+    } else {
+        canvas = document.createElement("canvas");
         canvas.width = targetWidth;
         canvas.height = targetHeight;
-        const context = canvas.getContext("2d");
-        context.drawImage(element, 0, 0, targetWidth, targetHeight);
-        const imageData = context.getImageData(0, 0, targetWidth, targetHeight);
-        const blurhash = encode(
-            imageData.data,
-            imageData.width,
-            imageData.height,
-            // use 4 components on the longer dimension, if square then both
-            imageData.width >= imageData.height ? 4 : 3,
-            imageData.height >= imageData.width ? 4 : 3,
-        );
-        canvas.toBlob(function(thumbnail) {
-            resolve({
-                info: {
-                    thumbnail_info: {
-                        w: targetWidth,
-                        h: targetHeight,
-                        mimetype: thumbnail.type,
-                        size: thumbnail.size,
-                    },
-                    w: inputWidth,
-                    h: inputHeight,
-                    [BLURHASH_FIELD]: blurhash,
-                },
-                thumbnail,
-            });
-        }, mimeType);
-    });
+    }
+
+    const context = canvas.getContext("2d");
+    context.drawImage(element, 0, 0, targetWidth, targetHeight);
+
+    let thumbnailPromise: Promise<Blob>;
+
+    if (window.OffscreenCanvas) {
+        thumbnailPromise = (canvas as OffscreenCanvas).convertToBlob({ type: mimeType });
+    } else {
+        thumbnailPromise = new Promise<Blob>(resolve => (canvas as HTMLCanvasElement).toBlob(resolve, mimeType));
+    }
+
+    const imageData = context.getImageData(0, 0, targetWidth, targetHeight);
+    // thumbnailPromise and blurhash promise are being awaited concurrently
+    const blurhash = await BlurhashEncoder.instance.getBlurhash(imageData);
+    const thumbnail = await thumbnailPromise;
+
+    return {
+        info: {
+            thumbnail_info: {
+                w: targetWidth,
+                h: targetHeight,
+                mimetype: thumbnail.type,
+                size: thumbnail.size,
+            },
+            w: inputWidth,
+            h: inputHeight,
+            [BLURHASH_FIELD]: blurhash,
+        },
+        thumbnail,
+    };
 }
 
 /**

src/workers/blurhash.worker.ts

@@ -0,0 +1,38 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { encode } from "blurhash";
+
+const ctx: Worker = self as any;
+
+interface IBlurhashWorkerRequest {
+    seq: number;
+    imageData: ImageData;
+}
+
+ctx.addEventListener("message", (event: MessageEvent<IBlurhashWorkerRequest>): void => {
+    const { seq, imageData } = event.data;
+    const blurhash = encode(
+        imageData.data,
+        imageData.width,
+        imageData.height,
+        // use 4 components on the longer dimension, if square then both
+        imageData.width >= imageData.height ? 4 : 3,
+        imageData.height >= imageData.width ? 4 : 3,
+    );
+
+    ctx.postMessage({ seq, blurhash });
+});