Prevent Element appearing in system media controls

jest.config.ts

@@ -22,7 +22,7 @@ const config: Config = {
     testEnvironment: "jsdom",
     testMatch: ["<rootDir>/test/**/*-test.[jt]s?(x)"],
     globalSetup: "<rootDir>/test/globalSetup.ts",
-    setupFiles: ["jest-canvas-mock"],
+    setupFiles: ["jest-canvas-mock", "web-streams-polyfill/polyfill"],
     setupFilesAfterEnv: ["<rootDir>/test/setupTests.ts"],
     moduleNameMapper: {
         "\\.(gif|png|ttf|woff2)$": "<rootDir>/__mocks__/imageMock.js",

package.json

@@ -227,7 +227,8 @@
         "stylelint-config-standard": "^36.0.0",
         "stylelint-scss": "^6.0.0",
         "ts-node": "^10.9.1",
-        "typescript": "5.5.2"
+        "typescript": "5.5.2",
+        "web-streams-polyfill": "^4.0.0"
     },
     "peerDependencies": {
         "postcss": "^8.4.19",

src/LegacyCallHandler.tsx

@@ -66,6 +66,7 @@ import { localNotificationsAreSilenced } from "./utils/notifications";
 import { SdkContextClass } from "./contexts/SDKContext";
 import { showCantStartACallDialog } from "./voice-broadcast/utils/showCantStartACallDialog";
 import { isNotNull } from "./Typeguards";
+import { BackgroundAudio } from "./audio/BackgroundAudio";
 
 export const PROTOCOL_PSTN = "m.protocol.pstn";
 export const PROTOCOL_PSTN_PREFIXED = "im.vector.protocol.pstn";
@@ -157,8 +158,6 @@ export default class LegacyCallHandler extends EventEmitter {
     // Calls started as an attended transfer, ie. with the intention of transferring another
     // call with a different party to this one.
     private transferees = new Map<string, MatrixCall>(); // callId (target) -> call (transferee)
-    private audioPromises = new Map<AudioID, Promise<void>>();
-    private audioElementsWithListeners = new Map<HTMLMediaElement, boolean>();
     private supportsPstnProtocol: boolean | null = null;
     private pstnSupportPrefixed: boolean | null = null; // True if the server only support the prefixed pstn protocol
     private supportsSipNativeVirtual: boolean | null = null; // im.vector.protocol.sip_virtual and im.vector.protocol.sip_native
@@ -170,6 +169,9 @@ export default class LegacyCallHandler extends EventEmitter {
 
     private silencedCalls = new Set<string>(); // callIds
 
+    private backgroundAudio = new BackgroundAudio();
+    private playingSources: Record<string, AudioBufferSourceNode> = {}; // Record them for stopping
+
     public static get instance(): LegacyCallHandler {
         if (!window.mxLegacyCallHandler) {
             window.mxLegacyCallHandler = new LegacyCallHandler();
@@ -199,61 +201,18 @@ export default class LegacyCallHandler extends EventEmitter {
     }
 
     public start(): void {
-        // add empty handlers for media actions, otherwise the media keys
-        // end up causing the audio elements with our ring/ringback etc
-        // audio clips in to play.
-        if (navigator.mediaSession) {
-            navigator.mediaSession.setActionHandler("play", function () {});
-            navigator.mediaSession.setActionHandler("pause", function () {});
-            navigator.mediaSession.setActionHandler("seekbackward", function () {});
-            navigator.mediaSession.setActionHandler("seekforward", function () {});
-            navigator.mediaSession.setActionHandler("previoustrack", function () {});
-            navigator.mediaSession.setActionHandler("nexttrack", function () {});
-        }
-
         if (SettingsStore.getValue(UIFeature.Voip)) {
             MatrixClientPeg.safeGet().on(CallEventHandlerEvent.Incoming, this.onCallIncoming);
         }
 
         this.checkProtocols(CHECK_PROTOCOLS_ATTEMPTS);
-
-        // Add event listeners for the <audio> elements
-        Object.values(AudioID).forEach((audioId) => {
-            const audioElement = document.getElementById(audioId) as HTMLMediaElement;
-            if (audioElement) {
-                this.addEventListenersForAudioElement(audioElement);
-            } else {
-                logger.warn(`LegacyCallHandler: missing <audio id="${audioId}"> from page`);
-            }
-        });
     }
 
     public stop(): void {
         const cli = MatrixClientPeg.get();
         if (cli) {
             cli.removeListener(CallEventHandlerEvent.Incoming, this.onCallIncoming);
         }
-
-        // Remove event listeners for the <audio> elements
-        Array.from(this.audioElementsWithListeners.keys()).forEach((audioElement) => {
-            this.removeEventListenersForAudioElement(audioElement);
-        });
-    }
-
-    private addEventListenersForAudioElement(audioElement: HTMLMediaElement): void {
-        // Only need to setup the listeners once
-        if (!this.audioElementsWithListeners.get(audioElement)) {
-            MEDIA_EVENT_TYPES.forEach((errorEventType) => {
-                audioElement.addEventListener(errorEventType, this);
-                this.audioElementsWithListeners.set(audioElement, true);
-            });
-        }
-    }
-
-    private removeEventListenersForAudioElement(audioElement: HTMLMediaElement): void {
-        MEDIA_EVENT_TYPES.forEach((errorEventType) => {
-            audioElement.removeEventListener(errorEventType, this);
-        });
     }
 
     /* istanbul ignore next (remove if we start using this function for things other than debug logging) */
@@ -465,74 +424,37 @@ export default class LegacyCallHandler extends EventEmitter {
         return this.transferees.get(callId);
     }
 
-    public play(audioId: AudioID): void {
+    public async play(audioId: AudioID): Promise<void> {
         const logPrefix = `LegacyCallHandler.play(${audioId}):`;
         logger.debug(`${logPrefix} beginning of function`);
-        // TODO: Attach an invisible element for this instead
-        // which listens?
-        const audio = document.getElementById(audioId) as HTMLMediaElement;
-        if (audio) {
-            this.addEventListenersForAudioElement(audio);
-            const playAudio = async (): Promise<void> => {
-                try {
-                    if (audio.muted) {
-                        logger.error(
-                            `${logPrefix} <audio> element was unexpectedly muted but we recovered ` +
-                                `gracefully by unmuting it`,
-                        );
-                        // Recover gracefully
-                        audio.muted = false;
-                    }
 
-                    // This still causes the chrome debugger to break on promise rejection if
-                    // the promise is rejected, even though we're catching the exception.
-                    logger.debug(`${logPrefix} attempting to play audio at volume=${audio.volume}`);
-                    await audio.play();
-                    logger.debug(`${logPrefix} playing audio successfully`);
-                } catch (e) {
-                    // This is usually because the user hasn't interacted with the document,
-                    // or chrome doesn't think so and is denying the request. Not sure what
-                    // we can really do here...
-                    // https://github.com/vector-im/element-web/issues/7657
-                    logger.warn(`${logPrefix} unable to play audio clip`, e);
-                }
-            };
-            if (this.audioPromises.has(audioId)) {
-                this.audioPromises.set(
-                    audioId,
-                    this.audioPromises.get(audioId)!.then(() => {
-                        audio.load();
-                        return playAudio();
-                    }),
-                );
-            } else {
-                this.audioPromises.set(audioId, playAudio());
-            }
-        } else {
-            logger.warn(`${logPrefix} unable to find <audio> element for ${audioId}`);
-        }
+        const audioInfo: Record<AudioID, [prefix: string, loop: boolean]> = {
+            [AudioID.Ring]: [`./media/ring`, true],
+            [AudioID.Ringback]: [`./media/ringback`, true],
+            [AudioID.CallEnd]: [`./media/callend`, false],
+            [AudioID.Busy]: [`./media/busy`, false],
+        };
+
+        const [urlPrefix, loop] = audioInfo[audioId];
+        const source = await this.backgroundAudio.pickFormatAndPlay(urlPrefix, ["mp3", "ogg"], loop);
+        this.playingSources[audioId] = source;
+        logger.debug(`${logPrefix} playing audio successfully`);
     }
 
     public pause(audioId: AudioID): void {
         const logPrefix = `LegacyCallHandler.pause(${audioId}):`;
         logger.debug(`${logPrefix} beginning of function`);
-        // TODO: Attach an invisible element for this instead
-        // which listens?
-        const audio = document.getElementById(audioId) as HTMLMediaElement;
-        const pauseAudio = (): void => {
-            logger.debug(`${logPrefix} pausing audio`);
-            // pause doesn't return a promise, so just do it
-            audio.pause();
-        };
-        if (audio) {
-            if (this.audioPromises.has(audioId)) {
-                this.audioPromises.set(audioId, this.audioPromises.get(audioId)!.then(pauseAudio));
-            } else {
-                pauseAudio();
-            }
-        } else {
-            logger.warn(`${logPrefix} unable to find <audio> element for ${audioId}`);
+
+        const source = this.playingSources[audioId];
+        if (!source) {
+            logger.debug(`${logPrefix} audio not playing`);
+            return;
         }
+
+        source.stop();
+        delete this.playingSources[audioId];
+
+        logger.debug(`${logPrefix} paused audio`);
     }
 
     private matchesCallForThisRoom(call: MatrixCall): boolean {

src/Notifier.ts

@@ -58,6 +58,7 @@ import ToastStore from "./stores/ToastStore";
 import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoEventType } from "./voice-broadcast";
 import { getSenderName } from "./utils/event/getSenderName";
 import { stripPlainReply } from "./utils/Reply";
+import { BackgroundAudio } from "./audio/BackgroundAudio";
 
 /*
  * Dispatches:
@@ -112,6 +113,8 @@ class NotifierClass {
     private toolbarHidden?: boolean;
     private isSyncing?: boolean;
 
+    private backgroundAudio = new BackgroundAudio();
+
     public notificationMessageForEvent(ev: MatrixEvent): string | null {
         const msgType = ev.getContent().msgtype;
         if (msgType && msgTypeHandlers.hasOwnProperty(msgType)) {
@@ -226,28 +229,14 @@ class NotifierClass {
             return;
         }
 
+        // Play notification sound here
         const sound = this.getSoundForRoom(room.roomId);
         logger.log(`Got sound ${(sound && sound.name) || "default"} for ${room.roomId}`);
 
-        try {
-            const selector = document.querySelector<HTMLAudioElement>(
-                sound ? `audio[src='${sound.url}']` : "#messageAudio",
-            );
-            let audioElement = selector;
-            if (!audioElement) {
-                if (!sound) {
-                    logger.error("No audio element or sound to play for notification");
-                    return;
-                }
-                audioElement = new Audio(sound.url);
-                if (sound.type) {
-                    audioElement.type = sound.type;
-                }
-                document.body.appendChild(audioElement);
-            }
-            await audioElement.play();
-        } catch (ex) {
-            logger.warn("Caught error when trying to fetch room notification sound:", ex);
+        if (sound) {
+            await this.backgroundAudio.play(sound.url);
+        } else {
+            await this.backgroundAudio.pickFormatAndPlay("media/message", ["mp3", "ogg"]);
         }
     }
 

src/audio/BackgroundAudio.ts

@@ -0,0 +1,74 @@
+/*
+Copyright 2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { logger } from "matrix-js-sdk/src/logger";
+
+import { createAudioContext } from "./compat";
+
+const formatMap = {
+    mp3: "audio/mpeg",
+    ogg: "audio/ogg",
+};
+
+export class BackgroundAudio {
+    private audioContext = createAudioContext();
+    private sounds: Record<string, AudioBuffer> = {};
+
+    public async pickFormatAndPlay<F extends Array<keyof typeof formatMap>>(
+        urlPrefix: string,
+        formats: F,
+        loop = false,
+    ): Promise<AudioBufferSourceNode> {
+        const format = this.pickFormat(...formats);
+        if (!format) {
+            console.log("Browser doesn't support any of the formats", formats);
+            // Will probably never happen. If happened, format="" and will fail to load audio. Who cares...
+        }
+
+        return this.play(`${urlPrefix}.${format}`, loop);
+    }
+
+    public async play(url: string, loop = false): Promise<AudioBufferSourceNode> {
+        if (!this.sounds.hasOwnProperty(url)) {
+            // No cache, fetch it
+            const response = await fetch(url);
+            if (response.status != 200) {
+                logger.warn("Failed to fetch error audio");
+            }
+            const buffer = await response.arrayBuffer();
+            const sound = await this.audioContext.decodeAudioData(buffer);
+            this.sounds[url] = sound;
+        }
+        const source = this.audioContext.createBufferSource();
+        source.buffer = this.sounds[url];
+        source.loop = loop;
+        source.connect(this.audioContext.destination);
+        source.start();
+        return source;
+    }
+
+    private pickFormat<F extends Array<keyof typeof formatMap>>(...formats: F): F[number] | null {
+        // Detect supported formats
+        const audioElement = document.createElement("audio");
+
+        for (const format of formats) {
+            if (audioElement.canPlayType(formatMap[format])) {
+                return format;
+            }
+        }
+        return null;
+    }
+}

src/voice-broadcast/models/VoiceBroadcastRecording.ts

@@ -47,6 +47,7 @@ import { VoiceBroadcastChunkEvents } from "../utils/VoiceBroadcastChunkEvents";
 import { RelationsHelper, RelationsHelperEvent } from "../../events/RelationsHelper";
 import { createReconnectedListener } from "../../utils/connection";
 import { localNotificationsAreSilenced } from "../../utils/notifications";
+import { BackgroundAudio } from "../../audio/BackgroundAudio";
 
 export enum VoiceBroadcastRecordingEvent {
     StateChanged = "liveness_changed",
@@ -75,6 +76,7 @@ export class VoiceBroadcastRecording
     private reconnectedListener: ClientEventHandlerMap[ClientEvent.Sync];
     private roomId: string;
     private infoEventId: string;
+    private backgroundAudio = new BackgroundAudio();
 
     /**
      * Broadcast chunks have a sequence number to bring them in the correct order and to know if a message is missing.
@@ -346,15 +348,7 @@ export class VoiceBroadcastRecording
             return;
         }
 
-        // Audio files are added to the document in Element Web.
-        // See <audio> elements in https://github.com/vector-im/element-web/blob/develop/src/vector/index.html
-        const audioElement = document.querySelector<HTMLAudioElement>("audio#errorAudio");
-
-        try {
-            await audioElement?.play();
-        } catch (e) {
-            logger.warn("error playing 'errorAudio'", e);
-        }
+        await this.backgroundAudio.pickFormatAndPlay("./media/error", ["mp3", "ogg"]);
     }
 
     private async uploadFile(chunk: ChunkRecordedPayload): ReturnType<typeof uploadFile> {