From 5834398fb08591bd5fe6d92c8f5ec25063c2e02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E7=85=9C=E6=81=92?= Date: Fri, 26 May 2023 18:58:03 +0800 Subject: [PATCH 01/19] Use WebAudio API to play notification sound So that it won't appear in system media control. --- src/Notifier.ts | 54 ++++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/Notifier.ts b/src/Notifier.ts index 96abb960ad9..90163233efb 100644 --- a/src/Notifier.ts +++ b/src/Notifier.ts @@ -106,6 +106,10 @@ class NotifierClass { private toolbarHidden?: boolean; private isSyncing?: boolean; + // Cache the sounds loaded + private audioContext: AudioContext = new AudioContext(); + private sounds: Record = {}; + public notificationMessageForEvent(ev: MatrixEvent): string | null { const msgType = ev.getContent().msgtype; if (msgType && msgTypeHandlers.hasOwnProperty(msgType)) { @@ -213,6 +217,30 @@ class NotifierClass { }; } + private async createAudioSourceForRoom(roomId: string): Promise { + const audio=await this.getSoundBufferForRoom(roomId); + const source=this.audioContext.createBufferSource(); + source.buffer=audio; + source.connect(this.audioContext.destination); + return source; + } + + private async getSoundBufferForRoom(roomId: string): Promise { + const sound = this.getSoundForRoom(roomId); + // TODO: Choose from mp3 and ogg + const url=sound?sound["url"]:"media/message.mp3"; + + if(this.sounds.hasOwnProperty(url)){ + return this.sounds[url]; + } + + const response=await fetch(url); + const buffer=await response.arrayBuffer(); + const audio=await this.audioContext.decodeAudioData(buffer); + this.sounds[url]=audio; + return audio; + } + // XXX: Exported for tests public async playAudioNotification(ev: MatrixEvent, room: Room): Promise { const cli = MatrixClientPeg.get(); @@ -220,29 +248,9 @@ class NotifierClass { return; } - const sound = this.getSoundForRoom(room.roomId); - logger.log(`Got sound ${(sound && sound.name) || "default"} for ${room.roomId}`); - - try { - const selector = document.querySelector( - sound ? `audio[src='${sound.url}']` : "#messageAudio", - ); - let audioElement = selector; - if (!audioElement) { - if (!sound) { - logger.error("No audio element or sound to play for notification"); - return; - } - audioElement = new Audio(sound.url); - if (sound.type) { - audioElement.type = sound.type; - } - document.body.appendChild(audioElement); - } - await audioElement.play(); - } catch (ex) { - logger.warn("Caught error when trying to fetch room notification sound:", ex); - } + // Play notification sound here + const source=await this.createAudioSourceForRoom(room.roomId); + source.start(); } public start(): void { From 249cfb7fb75483e501f14fd14e5bef331228b8d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E7=85=9C=E6=81=92?= Date: Fri, 26 May 2023 19:06:16 +0800 Subject: [PATCH 02/19] Run prettier --- src/Notifier.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Notifier.ts b/src/Notifier.ts index 90163233efb..54d7e13fcdb 100644 --- a/src/Notifier.ts +++ b/src/Notifier.ts @@ -218,9 +218,9 @@ class NotifierClass { } private async createAudioSourceForRoom(roomId: string): Promise { - const audio=await this.getSoundBufferForRoom(roomId); - const source=this.audioContext.createBufferSource(); - source.buffer=audio; + const audio = await this.getSoundBufferForRoom(roomId); + const source = this.audioContext.createBufferSource(); + source.buffer = audio; source.connect(this.audioContext.destination); return source; } @@ -228,16 +228,16 @@ class NotifierClass { private async getSoundBufferForRoom(roomId: string): Promise { const sound = this.getSoundForRoom(roomId); // TODO: Choose from mp3 and ogg - const url=sound?sound["url"]:"media/message.mp3"; + const url = sound ? sound["url"] : "media/message.mp3"; - if(this.sounds.hasOwnProperty(url)){ + if (this.sounds.hasOwnProperty(url)) { return this.sounds[url]; } - const response=await fetch(url); - const buffer=await response.arrayBuffer(); - const audio=await this.audioContext.decodeAudioData(buffer); - this.sounds[url]=audio; + const response = await fetch(url); + const buffer = await response.arrayBuffer(); + const audio = await this.audioContext.decodeAudioData(buffer); + this.sounds[url] = audio; return audio; } @@ -249,7 +249,7 @@ class NotifierClass { } // Play notification sound here - const source=await this.createAudioSourceForRoom(room.roomId); + const source = await this.createAudioSourceForRoom(room.roomId); source.start(); } From e19a8bf4f7f08406ecdbd618fdbb5efa93f3b2ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E7=85=9C=E6=81=92?= Date: Tue, 30 May 2023 18:46:23 +0800 Subject: [PATCH 03/19] Chosse from mp3 and ogg --- src/Notifier.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Notifier.ts b/src/Notifier.ts index 54d7e13fcdb..c648bc8d8c8 100644 --- a/src/Notifier.ts +++ b/src/Notifier.ts @@ -227,8 +227,19 @@ class NotifierClass { private async getSoundBufferForRoom(roomId: string): Promise { const sound = this.getSoundForRoom(roomId); - // TODO: Choose from mp3 and ogg - const url = sound ? sound["url"] : "media/message.mp3"; + + const audioElement = document.createElement("audio"); + let format=""; + if(audioElement.canPlayType("audio/mpeg")){ + format="mp3"; + }else if(audioElement.canPlayType("audio/ogg")){ + format="ogg"; + }else{ + console.log("Browser doens't support mp3 or ogg"); + // Will probably never happen. If happened, format="" and will fail to load audio. Who cares... + } + + const url = sound ? sound["url"] : "media/message."+format; if (this.sounds.hasOwnProperty(url)) { return this.sounds[url]; From 9e588e51a40a165cfcc3635b41839f9b4b8eedb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E7=85=9C=E6=81=92?= Date: Sat, 17 Jun 2023 12:19:02 +0800 Subject: [PATCH 04/19] Run prettier --- src/Notifier.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Notifier.ts b/src/Notifier.ts index c648bc8d8c8..1f4a3b53ad6 100644 --- a/src/Notifier.ts +++ b/src/Notifier.ts @@ -229,17 +229,17 @@ class NotifierClass { const sound = this.getSoundForRoom(roomId); const audioElement = document.createElement("audio"); - let format=""; - if(audioElement.canPlayType("audio/mpeg")){ - format="mp3"; - }else if(audioElement.canPlayType("audio/ogg")){ - format="ogg"; - }else{ + let format = ""; + if (audioElement.canPlayType("audio/mpeg")) { + format = "mp3"; + } else if (audioElement.canPlayType("audio/ogg")) { + format = "ogg"; + } else { console.log("Browser doens't support mp3 or ogg"); // Will probably never happen. If happened, format="" and will fail to load audio. Who cares... } - const url = sound ? sound["url"] : "media/message."+format; + const url = sound ? sound["url"] : "media/message." + format; if (this.sounds.hasOwnProperty(url)) { return this.sounds[url]; From 495dd5fb458bc34e7a84529d79bc9860cce4e978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E7=85=9C=E6=81=92?= Date: Tue, 27 Jun 2023 12:25:10 +0800 Subject: [PATCH 05/19] Use WebAudioAPI everywhere There's still one remoteAudio. I'm not sure what it does. It seems it's only used in tests... --- src/LegacyCallHandler.tsx | 170 +++++++----------- src/Notifier.ts | 7 +- .../models/VoiceBroadcastRecording.ts | 34 +++- 3 files changed, 100 insertions(+), 111 deletions(-) diff --git a/src/LegacyCallHandler.tsx b/src/LegacyCallHandler.tsx index 1414fc2893c..f73f3ac6b10 100644 --- a/src/LegacyCallHandler.tsx +++ b/src/LegacyCallHandler.tsx @@ -159,8 +159,6 @@ export default class LegacyCallHandler extends EventEmitter { // Calls started as an attended transfer, ie. with the intention of transferring another // call with a different party to this one. private transferees = new Map(); // callId (target) -> call (transferee) - private audioPromises = new Map>(); - private audioElementsWithListeners = new Map(); private supportsPstnProtocol: boolean | null = null; private pstnSupportPrefixed: boolean | null = null; // True if the server only support the prefixed pstn protocol private supportsSipNativeVirtual: boolean | null = null; // im.vector.protocol.sip_virtual and im.vector.protocol.sip_native @@ -172,6 +170,10 @@ export default class LegacyCallHandler extends EventEmitter { private silencedCalls = new Set(); // callIds + private audioContext: AudioContext = new AudioContext(); + private sounds: Record = {}; // Cache the sounds loaded + private playingSources: Record = {}; // Record them for stopping + public static get instance(): LegacyCallHandler { if (!window.mxLegacyCallHandler) { window.mxLegacyCallHandler = new LegacyCallHandler(); @@ -201,33 +203,11 @@ export default class LegacyCallHandler extends EventEmitter { } public start(): void { - // add empty handlers for media actions, otherwise the media keys - // end up causing the audio elements with our ring/ringback etc - // audio clips in to play. - if (navigator.mediaSession) { - navigator.mediaSession.setActionHandler("play", function () {}); - navigator.mediaSession.setActionHandler("pause", function () {}); - navigator.mediaSession.setActionHandler("seekbackward", function () {}); - navigator.mediaSession.setActionHandler("seekforward", function () {}); - navigator.mediaSession.setActionHandler("previoustrack", function () {}); - navigator.mediaSession.setActionHandler("nexttrack", function () {}); - } - if (SettingsStore.getValue(UIFeature.Voip)) { MatrixClientPeg.safeGet().on(CallEventHandlerEvent.Incoming, this.onCallIncoming); } this.checkProtocols(CHECK_PROTOCOLS_ATTEMPTS); - - // Add event listeners for the