diff --git a/src/Lifecycle.ts b/src/Lifecycle.ts index 64cdf445d95..5d864cc1cc7 100644 --- a/src/Lifecycle.ts +++ b/src/Lifecycle.ts @@ -60,6 +60,8 @@ import StorageEvictedDialog from "./components/views/dialogs/StorageEvictedDialo import { setSentryUser } from "./sentry"; import SdkConfig from "./SdkConfig"; import { DialogOpener } from "./utils/DialogOpener"; +import VideoChannelStore from "./stores/VideoChannelStore"; +import { fixStuckDevices } from "./utils/VideoChannelUtils"; import { Action } from "./dispatcher/actions"; import AbstractLocalStorageSettingsHandler from "./settings/handlers/AbstractLocalStorageSettingsHandler"; @@ -835,6 +837,11 @@ async function startMatrixClient(startSyncing = true): Promise { // Now that we have a MatrixClientPeg, update the Jitsi info Jitsi.getInstance().start(); + // In case we disconnected uncleanly from a video room, clean up the stuck device + if (VideoChannelStore.instance.roomId) { + fixStuckDevices(MatrixClientPeg.get().getRoom(VideoChannelStore.instance.roomId), false); + } + // dispatch that we finished starting up to wire up any other bits // of the matrix client that cannot be set prior to starting up. dis.dispatch({ action: 'client_started' }); diff --git a/src/components/structures/VideoRoomView.tsx b/src/components/structures/VideoRoomView.tsx index e2cd62e08d0..5535c5c14f8 100644 --- a/src/components/structures/VideoRoomView.tsx +++ b/src/components/structures/VideoRoomView.tsx @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, { FC, useContext, useState, useMemo } from "react"; +import React, { FC, useContext, useState, useMemo, useEffect } from "react"; import { logger } from "matrix-js-sdk/src/logger"; import { Room } from "matrix-js-sdk/src/models/room"; import MatrixClientContext from "../../contexts/MatrixClientContext"; import { useEventEmitter } from "../../hooks/useEventEmitter"; import WidgetUtils from "../../utils/WidgetUtils"; -import { addVideoChannel, getVideoChannel } from "../../utils/VideoChannelUtils"; +import { addVideoChannel, getVideoChannel, fixStuckDevices } from "../../utils/VideoChannelUtils"; import WidgetStore, { IApp } from "../../stores/WidgetStore"; import { UPDATE_EVENT } from "../../stores/AsyncStore"; import VideoChannelStore, { VideoChannelEvent } from "../../stores/VideoChannelStore"; @@ -62,6 +62,12 @@ const VideoRoomView: FC = ({ room, resizing }) => { } }, [room, widgetStoreReady, widgetLoaded]); // eslint-disable-line react-hooks/exhaustive-deps + // We'll also take this opportunity to fix any stuck devices. + // The linter thinks that store.connected should be a dependency, but we explicitly + // *only* want this to happen at mount to avoid racing with normal device updates. + // eslint-disable-next-line react-hooks/exhaustive-deps + useEffect(() => { fixStuckDevices(room, store.connected); }, [room]); + const [connected, setConnected] = useState(store.connected && store.roomId === room.roomId); useEventEmitter(store, VideoChannelEvent.Connect, () => setConnected(store.roomId === room.roomId)); useEventEmitter(store, VideoChannelEvent.Disconnect, () => setConnected(false)); diff --git a/src/settings/Settings.tsx b/src/settings/Settings.tsx index 4adbda2acb9..0df3d87481e 100644 --- a/src/settings/Settings.tsx +++ b/src/settings/Settings.tsx @@ -929,6 +929,18 @@ export const SETTINGS: {[setting: string]: ISetting} = { supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, default: false, }, + "audioInputMuted": { + supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + default: false, + }, + "videoInputMuted": { + supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + default: false, + }, + "videoChannelRoomId": { + supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + default: null, + }, [UIFeature.RoomHistorySettings]: { supportedLevels: LEVELS_UI_FEATURE, default: true, diff --git a/src/stores/VideoChannelStore.ts b/src/stores/VideoChannelStore.ts index 8ea50265c79..9ab521b50ff 100644 --- a/src/stores/VideoChannelStore.ts +++ b/src/stores/VideoChannelStore.ts @@ -18,15 +18,13 @@ import EventEmitter from "events"; import { Room, RoomEvent } from "matrix-js-sdk/src/models/room"; import { ClientWidgetApi, IWidgetApiRequest } from "matrix-widget-api"; +import SettingsStore from "../settings/SettingsStore"; +import { SettingLevel } from "../settings/SettingLevel"; import defaultDispatcher from "../dispatcher/dispatcher"; import { ActionPayload } from "../dispatcher/payloads"; import { ElementWidgetActions } from "./widgets/ElementWidgetActions"; import { WidgetMessagingStore, WidgetMessagingStoreEvent } from "./widgets/WidgetMessagingStore"; -import { - VIDEO_CHANNEL_MEMBER, - IVideoChannelMemberContent, - getVideoChannel, -} from "../utils/VideoChannelUtils"; +import { getVideoChannel, addOurDevice, removeOurDevice } from "../utils/VideoChannelUtils"; import { timeout } from "../utils/promise"; import WidgetUtils from "../utils/WidgetUtils"; import { AsyncStoreWithClient } from "./AsyncStoreWithClient"; @@ -83,9 +81,13 @@ export default class VideoChannelStore extends AsyncStoreWithClient { private activeChannel: ClientWidgetApi; - private _roomId: string; - public get roomId(): string { return this._roomId; } - private set roomId(value: string) { this._roomId = value; } + // This is persisted to settings so we can detect unclean disconnects + public get roomId(): string | null { return SettingsStore.getValue("videoChannelRoomId"); } + private set roomId(value: string | null) { + SettingsStore.setValue("videoChannelRoomId", null, SettingLevel.DEVICE, value); + } + + private get room(): Room { return this.matrixClient.getRoom(this.roomId); } private _connected = false; public get connected(): boolean { return this._connected; } @@ -95,18 +97,14 @@ export default class VideoChannelStore extends AsyncStoreWithClient { public get participants(): IJitsiParticipant[] { return this._participants; } private set participants(value: IJitsiParticipant[]) { this._participants = value; } - private _audioMuted = localStorage.getItem("mx_audioMuted") === "true"; - public get audioMuted(): boolean { return this._audioMuted; } + public get audioMuted(): boolean { return SettingsStore.getValue("audioInputMuted"); } public set audioMuted(value: boolean) { - this._audioMuted = value; - localStorage.setItem("mx_audioMuted", value.toString()); + SettingsStore.setValue("audioInputMuted", null, SettingLevel.DEVICE, value); } - private _videoMuted = localStorage.getItem("mx_videoMuted") === "true"; - public get videoMuted(): boolean { return this._videoMuted; } + public get videoMuted(): boolean { return SettingsStore.getValue("videoInputMuted"); } public set videoMuted(value: boolean) { - this._videoMuted = value; - localStorage.setItem("mx_videoMuted", value.toString()); + SettingsStore.setValue("videoInputMuted", null, SettingLevel.DEVICE, value); } public connect = async (roomId: string, audioDevice: MediaDeviceInfo, videoDevice: MediaDeviceInfo) => { @@ -198,13 +196,13 @@ export default class VideoChannelStore extends AsyncStoreWithClient { } this.connected = true; - this.matrixClient.getRoom(roomId).on(RoomEvent.MyMembership, this.onMyMembership); + this.room.on(RoomEvent.MyMembership, this.onMyMembership); window.addEventListener("beforeunload", this.setDisconnected); this.emit(VideoChannelEvent.Connect, roomId); // Tell others that we're connected, by adding our device to room state - this.updateDevices(roomId, devices => Array.from(new Set(devices).add(this.matrixClient.getDeviceId()))); + await addOurDevice(this.room); }; public disconnect = async () => { @@ -221,10 +219,11 @@ export default class VideoChannelStore extends AsyncStoreWithClient { public setDisconnected = async () => { const roomId = this.roomId; + const room = this.room; this.activeChannel.off(`action:${ElementWidgetActions.HangupCall}`, this.onHangup); this.activeChannel.off(`action:${ElementWidgetActions.CallParticipants}`, this.onParticipants); - this.matrixClient.getRoom(roomId).off(RoomEvent.MyMembership, this.onMyMembership); + room.off(RoomEvent.MyMembership, this.onMyMembership); window.removeEventListener("beforeunload", this.setDisconnected); this.activeChannel = null; @@ -235,11 +234,7 @@ export default class VideoChannelStore extends AsyncStoreWithClient { this.emit(VideoChannelEvent.Disconnect, roomId); // Tell others that we're disconnected, by removing our device from room state - await this.updateDevices(roomId, devices => { - const devicesSet = new Set(devices); - devicesSet.delete(this.matrixClient.getDeviceId()); - return Array.from(devicesSet); - }); + await removeOurDevice(room); }; private ack = (ev: CustomEvent) => { @@ -248,18 +243,6 @@ export default class VideoChannelStore extends AsyncStoreWithClient { this.activeChannel.transport.reply(ev.detail, {}); }; - private updateDevices = async (roomId: string, fn: (devices: string[]) => string[]) => { - const room = this.matrixClient.getRoom(roomId); - if (room.getMyMembership() !== "join") return; - - const devicesState = room.currentState.getStateEvents(VIDEO_CHANNEL_MEMBER, this.matrixClient.getUserId()); - const devices = devicesState?.getContent()?.devices ?? []; - - await this.matrixClient.sendStateEvent( - roomId, VIDEO_CHANNEL_MEMBER, { devices: fn(devices) }, this.matrixClient.getUserId(), - ); - }; - private onHangup = async (ev: CustomEvent) => { this.ack(ev); // In case this hangup is caused by Jitsi Meet crashing at startup, diff --git a/src/utils/VideoChannelUtils.ts b/src/utils/VideoChannelUtils.ts index cc3c99d980c..c4c757e29f1 100644 --- a/src/utils/VideoChannelUtils.ts +++ b/src/utils/VideoChannelUtils.ts @@ -16,6 +16,8 @@ limitations under the License. import { useState } from "react"; import { throttle } from "lodash"; +import { Optional } from "matrix-events-sdk"; +import { IMyDevice } from "matrix-js-sdk/src/client"; import { CallType } from "matrix-js-sdk/src/webrtc/call"; import { Room } from "matrix-js-sdk/src/models/room"; import { RoomStateEvent } from "matrix-js-sdk/src/models/room-state"; @@ -26,14 +28,16 @@ import WidgetStore, { IApp } from "../stores/WidgetStore"; import { WidgetType } from "../widgets/WidgetType"; import WidgetUtils from "./WidgetUtils"; -export const VIDEO_CHANNEL = "io.element.video"; -export const VIDEO_CHANNEL_MEMBER = "io.element.video.member"; +const STUCK_DEVICE_TIMEOUT_MS = 1000 * 60 * 60; -export interface IVideoChannelMemberContent { +interface IVideoChannelMemberContent { // Connected device IDs devices: string[]; } +export const VIDEO_CHANNEL = "io.element.video"; +export const VIDEO_CHANNEL_MEMBER = "io.element.video.member"; + export const getVideoChannel = (roomId: string): IApp => { const apps = WidgetStore.instance.getApps(roomId); return apps.find(app => WidgetType.JITSI.matches(app.type) && app.id === VIDEO_CHANNEL); @@ -72,3 +76,54 @@ export const useConnectedMembers = ( }, throttleMs, { leading: true, trailing: true })); return members; }; + +const updateDevices = async (room: Optional, fn: (devices: string[] | null) => string[]) => { + if (room?.getMyMembership() !== "join") return; + + const devicesState = room.currentState.getStateEvents(VIDEO_CHANNEL_MEMBER, room.client.getUserId()); + const devices = devicesState?.getContent()?.devices ?? []; + const newDevices = fn(devices); + + if (newDevices) { + await room.client.sendStateEvent( + room.roomId, VIDEO_CHANNEL_MEMBER, { devices: newDevices }, room.client.getUserId(), + ); + } +}; + +export const addOurDevice = async (room: Room) => { + await updateDevices(room, devices => Array.from(new Set(devices).add(room.client.getDeviceId()))); +}; + +export const removeOurDevice = async (room: Room) => { + await updateDevices(room, devices => { + const devicesSet = new Set(devices); + devicesSet.delete(room.client.getDeviceId()); + return Array.from(devicesSet); + }); +}; + +/** + * Fixes devices that may have gotten stuck in video channel member state after + * an unclean disconnection, by filtering out logged out devices, inactive + * devices, and our own device (if we're disconnected). + * @param {Room} room The room to fix + * @param {boolean} connectedLocalEcho Local echo of whether this device is connected + */ +export const fixStuckDevices = async (room: Room, connectedLocalEcho: boolean) => { + const now = new Date().valueOf(); + const { devices: myDevices } = await room.client.getDevices(); + const deviceMap = new Map(myDevices.map(d => [d.device_id, d])); + + await updateDevices(room, devices => { + const newDevices = devices.filter(d => { + const device = deviceMap.get(d); + return device?.last_seen_ts + && !(d === room.client.getDeviceId() && !connectedLocalEcho) + && (now - device.last_seen_ts) < STUCK_DEVICE_TIMEOUT_MS; + }); + + // Skip the update if the devices are unchanged + return newDevices.length === devices.length ? null : newDevices; + }); +}; diff --git a/test/components/structures/VideoRoomView-test.tsx b/test/components/structures/VideoRoomView-test.tsx index 1b936f488a4..cf73c8a1394 100644 --- a/test/components/structures/VideoRoomView-test.tsx +++ b/test/components/structures/VideoRoomView-test.tsx @@ -17,7 +17,8 @@ limitations under the License. import React from "react"; import { mount } from "enzyme"; import { act } from "react-dom/test-utils"; -import { MatrixClient } from "matrix-js-sdk/src/client"; +import { mocked } from "jest-mock"; +import { MatrixClient, IMyDevice } from "matrix-js-sdk/src/client"; import { Room } from "matrix-js-sdk/src/models/room"; import { MatrixWidgetType } from "matrix-widget-api"; @@ -27,9 +28,11 @@ import { StubVideoChannelStore, mkRoom, wrapInMatrixClientContext, + mockStateEventImplementation, + mkVideoChannelMember, } from "../../test-utils"; import { MatrixClientPeg } from "../../../src/MatrixClientPeg"; -import { VIDEO_CHANNEL } from "../../../src/utils/VideoChannelUtils"; +import { VIDEO_CHANNEL, VIDEO_CHANNEL_MEMBER } from "../../../src/utils/VideoChannelUtils"; import WidgetStore from "../../../src/stores/WidgetStore"; import _VideoRoomView from "../../../src/components/structures/VideoRoomView"; import VideoLobby from "../../../src/components/views/voip/VideoLobby"; @@ -64,6 +67,37 @@ describe("VideoRoomView", () => { room = mkRoom(cli, "!1:example.org"); }); + it("removes stuck devices on mount", async () => { + // Simulate an unclean disconnect + store.roomId = "!1:example.org"; + + const devices: IMyDevice[] = [ + { + device_id: cli.getDeviceId(), + last_seen_ts: new Date().valueOf(), + }, + { + device_id: "went offline 2 hours ago", + last_seen_ts: new Date().valueOf() - 1000 * 60 * 60 * 2, + }, + ]; + mocked(cli).getDevices.mockResolvedValue({ devices }); + + // Make both devices be stuck + mocked(room.currentState).getStateEvents.mockImplementation(mockStateEventImplementation([ + mkVideoChannelMember(cli.getUserId(), devices.map(d => d.device_id)), + ])); + + mount(); + // Wait for state to settle + await act(() => Promise.resolve()); + + // All devices should have been removed + expect(cli.sendStateEvent).toHaveBeenLastCalledWith( + "!1:example.org", VIDEO_CHANNEL_MEMBER, { devices: [] }, cli.getUserId(), + ); + }); + it("shows lobby and keeps widget loaded when disconnected", async () => { const view = mount(); // Wait for state to settle diff --git a/test/stores/VideoChannelStore-test.ts b/test/stores/VideoChannelStore-test.ts index fc8752ec76f..09508d477a4 100644 --- a/test/stores/VideoChannelStore-test.ts +++ b/test/stores/VideoChannelStore-test.ts @@ -17,7 +17,7 @@ limitations under the License. import { mocked } from "jest-mock"; import { Widget, ClientWidgetApi, MatrixWidgetType, IWidgetApiRequest } from "matrix-widget-api"; -import { stubClient, setupAsyncStoreWithClient } from "../test-utils"; +import { stubClient, setupAsyncStoreWithClient, mkRoom } from "../test-utils"; import { MatrixClientPeg } from "../../src/MatrixClientPeg"; import WidgetStore, { IApp } from "../../src/stores/WidgetStore"; import { WidgetMessagingStore } from "../../src/stores/widgets/WidgetMessagingStore"; @@ -51,6 +51,7 @@ describe("VideoChannelStore", () => { const cli = MatrixClientPeg.get(); setupAsyncStoreWithClient(WidgetMessagingStore.instance, cli); setupAsyncStoreWithClient(store, cli); + mocked(cli).getRoom.mockReturnValue(mkRoom(cli, "!1:example.org")); let resolveMessageSent: () => void; messageSent = new Promise(resolve => resolveMessageSent = resolve); diff --git a/test/test-utils/test-utils.ts b/test/test-utils/test-utils.ts index 4a3ccb8e503..6db49c731d5 100644 --- a/test/test-utils/test-utils.ts +++ b/test/test-utils/test-utils.ts @@ -79,6 +79,7 @@ export function createTestClient(): MatrixClient { getUserId: jest.fn().mockReturnValue("@userId:matrix.rog"), getUser: jest.fn().mockReturnValue({ on: jest.fn() }), getDeviceId: jest.fn().mockReturnValue("ABCDEFGHI"), + getDevices: jest.fn().mockResolvedValue({ devices: [{ device_id: "ABCDEFGHI" }] }), credentials: { userId: "@userId:matrix.rog" }, getPushActionsForEvent: jest.fn(), diff --git a/test/test-utils/video.ts b/test/test-utils/video.ts index 77fdfb8fcc0..ffc3ac2883a 100644 --- a/test/test-utils/video.ts +++ b/test/test-utils/video.ts @@ -22,24 +22,25 @@ import { VIDEO_CHANNEL_MEMBER } from "../../src/utils/VideoChannelUtils"; import VideoChannelStore, { VideoChannelEvent, IJitsiParticipant } from "../../src/stores/VideoChannelStore"; export class StubVideoChannelStore extends EventEmitter { - private _roomId: string; - public get roomId(): string { return this._roomId; } + private _roomId: string | null; + public get roomId(): string | null { return this._roomId; } + public set roomId(value: string | null) { this._roomId = value; } private _connected: boolean; public get connected(): boolean { return this._connected; } public get participants(): IJitsiParticipant[] { return []; } public startConnect = (roomId: string) => { - this._roomId = roomId; + this.roomId = roomId; this.emit(VideoChannelEvent.StartConnect, roomId); }; public connect = jest.fn((roomId: string) => { - this._roomId = roomId; + this.roomId = roomId; this._connected = true; this.emit(VideoChannelEvent.Connect, roomId); }); public disconnect = jest.fn(() => { const roomId = this._roomId; - this._roomId = null; + this.roomId = null; this._connected = false; this.emit(VideoChannelEvent.Disconnect, roomId); });