Add sticky event support (#3513)

* add sticky event support
 - use new js-sdk
 - use custom synapse
 - don't filter rooms by existing call state events

Signed-off-by: Timo K <[email protected]>

* enable sticky events in the joinSessionConfig

Signed-off-by: Timo K <[email protected]>

* Remove unused useNewMembershipmanager setting

* Add prefer sticky setting]

* Fixup call detection logic to allow sticky events

* lint

* update docker image

* More tidy

* update checksum

* bump js-sdk fix sticky events type

Signed-off-by: Timo K <[email protected]>

* fix demo

Signed-off-by: Timo K <[email protected]>

* always use multi sfu if we are using sticky events.

Signed-off-by: Timo K <[email protected]>

* review

Signed-off-by: Timo K <[email protected]>

* lint

Signed-off-by: Timo K <[email protected]>

* Always consider multi-SFU mode enabled when using sticky events

CallViewModel would pass the wrong transport to enterRtcSession when the user enabled sticky events but didn't manually enable multi-SFU mode as well. This likely would've added some confusion to our attempts to test these modes.

* Fix test type errors

* add todo comment

Signed-off-by: Timo K <[email protected]>

---------

Signed-off-by: Timo K <[email protected]>
Co-authored-by: Half-Shot <[email protected]>
Co-authored-by: Robin <[email protected]>

Author: 3 people

backend/dev_homeserver.yaml

@@ -38,6 +38,8 @@ experimental_features:
   # MSC4222 needed for syncv2 state_after. This allow clients to
   # correctly track the state of the room.
   msc4222_enabled: true
+  # sticky events for matrixRTC user state
+  msc4354_enabled: true
 
 # The maximum allowed duration by which sent events can be delayed, as
 # per MSC4140. Must be a positive value if set.  Defaults to no

dev-backend-docker-compose.yml

@@ -88,7 +88,7 @@ services:
 
   synapse:
     hostname: homeserver
-    image: docker.io/matrixdotorg/synapse:latest
+    image: ghcr.io/element-hq/synapse:msc4354-5
     pull_policy: always
     environment:
       - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml

locales/en/app.json

@@ -74,9 +74,12 @@
     "matrix_id": "Matrix ID: {{id}}",
     "multi_sfu": "Multi-SFU media transport",
     "mute_all_audio": "Mute all audio (participants, reactions, join sounds)",
+    "prefer_sticky_events": {
+      "description": "Improves reliability of calls (requires homeserver support)",
+      "label": "Prefer sticky events"
+    },
     "show_connection_stats": "Show connection statistics",
     "url_params": "URL parameters",
-    "use_new_membership_manager": "Use the new implementation of the call MembershipManager",
     "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key"
   },
   "disconnected_banner": "Connectivity to the server has been lost.",

package.json

@@ -109,7 +109,7 @@
     "livekit-client": "^2.13.0",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21",
     "matrix-widget-api": "^1.13.0",
     "normalize.css": "^8.0.1",
     "observable-hooks": "^4.2.3",

src/home/useGroupCallRooms.ts

@@ -20,7 +20,6 @@ import {
   MatrixRTCSessionManagerEvents,
   type MatrixRTCSession,
 } from "matrix-js-sdk/lib/matrixrtc";
-import { logger } from "matrix-js-sdk/lib/logger";
 
 import { getKeyForRoom } from "../e2ee/sharedKeyManagement";
 
@@ -114,19 +113,49 @@ const roomIsJoinable = (room: Room): boolean => {
   }
 };
 
+/**
+ * Determines if a given room has call events in it, and therefore
+ * is likely to be a call room.
+ * @param room The Matrix room instance.
+ * @returns `true` if the room has call events.
+ */
 const roomHasCallMembershipEvents = (room: Room): boolean => {
-  switch (room.getMyMembership()) {
-    case KnownMembership.Join:
-      return !!room
-        .getLiveTimeline()
-        .getState(EventTimeline.FORWARDS)
-        ?.events?.get(EventType.GroupCallMemberPrefix);
-    case KnownMembership.Knock:
-      // Assume that a room you've knocked on is able to hold calls
+  // Check our room membership first, to rule out any rooms
+  // we can't have a call in.
+  const myMembership = room.getMyMembership();
+  if (myMembership === KnownMembership.Knock) {
+    // Assume that a room you've knocked on is able to hold calls
+    return true;
+  } else if (myMembership !== KnownMembership.Join) {
+    // Otherwise, non-joined rooms should never show up.
+    return false;
+  }
+
+  // Legacy member state checks (cheaper to check.)
+  const timeline = room.getLiveTimeline();
+  if (
+    timeline
+      .getState(EventTimeline.FORWARDS)
+      ?.events?.has(EventType.GroupCallMemberPrefix)
+  ) {
+    return true;
+  }
+
+  // Check for *active* calls using sticky events.
+  for (const sticky of room._unstable_getStickyEvents()) {
+    if (sticky.getType() === EventType.RTCMembership) {
       return true;
-    default:
-      return false;
+    }
   }
+
+  // Otherwise, check recent event history to see if anyone had
+  // sent a call membership in here.
+  return timeline.getEvents().some(
+    (e) =>
+      // Membership events only count if both of these are true
+      e.unstableStickyInfo && e.getType() === EventType.GroupCallMemberPrefix,
+  );
+  // Otherwise, it's *unlikely* this room was ever a call.
 };
 
 export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] {
@@ -140,24 +169,22 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] {
         .filter(roomHasCallMembershipEvents)
         .filter(roomIsJoinable);
       const sortedRooms = sortRooms(client, rooms);
-      Promise.all(
-        sortedRooms.map((room) => {
-          const session = client.matrixRTC.getRoomSession(room);
-          return {
-            roomAlias: room.getCanonicalAlias() ?? undefined,
-            roomName: room.name,
-            avatarUrl: room.getMxcAvatarUrl()!,
-            room,
-            session,
-            participants: session.memberships
-              .filter((m) => m.userId)
-              .map((m) => room.getMember(m.userId!))
-              .filter((m) => m) as RoomMember[],
-          };
-        }),
-      )
-        .then((items) => setRooms(items))
-        .catch(logger.error);
+      const items = sortedRooms.map((room) => {
+        const session = client.matrixRTC.getRoomSession(room);
+        return {
+          roomAlias: room.getCanonicalAlias() ?? undefined,
+          roomName: room.name,
+          avatarUrl: room.getMxcAvatarUrl()!,
+          room,
+          session,
+          participants: session.memberships
+            .filter((m) => m.sender)
+            .map((m) => room.getMember(m.sender!))
+            .filter((m) => m) as RoomMember[],
+        };
+      });
+
+      setRooms(items);
     }
 
     updateRooms();

src/room/GroupCallView.tsx

@@ -70,10 +70,6 @@ import {
   UnknownCallError,
 } from "../utils/errors.ts";
 import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx";
-import {
-  useNewMembershipManager as useNewMembershipManagerSetting,
-  useSetting,
-} from "../settings/settings";
 import { useTypedEventEmitter } from "../useEvents";
 import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
 import { useAppBarTitle } from "../AppBar.tsx";
@@ -186,7 +182,6 @@ export const GroupCallView: FC<Props> = ({
     password: passwordFromUrl,
   } = useUrlParams();
   const e2eeSystem = useRoomEncryptionSystem(room.roomId);
-  const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting);
 
   // Save the password once we start the groupCallView
   useEffect(() => {
@@ -310,7 +305,6 @@ export const GroupCallView: FC<Props> = ({
     mediaDevices,
     latestMuteStates,
     setJoined,
-    useNewMembershipManager,
   ]);
 
   // TODO refactor this + "joined" to just one callState

src/rtcSessionHelpers.test.ts

@@ -96,6 +96,7 @@ test("It joins the correct Session", async () => {
     {
       encryptMedia: true,
       useMultiSfu: USE_MUTI_SFU,
+      preferStickyEvents: false,
     },
   );
 
@@ -111,7 +112,6 @@ test("It joins the correct Session", async () => {
     expect.objectContaining({
       manageMediaKeys: true,
       useLegacyMemberEvents: false,
-      useNewMembershipManager: true,
       useExperimentalToDeviceTransport: false,
     }),
   );
@@ -197,6 +197,7 @@ test("It should not fail with configuration error if homeserver config has livek
     {
       encryptMedia: true,
       useMultiSfu: USE_MUTI_SFU,
+      preferStickyEvents: false,
     },
   );
 });

src/rtcSessionHelpers.ts

@@ -99,12 +99,11 @@ export async function makeTransport(
 
 export interface EnterRTCSessionOptions {
   encryptMedia: boolean;
-  // TODO: remove this flag, the new membership manager is stable enough
-  useNewMembershipManager?: boolean;
   // TODO: remove this flag, to-device transport is stable enough now
   useExperimentalToDeviceTransport?: boolean;
   /** EXPERIMENTAL: If true, will use the multi-sfu codepath where each member connects to its SFU instead of everyone connecting to an elected on. */
-  useMultiSfu?: boolean;
+  useMultiSfu: boolean;
+  preferStickyEvents: boolean;
 }
 
 /**
@@ -116,20 +115,13 @@ export interface EnterRTCSessionOptions {
 export async function enterRTCSession(
   rtcSession: MatrixRTCSession,
   transport: LivekitTransport,
-  options: EnterRTCSessionOptions = {
-    encryptMedia: true,
-    useNewMembershipManager: true,
-    useExperimentalToDeviceTransport: false,
-    useMultiSfu: true,
-  },
-): Promise<void> {
-  const {
+  {
     encryptMedia,
-    useNewMembershipManager = true,
     useExperimentalToDeviceTransport = false,
-    useMultiSfu = true,
-  } = options;
-
+    useMultiSfu,
+    preferStickyEvents,
+  }: EnterRTCSessionOptions,
+): Promise<void> {
   PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
   PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
 
@@ -148,7 +140,6 @@ export async function enterRTCSession(
     {
       notificationType,
       callIntent,
-      useNewMembershipManager,
       manageMediaKeys: encryptMedia,
       ...(useDeviceSessionMemberEvents !== undefined && {
         useLegacyMemberEvents: !useDeviceSessionMemberEvents,
@@ -164,6 +155,7 @@ export async function enterRTCSession(
       membershipEventExpiryMs:
         matrixRtcSessionConfig?.membership_event_expiry_ms,
       useExperimentalToDeviceTransport,
+      unstableSendStickyEvents: preferStickyEvents,
     },
   );
   if (widget) {

src/settings/DeveloperSettingsTab.tsx

@@ -5,25 +5,37 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type ChangeEvent, type FC, useCallback, useMemo } from "react";
+import {
+  type ChangeEvent,
+  type FC,
+  useCallback,
+  useEffect,
+  useMemo,
+  useState,
+} from "react";
 import { useTranslation } from "react-i18next";
+import {
+  UNSTABLE_MSC4354_STICKY_EVENTS,
+  type MatrixClient,
+} from "matrix-js-sdk";
+import { logger } from "matrix-js-sdk/lib/logger";
 
 import { FieldRow, InputField } from "../input/Input";
 import {
   useSetting,
   duplicateTiles as duplicateTilesSetting,
   debugTileLayout as debugTileLayoutSetting,
   showConnectionStats as showConnectionStatsSetting,
-  useNewMembershipManager as useNewMembershipManagerSetting,
   useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
   multiSfu as multiSfuSetting,
   muteAllAudio as muteAllAudioSetting,
   alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
+  preferStickyEvents as preferStickyEventsSetting,
 } from "./settings";
-import type { MatrixClient } from "matrix-js-sdk";
 import type { Room as LivekitRoom } from "livekit-client";
 import styles from "./DeveloperSettingsTab.module.css";
 import { useUrlParams } from "../UrlParams";
+
 interface Props {
   client: MatrixClient;
   livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[];
@@ -36,12 +48,24 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
     debugTileLayoutSetting,
   );
 
-  const [showConnectionStats, setShowConnectionStats] = useSetting(
-    showConnectionStatsSetting,
+  const [stickyEventsSupported, setStickyEventsSupported] = useState(false);
+  useEffect(() => {
+    client
+      .doesServerSupportUnstableFeature(UNSTABLE_MSC4354_STICKY_EVENTS)
+      .then((result) => {
+        setStickyEventsSupported(result);
+      })
+      .catch((ex) => {
+        logger.warn("Failed to check if sticky events are supported", ex);
+      });
+  }, [client]);
+
+  const [preferStickyEvents, setPreferStickyEvents] = useSetting(
+    preferStickyEventsSetting,
   );
 
-  const [useNewMembershipManager, setNewMembershipManager] = useSetting(
-    useNewMembershipManagerSetting,
+  const [showConnectionStats, setShowConnectionStats] = useSetting(
+    showConnectionStatsSetting,
   );
 
   const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting(
@@ -128,29 +152,31 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
       </FieldRow>
       <FieldRow>
         <InputField
-          id="showConnectionStats"
+          id="preferStickyEvents"
           type="checkbox"
-          label={t("developer_mode.show_connection_stats")}
-          checked={!!showConnectionStats}
+          label={t("developer_mode.prefer_sticky_events.label")}
+          disabled={!stickyEventsSupported}
+          description={t("developer_mode.prefer_sticky_events.description")}
+          checked={!!preferStickyEvents}
           onChange={useCallback(
             (event: ChangeEvent<HTMLInputElement>): void => {
-              setShowConnectionStats(event.target.checked);
+              setPreferStickyEvents(event.target.checked);
             },
-            [setShowConnectionStats],
+            [setPreferStickyEvents],
           )}
         />
       </FieldRow>
       <FieldRow>
         <InputField
-          id="useNewMembershipManager"
+          id="showConnectionStats"
           type="checkbox"
-          label={t("developer_mode.use_new_membership_manager")}
-          checked={!!useNewMembershipManager}
+          label={t("developer_mode.show_connection_stats")}
+          checked={!!showConnectionStats}
           onChange={useCallback(
             (event: ChangeEvent<HTMLInputElement>): void => {
-              setNewMembershipManager(event.target.checked);
+              setShowConnectionStats(event.target.checked);
             },
-            [setNewMembershipManager],
+            [setShowConnectionStats],
           )}
         />
       </FieldRow>
@@ -173,7 +199,9 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
           id="multiSfu"
           type="checkbox"
           label={t("developer_mode.multi_sfu")}
-          checked={multiSfu}
+          // If using sticky events we implicitly prefer use multi-sfu
+          checked={multiSfu || preferStickyEvents}
+          disabled={preferStickyEvents}
           onChange={useCallback(
             (event: ChangeEvent<HTMLInputElement>): void => {
               setMultiSfu(event.target.checked);