publish new codec when subscriber need (#267)

* publish new codec when subscriber need


Co-authored-by: David Zhao <[email protected]>

Author: cnderrauber

.changeset/loud-buses-swim.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Start publishing backwards-compatible codec only when subscribers need it

example/sample.ts

@@ -1,16 +1,27 @@
 import {
-  ConnectionQuality, ConnectionState, DataPacket_Kind,
-  LocalParticipant, LogLevel, MediaDeviceFailure,
+  ConnectionQuality,
+  ConnectionState,
+  DataPacket_Kind,
+  LocalParticipant,
+  LogLevel,
+  MediaDeviceFailure,
   Participant,
   ParticipantEvent,
-  RemoteParticipant, RemoteTrackPublication, RemoteVideoTrack, Room,
+  RemoteParticipant,
+  RemoteTrackPublication,
+  RemoteVideoTrack,
+  Room,
   RoomConnectOptions,
   RoomEvent,
-  RoomOptions, setLogLevel,
+  RoomOptions,
+  setLogLevel,
   Track,
   TrackPublication,
-  VideoCaptureOptions, VideoCodec, VideoPresets, VideoQuality
-} from '../src/index'
+  VideoCaptureOptions,
+  VideoCodec,
+  VideoPresets,
+  VideoQuality,
+} from '../src/index';
 
 const $ = (id: string) => document.getElementById(id);
 

src/proto/livekit_rtc.ts

@@ -209,6 +209,8 @@ export interface AddTrackRequest {
   source: TrackSource;
   layers: VideoLayer[];
   simulcastCodecs: SimulcastCodec[];
+  /** server ID of track, publish new codec to exist track */
+  sid: string;
 }
 
 export interface TrickleRequest {
@@ -993,6 +995,7 @@ function createBaseAddTrackRequest(): AddTrackRequest {
     source: 0,
     layers: [],
     simulcastCodecs: [],
+    sid: '',
   };
 }
 
@@ -1028,6 +1031,9 @@ export const AddTrackRequest = {
     for (const v of message.simulcastCodecs) {
       SimulcastCodec.encode(v!, writer.uint32(82).fork()).ldelim();
     }
+    if (message.sid !== '') {
+      writer.uint32(90).string(message.sid);
+    }
     return writer;
   },
 
@@ -1068,6 +1074,9 @@ export const AddTrackRequest = {
         case 10:
           message.simulcastCodecs.push(SimulcastCodec.decode(reader, reader.uint32()));
           break;
+        case 11:
+          message.sid = reader.string();
+          break;
         default:
           reader.skipType(tag & 7);
           break;
@@ -1092,6 +1101,7 @@ export const AddTrackRequest = {
       simulcastCodecs: Array.isArray(object?.simulcastCodecs)
         ? object.simulcastCodecs.map((e: any) => SimulcastCodec.fromJSON(e))
         : [],
+      sid: isSet(object.sid) ? String(object.sid) : '',
     };
   },
 
@@ -1117,6 +1127,7 @@ export const AddTrackRequest = {
     } else {
       obj.simulcastCodecs = [];
     }
+    message.sid !== undefined && (obj.sid = message.sid);
     return obj;
   },
 
@@ -1133,6 +1144,7 @@ export const AddTrackRequest = {
     message.layers = object.layers?.map((e) => VideoLayer.fromPartial(e)) || [];
     message.simulcastCodecs =
       object.simulcastCodecs?.map((e) => SimulcastCodec.fromPartial(e)) || [];
+    message.sid = object.sid ?? '';
     return message;
   },
 };

src/room/Room.ts

@@ -108,14 +108,6 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     this.identityToSid = new Map();
     this.options = options || {};
 
-    switch (this.options?.publishDefaults?.videoCodec) {
-      case 'av1':
-      case 'vp9':
-        this.options.publishDefaults.simulcast = undefined;
-        break;
-      default:
-    }
-
     this.options.audioCaptureDefaults = {
       ...audioDefaults,
       ...options?.audioCaptureDefaults,

src/room/participant/LocalParticipant.ts

@@ -15,10 +15,7 @@ import RTCEngine from '../RTCEngine';
 import LocalAudioTrack from '../track/LocalAudioTrack';
 import LocalTrack from '../track/LocalTrack';
 import LocalTrackPublication from '../track/LocalTrackPublication';
-import LocalVideoTrack, {
-  SimulcastTrackInfo,
-  videoLayersFromEncodings,
-} from '../track/LocalVideoTrack';
+import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTrack';
 import {
   AudioCaptureOptions,
   CreateLocalTracksOptions,
@@ -36,7 +33,7 @@ import { ParticipantTrackPermission, trackPermissionToProto } from './Participan
 import { computeVideoEncodings, mediaTrackToLocalTrack } from './publishUtils';
 import RemoteParticipant from './RemoteParticipant';
 
-const compatibleCodecForSVC = 'vp8';
+const compatibleCodec = 'vp8';
 export default class LocalParticipant extends Participant {
   audioTracks: Map<string, LocalTrackPublication>;
 
@@ -455,7 +452,6 @@ export default class LocalParticipant extends Participant {
     // compute encodings and layers for video
     let encodings: RTCRtpEncodingParameters[] | undefined;
     let simEncodings: RTCRtpEncodingParameters[] | undefined;
-    let simulcastTracks: SimulcastTrackInfo[] | undefined;
     if (track.kind === Track.Kind.Video) {
       // TODO: support react native, which doesn't expose getSettings
       const settings = track.mediaStreamTrack.getSettings();
@@ -465,37 +461,38 @@ export default class LocalParticipant extends Participant {
       req.width = width ?? 0;
       req.height = height ?? 0;
       // for svc codecs, disable simulcast and use vp8 for backup codec
-      if (
-        track instanceof LocalVideoTrack &&
-        (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1')
-      ) {
-        // set scalabilityMode to 'L3T3' by default
-        opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
-
-        // add backup codec track
-        const simOpts = { ...opts };
-        simOpts.simulcast = true;
-        simOpts.scalabilityMode = undefined;
-        simEncodings = computeVideoEncodings(
-          track.source === Track.Source.ScreenShare,
-          width,
-          height,
-          simOpts,
-        );
-        const simulcastTrack = track.addSimulcastTrack(compatibleCodecForSVC, simEncodings);
-        simulcastTracks = [simulcastTrack];
-        req.simulcastCodecs = [
-          {
-            codec: opts.videoCodec,
-            cid: track.mediaStreamTrack.id,
-            enableSimulcastLayers: true,
-          },
-          {
-            codec: simulcastTrack.codec,
-            cid: simulcastTrack.mediaStreamTrack.id,
-            enableSimulcastLayers: true,
-          },
-        ];
+      if (track instanceof LocalVideoTrack) {
+        if (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1') {
+          // set scalabilityMode to 'L3T3' by default
+          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
+
+          // add backup codec track
+          const simOpts = { ...opts };
+          simOpts.simulcast = true;
+          simOpts.scalabilityMode = undefined;
+          simEncodings = computeVideoEncodings(
+            track.source === Track.Source.ScreenShare,
+            width,
+            height,
+            simOpts,
+          );
+        }
+
+        // set vp8 codec as backup for any other codecs
+        if (opts.videoCodec && opts.videoCodec !== 'vp8') {
+          req.simulcastCodecs = [
+            {
+              codec: opts.videoCodec,
+              cid: track.mediaStreamTrack.id,
+              enableSimulcastLayers: true,
+            },
+            {
+              codec: compatibleCodec,
+              cid: '',
+              enableSimulcastLayers: true,
+            },
+          ];
+        }
       }
 
       encodings = computeVideoEncodings(
@@ -541,22 +538,6 @@ export default class LocalParticipant extends Participant {
       track.codec = opts.videoCodec;
     }
 
-    const localTrack = track as LocalVideoTrack;
-    if (simulcastTracks) {
-      for await (const simulcastTrack of simulcastTracks) {
-        const simTransceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
-        if (simulcastTrack.encodings) {
-          simTransceiverInit.sendEncodings = simulcastTrack.encodings;
-        }
-        const simTransceiver = await this.engine.publisher!.pc.addTransceiver(
-          simulcastTrack.mediaStreamTrack,
-          simTransceiverInit,
-        );
-        this.setPreferredCodec(simTransceiver, localTrack.kind, simulcastTrack.codec);
-        localTrack.setSimulcastTrackSender(simulcastTrack.codec, simTransceiver.sender);
-      }
-    }
-
     this.engine.negotiate();
 
     // store RTPSender
@@ -574,6 +555,91 @@ export default class LocalParticipant extends Participant {
     return publication;
   }
 
+  /** @internal
+   * publish additional codec to existing track
+   */
+  async publishAdditionalCodecForTrack(
+    track: LocalTrack | MediaStreamTrack,
+    videoCodec: VideoCodec,
+    options?: TrackPublishOptions,
+  ) {
+    const opts: TrackPublishOptions = {
+      ...this.roomOptions?.publishDefaults,
+      ...options,
+    };
+    // clear scalabilityMode setting for backup codec
+    opts.scalabilityMode = undefined;
+    opts.videoCodec = videoCodec;
+    // is it not published? if so skip
+    let existingPublication: LocalTrackPublication | undefined;
+    this.tracks.forEach((publication) => {
+      if (!publication.track) {
+        return;
+      }
+      if (publication.track === track) {
+        existingPublication = <LocalTrackPublication>publication;
+      }
+    });
+    if (!existingPublication) {
+      throw new TrackInvalidError('track is not published');
+    }
+
+    if (!(track instanceof LocalVideoTrack)) {
+      throw new TrackInvalidError('track is not a video track');
+    }
+
+    const settings = track.mediaStreamTrack.getSettings();
+    const width = settings.width ?? track.dimensions?.width;
+    const height = settings.height ?? track.dimensions?.height;
+
+    const encodings = computeVideoEncodings(
+      track.source === Track.Source.ScreenShare,
+      width,
+      height,
+      opts,
+    );
+    const simulcastTrack = track.addSimulcastTrack(opts.videoCodec, encodings);
+    const req = AddTrackRequest.fromPartial({
+      cid: simulcastTrack.mediaStreamTrack.id,
+      type: Track.kindToProto(track.kind),
+      muted: track.isMuted,
+      source: Track.sourceToProto(track.source),
+      sid: track.sid,
+      simulcastCodecs: [
+        {
+          codec: opts.videoCodec,
+          cid: simulcastTrack.mediaStreamTrack.id,
+          enableSimulcastLayers: opts.simulcast,
+        },
+      ],
+    });
+    req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
+
+    if (!this.engine || this.engine.isClosed) {
+      throw new UnexpectedConnectionState('cannot publish track when not connected');
+    }
+
+    const ti = await this.engine.addTrack(req);
+
+    if (!this.engine.publisher) {
+      throw new UnexpectedConnectionState('publisher is closed');
+    }
+    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+    if (encodings) {
+      transceiverInit.sendEncodings = encodings;
+    }
+    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
+    const transceiver = await this.engine.publisher.pc.addTransceiver(
+      simulcastTrack.mediaStreamTrack,
+      transceiverInit,
+    );
+    this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+    track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
+
+    this.engine.negotiate();
+    log.debug(`published ${opts.videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
+  }
+
   unpublishTrack(
     track: LocalTrack | MediaStreamTrack,
     stopOnUnpublish?: boolean,
@@ -761,7 +827,7 @@ export default class LocalParticipant extends Participant {
     this.onTrackMuted(track, track.isMuted);
   };
 
-  private handleSubscribedQualityUpdate = (update: SubscribedQualityUpdate) => {
+  private handleSubscribedQualityUpdate = async (update: SubscribedQualityUpdate) => {
     if (!this.roomOptions?.dynacast) {
       return;
     }
@@ -774,7 +840,14 @@ export default class LocalParticipant extends Participant {
       return;
     }
     if (update.subscribedCodecs.length > 0) {
-      pub.videoTrack?.setPublishingCodecs(update.subscribedCodecs);
+      if (!pub.videoTrack) {
+        return;
+      }
+      const newCodecs = await pub.videoTrack.setPublishingCodecs(update.subscribedCodecs);
+      for await (const codec of newCodecs) {
+        log.debug(`publish ${codec} for ${pub.videoTrack.sid}`);
+        await this.publishAdditionalCodecForTrack(pub.videoTrack, codec, pub.options);
+      }
     } else if (update.subscribedQualities.length > 0) {
       pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
     }