Support track level stereo and red setting (#470)

* Support track level stereo and red setting

* check stereo is already exist

* move options calc to publishTrack

* prettier

* Move all logic into `publishTrack` and set explicit defaults (#474)

* Move all logic into  and set explicit defaults

* remove unnecessary null check

* fix check

* only log warning when red or dtx aren't set

* not disable red in stereo by default

* Update src/room/participant/LocalParticipant.ts

Co-authored-by: David Zhao <[email protected]>

Co-authored-by: lukasIO <[email protected]>
Co-authored-by: David Zhao <[email protected]>

Author: 3 people

.changeset/fifty-chefs-smoke.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Add stereo and red support for track level

src/proto/livekit_rtc.ts

@@ -195,6 +195,9 @@ export interface AddTrackRequest {
   simulcastCodecs: SimulcastCodec[];
   /** server ID of track, publish new codec to exist track */
   sid: string;
+  stereo: boolean;
+  /** true if RED (Redundant Encoding) is disabled for audio */
+  disableRed: boolean;
 }
 
 export interface TrickleRequest {
@@ -1078,6 +1081,8 @@ function createBaseAddTrackRequest(): AddTrackRequest {
     layers: [],
     simulcastCodecs: [],
     sid: "",
+    stereo: false,
+    disableRed: false,
   };
 }
 
@@ -1116,6 +1121,12 @@ export const AddTrackRequest = {
     if (message.sid !== "") {
       writer.uint32(90).string(message.sid);
     }
+    if (message.stereo === true) {
+      writer.uint32(96).bool(message.stereo);
+    }
+    if (message.disableRed === true) {
+      writer.uint32(104).bool(message.disableRed);
+    }
     return writer;
   },
 
@@ -1159,6 +1170,12 @@ export const AddTrackRequest = {
         case 11:
           message.sid = reader.string();
           break;
+        case 12:
+          message.stereo = reader.bool();
+          break;
+        case 13:
+          message.disableRed = reader.bool();
+          break;
         default:
           reader.skipType(tag & 7);
           break;
@@ -1182,6 +1199,8 @@ export const AddTrackRequest = {
         ? object.simulcastCodecs.map((e: any) => SimulcastCodec.fromJSON(e))
         : [],
       sid: isSet(object.sid) ? String(object.sid) : "",
+      stereo: isSet(object.stereo) ? Boolean(object.stereo) : false,
+      disableRed: isSet(object.disableRed) ? Boolean(object.disableRed) : false,
     };
   },
 
@@ -1206,6 +1225,8 @@ export const AddTrackRequest = {
       obj.simulcastCodecs = [];
     }
     message.sid !== undefined && (obj.sid = message.sid);
+    message.stereo !== undefined && (obj.stereo = message.stereo);
+    message.disableRed !== undefined && (obj.disableRed = message.disableRed);
     return obj;
   },
 
@@ -1222,6 +1243,8 @@ export const AddTrackRequest = {
     message.layers = object.layers?.map((e) => VideoLayer.fromPartial(e)) || [];
     message.simulcastCodecs = object.simulcastCodecs?.map((e) => SimulcastCodec.fromPartial(e)) || [];
     message.sid = object.sid ?? "";
+    message.stereo = object.stereo ?? false;
+    message.disableRed = object.disableRed ?? false;
     return message;
   },
 };

src/room/PCTransport.ts

@@ -22,6 +22,8 @@ export default class PCTransport {
 
   trackBitrates: TrackBitrateInfo[] = [];
 
+  remoteStereoMids: string[] = [];
+
   onOffer?: (offer: RTCSessionDescriptionInit) => void;
 
   constructor(config?: RTCConfiguration) {
@@ -40,6 +42,9 @@ export default class PCTransport {
   }
 
   async setRemoteDescription(sd: RTCSessionDescriptionInit): Promise<void> {
+    if (sd.type === 'offer') {
+      this.remoteStereoMids = extractStereoTracksFromOffer(sd);
+    }
     await this.pc.setRemoteDescription(sd);
 
     this.pendingCandidates.forEach((candidate) => {
@@ -101,7 +106,7 @@ export default class PCTransport {
     const sdpParsed = parse(offer.sdp ?? '');
     sdpParsed.media.forEach((media) => {
       if (media.type === 'audio') {
-        ensureAudioNack(media);
+        ensureAudioNackAndStereo(media, []);
       } else if (media.type === 'video') {
         // mung sdp for codec bitrate setting that can't apply by sendEncoding
         this.trackBitrates.some((trackbr): boolean => {
@@ -154,7 +159,7 @@ export default class PCTransport {
     const sdpParsed = parse(answer.sdp ?? '');
     sdpParsed.media.forEach((media) => {
       if (media.type === 'audio') {
-        ensureAudioNack(media);
+        ensureAudioNackAndStereo(media, this.remoteStereoMids);
       }
     });
     await this.setMungedLocalDescription(answer, write(sdpParsed));
@@ -203,13 +208,14 @@ export default class PCTransport {
   }
 }
 
-function ensureAudioNack(
+function ensureAudioNackAndStereo(
   media: {
     type: string;
     port: number;
     protocol: string;
     payloads?: string | undefined;
   } & MediaDescription,
+  stereoMids: string[],
 ) {
   // found opus codec to add nack fb
   let opusPayload = 0;
@@ -233,5 +239,45 @@ function ensureAudioNack(
         type: 'nack',
       });
     }
+
+    if (stereoMids.includes(media.mid!)) {
+      media.fmtp.some((fmtp): boolean => {
+        if (fmtp.payload === opusPayload) {
+          if (!fmtp.config.includes('stereo=1')) {
+            fmtp.config += ';stereo=1';
+          }
+          return true;
+        }
+        return false;
+      });
+    }
   }
 }
+
+function extractStereoTracksFromOffer(offer: RTCSessionDescriptionInit): string[] {
+  const stereoMids: string[] = [];
+  const sdpParsed = parse(offer.sdp ?? '');
+  let opusPayload = 0;
+  sdpParsed.media.forEach((media) => {
+    if (media.type === 'audio') {
+      media.rtp.some((rtp): boolean => {
+        if (rtp.codec === 'opus') {
+          opusPayload = rtp.payload;
+          return true;
+        }
+        return false;
+      });
+
+      media.fmtp.some((fmtp): boolean => {
+        if (fmtp.payload === opusPayload) {
+          if (fmtp.config.includes('sprop-stereo=1')) {
+            stereoMids.push(media.mid!);
+          }
+          return true;
+        }
+        return false;
+      });
+    }
+  });
+  return stereoMids;
+}

src/room/defaults.ts

@@ -12,12 +12,14 @@ import {
 export const publishDefaults: TrackPublishDefaults = {
   audioBitrate: AudioPresets.music.maxBitrate,
   dtx: true,
+  red: true,
+  forceStereo: false,
   simulcast: true,
   screenShareEncoding: ScreenSharePresets.h1080fps15.encoding,
   stopMicTrackOnMute: false,
   videoCodec: 'vp8',
   backupCodec: { codec: 'vp8', encoding: VideoPresets.h540.encoding },
-};
+} as const;
 
 export const audioDefaults: AudioCaptureOptions = {
   autoGainControl: true,

src/room/participant/LocalParticipant.ts

@@ -1,6 +1,6 @@
 import 'webrtc-adapter';
 import log from '../../logger';
-import type { RoomOptions } from '../../options';
+import type { InternalRoomOptions } from '../../options';
 import {
   DataPacket,
   DataPacket_Kind,
@@ -66,10 +66,10 @@ export default class LocalParticipant extends Participant {
   private allParticipantsAllowedToSubscribe: boolean = true;
 
   // keep a pointer to room options
-  private roomOptions?: RoomOptions;
+  private roomOptions: InternalRoomOptions;
 
   /** @internal */
-  constructor(sid: string, identity: string, engine: RTCEngine, options: RoomOptions) {
+  constructor(sid: string, identity: string, engine: RTCEngine, options: InternalRoomOptions) {
     super(sid, identity);
     this.audioTracks = new Map();
     this.videoTracks = new Map();
@@ -404,11 +404,6 @@ export default class LocalParticipant extends Participant {
     track: LocalTrack | MediaStreamTrack,
     options?: TrackPublishOptions,
   ): Promise<LocalTrackPublication> {
-    const opts: TrackPublishOptions = {
-      ...this.roomOptions?.publishDefaults,
-      ...options,
-    };
-
     // convert raw media track into audio or video track
     if (track instanceof MediaStreamTrack) {
       switch (track.kind) {
@@ -423,6 +418,30 @@ export default class LocalParticipant extends Participant {
       }
     }
 
+    const isStereo =
+      options?.forceStereo ||
+      ('channelCount' in track.mediaStreamTrack.getSettings() &&
+        // @ts-ignore `channelCount` on getSettings() is currently only available for Safari, but is generally the best way to determine a stereo track https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings/channelCount
+        track.mediaStreamTrack.getSettings().channelCount === 2) ||
+      track.mediaStreamTrack.getConstraints().channelCount === 2;
+
+    // disable dtx for stereo track if not enabled explicitly
+    if (isStereo) {
+      if (!options) {
+        options = {};
+      }
+      if (options.dtx === undefined) {
+        log.info(
+          `Opus DTX will be disabled for stereo tracks by default. Enable them explicitly to make it work.`,
+        );
+      }
+      options.dtx ??= false;
+    }
+    const opts: TrackPublishOptions = {
+      ...this.roomOptions.publishDefaults,
+      ...options,
+    };
+
     // is it already published? if so skip
     let existingPublication: LocalTrackPublication | undefined;
     this.tracks.forEach((publication) => {
@@ -486,7 +505,9 @@ export default class LocalParticipant extends Participant {
       type: Track.kindToProto(track.kind),
       muted: track.isMuted,
       source: Track.sourceToProto(track.source),
-      disableDtx: !(opts?.dtx ?? true),
+      disableDtx: !(opts.dtx ?? true),
+      stereo: isStereo,
+      disableRed: !(opts.red ?? true),
     });
 
     // compute encodings and layers for video

src/room/track/options.ts

@@ -32,6 +32,16 @@ export interface TrackPublishDefaults {
    */
   dtx?: boolean;
 
+  /**
+   * red (Redundant Audio Data), defaults to true
+   */
+  red?: boolean;
+
+  /**
+   * stereo audio track. defaults determined by capture channel count.
+   */
+  forceStereo?: boolean;
+
   /**
    * use simulcast, defaults to true.
    * When using simulcast, LiveKit will publish up to three versions of the stream