Skip to content

Commit bc057cf

Browse files
kanatkanat
authored
Support for simulcast in iOS SDK (#3)
Co-authored-by: kanat <>
1 parent 151be74 commit bc057cf

File tree

10 files changed

+216
-17
lines changed

10 files changed

+216
-17
lines changed

‎sdk/BUILD.gn‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -230,9 +230,6 @@ if (is_ios || is_mac) {
230230
"objc/native/api/audio_device_module.h",
231231
"objc/native/api/audio_device_module.mm",
232232
]
233-
if (is_mac) {
234-
frameworks = [ "AudioUnit.framework" ]
235-
}
236233

237234
deps = [
238235
":audio_device",
@@ -293,9 +290,6 @@ if (is_ios || is_mac) {
293290
absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ]
294291

295292
frameworks = [ "AudioToolbox.framework" ]
296-
if (is_mac) {
297-
frameworks += [ "AudioUnit.framework" ]
298-
}
299293
}
300294

301295
# This target exists to expose :audio_session_objc and
@@ -436,9 +430,6 @@ if (is_ios || is_mac) {
436430
"../rtc_base:threading",
437431
"../rtc_base:timeutils",
438432
]
439-
if (is_mac) {
440-
frameworks = [ "AudioUnit.framework" ]
441-
}
442433
}
443434

444435
rtc_library("objc_audio_device_module") {
@@ -456,9 +447,6 @@ if (is_ios || is_mac) {
456447
"../modules/audio_device:audio_device_api",
457448
"../rtc_base:logging",
458449
]
459-
if (is_mac) {
460-
frameworks = [ "AudioUnit.framework" ]
461-
}
462450
}
463451

464452
rtc_library("videosource_objc") {
@@ -741,6 +729,7 @@ if (is_ios || is_mac) {
741729
]
742730

743731
deps = [
732+
":simulcast",
744733
":base_objc",
745734
":native_video",
746735
":videocodec_objc",
@@ -792,6 +781,22 @@ if (is_ios || is_mac) {
792781
]
793782
}
794783

784+
rtc_library("simulcast") {
785+
sources = [
786+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
787+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm",
788+
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
789+
"objc/api/video_codec/RTCVideoEncoderSimulcast.mm",
790+
]
791+
792+
deps = [
793+
":base_objc",
794+
":wrapped_native_codec_objc",
795+
"../media:rtc_media_base",
796+
"../media:rtc_simulcast_encoder_adapter",
797+
]
798+
}
799+
795800
rtc_library("vp9") {
796801
visibility = [ "*" ]
797802
allow_poison = [ "software_video_codecs" ]
@@ -1304,6 +1309,7 @@ if (is_ios || is_mac) {
13041309
"objc/components/video_codec/RTCVideoDecoderH264.h",
13051310
"objc/components/video_codec/RTCVideoEncoderFactoryH264.h",
13061311
"objc/components/video_codec/RTCVideoEncoderH264.h",
1312+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
13071313
"objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
13081314
"objc/helpers/RTCCameraPreviewView.h",
13091315
"objc/helpers/RTCDispatcher.h",
@@ -1351,6 +1357,7 @@ if (is_ios || is_mac) {
13511357
"objc/api/video_codec/RTCVideoEncoderVP8.h",
13521358
"objc/api/video_codec/RTCVideoEncoderVP9.h",
13531359
"objc/api/video_codec/RTCVideoEncoderAV1.h",
1360+
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
13541361
"objc/api/video_frame_buffer/RTCNativeI420Buffer.h",
13551362
"objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h",
13561363
]
@@ -1605,7 +1612,6 @@ if (is_ios || is_mac) {
16051612
"../api/video:video_rtp_headers",
16061613
"../api/video_codecs:video_codecs_api",
16071614
"../common_video",
1608-
"../pc:video_track_source_proxy",
16091615
"../rtc_base:buffer",
16101616
"../rtc_base:logging",
16111617
"../rtc_base:ssl",

‎sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ RTC_OBJC_EXPORT
6969
https://w3c.github.io/webrtc-extensions/#dom-rtcrtpencodingparameters-adaptiveptime */
7070
@property(nonatomic, assign) BOOL adaptiveAudioPacketTime;
7171

72+
@property(nonatomic, copy, nullable) NSString *scalabilityMode;
73+
7274
- (instancetype)init;
7375

7476
@end

‎sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ @implementation RTC_OBJC_TYPE (RTCRtpEncodingParameters)
2525
@synthesize bitratePriority = _bitratePriority;
2626
@synthesize networkPriority = _networkPriority;
2727
@synthesize adaptiveAudioPacketTime = _adaptiveAudioPacketTime;
28+
@synthesize scalabilityMode = _scalabilityMode;
2829

2930
- (instancetype)init {
3031
webrtc::RtpEncodingParameters nativeParameters;
@@ -59,6 +60,9 @@ - (instancetype)initWithNativeParameters:
5960
if (nativeParameters.ssrc) {
6061
_ssrc = [NSNumber numberWithUnsignedLong:*nativeParameters.ssrc];
6162
}
63+
if (nativeParameters.scalability_mode.has_value()) {
64+
_scalabilityMode = [NSString stringForStdString:nativeParameters.scalability_mode.value()];
65+
}
6266
_bitratePriority = nativeParameters.bitrate_priority;
6367
_networkPriority = [RTC_OBJC_TYPE(RTCRtpEncodingParameters)
6468
priorityFromNativePriority:nativeParameters.network_priority];
@@ -96,6 +100,9 @@ - (instancetype)initWithNativeParameters:
96100
parameters.network_priority =
97101
[RTC_OBJC_TYPE(RTCRtpEncodingParameters) nativePriorityFromPriority:_networkPriority];
98102
parameters.adaptive_ptime = _adaptiveAudioPacketTime;
103+
if (_scalabilityMode != nil) {
104+
parameters.scalability_mode = [NSString stdStringForString:_scalabilityMode];
105+
}
99106
return parameters;
100107
}
101108

‎sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,24 @@
1515
@implementation RTC_OBJC_TYPE (RTCVideoCodecInfo)
1616
(Private)
1717

18-
- (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format {
18+
- (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format {
1919
NSMutableDictionary *params = [NSMutableDictionary dictionary];
2020
for (auto it = format.parameters.begin(); it != format.parameters.end(); ++it) {
2121
[params setObject:[NSString stringForStdString:it->second]
2222
forKey:[NSString stringForStdString:it->first]];
2323
}
24-
return [self initWithName:[NSString stringForStdString:format.name] parameters:params];
24+
25+
NSMutableArray *scalabilityModes = [[NSMutableArray alloc] init];
26+
if (!format.scalability_modes.empty()) {
27+
for (const auto scalability_mode : format.scalability_modes) {
28+
uint8_t value = static_cast<uint8_t>(scalability_mode);
29+
[scalabilityModes addObject: [NSNumber numberWithUnsignedInt:value]];
30+
}
31+
}
32+
33+
return [self initWithName:[NSString stringForStdString:format.name]
34+
parameters:params
35+
scalabilityModes:scalabilityModes];
2536
}
2637

2738
- (webrtc::SdpVideoFormat)nativeSdpVideoFormat {
@@ -32,7 +43,13 @@ - (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format {
3243
parameters[key] = value;
3344
}
3445

35-
return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters);
46+
absl::InlinedVector<webrtc::ScalabilityMode, webrtc::kScalabilityModeCount> scalabilityModes;
47+
for (NSNumber *scalabilityMode in self.scalabilityModes) {
48+
unsigned int value = [scalabilityMode unsignedIntValue];
49+
scalabilityModes.push_back(static_cast<webrtc::ScalabilityMode>(value));
50+
}
51+
52+
return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters, scalabilityModes);
3653
}
3754

3855
@end

‎sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#import "RTCMacros.h"
2+
#import "RTCVideoEncoder.h"
3+
#import "RTCVideoEncoderFactory.h"
4+
#import "RTCVideoCodecInfo.h"
5+
6+
RTC_OBJC_EXPORT
7+
@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) : NSObject
8+
9+
+ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
10+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
11+
videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo;
12+
13+
@end

‎sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#import <Foundation/Foundation.h>
2+
3+
#import "RTCMacros.h"
4+
#import "RTCVideoEncoderSimulcast.h"
5+
#import "RTCWrappedNativeVideoEncoder.h"
6+
#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
7+
8+
#include "native/api/video_encoder_factory.h"
9+
#include "media/engine/simulcast_encoder_adapter.h"
10+
11+
@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcast)
12+
13+
+ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
14+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
15+
videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo {
16+
auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(primary);
17+
auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(fallback);
18+
auto nativeFormat = [videoCodecInfo nativeSdpVideoFormat];
19+
return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc]
20+
initWithNativeEncoder: std::make_unique<webrtc::SimulcastEncoderAdapter>(
21+
nativePrimary.release(),
22+
nativeFallback.release(),
23+
std::move(nativeFormat))];
24+
}
25+
26+
@end

‎sdk/objc/base/RTCVideoCodecInfo.h‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,43 @@
1414

1515
NS_ASSUME_NONNULL_BEGIN
1616

17+
typedef NS_ENUM(uint8_t, RTCScalabilityMode) {
18+
RTCScalabilityModeL1T1,
19+
RTCScalabilityModeL1T2,
20+
RTCScalabilityModeL1T3,
21+
RTCScalabilityModeL2T1,
22+
RTCScalabilityModeL2T1h,
23+
RTCScalabilityModeL2T1_KEY,
24+
RTCScalabilityModeL2T2,
25+
RTCScalabilityModeL2T2h,
26+
RTCScalabilityModeL2T2_KEY,
27+
RTCScalabilityModeL2T2_KEY_SHIFT,
28+
RTCScalabilityModeL2T3,
29+
RTCScalabilityModeL2T3h,
30+
RTCScalabilityModeL2T3_KEY,
31+
RTCScalabilityModeL3T1,
32+
RTCScalabilityModeL3T1h,
33+
RTCScalabilityModeL3T1_KEY,
34+
RTCScalabilityModeL3T2,
35+
RTCScalabilityModeL3T2h,
36+
RTCScalabilityModeL3T2_KEY,
37+
RTCScalabilityModeL3T3,
38+
RTCScalabilityModeL3T3h,
39+
RTCScalabilityModeL3T3_KEY,
40+
RTCScalabilityModeS2T1,
41+
RTCScalabilityModeS2T1h,
42+
RTCScalabilityModeS2T2,
43+
RTCScalabilityModeS2T2h,
44+
RTCScalabilityModeS2T3,
45+
RTCScalabilityModeS2T3h,
46+
RTCScalabilityModeS3T1,
47+
RTCScalabilityModeS3T1h,
48+
RTCScalabilityModeS3T2,
49+
RTCScalabilityModeS3T2h,
50+
RTCScalabilityModeS3T3,
51+
RTCScalabilityModeS3T3h,
52+
};
53+
1754
/** Holds information to identify a codec. Corresponds to webrtc::SdpVideoFormat. */
1855
RTC_OBJC_EXPORT
1956
@interface RTC_OBJC_TYPE (RTCVideoCodecInfo) : NSObject <NSCoding>
@@ -22,14 +59,18 @@ RTC_OBJC_EXPORT
2259

2360
- (instancetype)initWithName:(NSString *)name;
2461

62+
- (instancetype)initWithName:(NSString *)name
63+
parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters;
2564
- (instancetype)initWithName:(NSString *)name
2665
parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters
66+
scalabilityModes:(nullable NSArray<NSNumber *> *)scalabilityModes
2767
NS_DESIGNATED_INITIALIZER;
2868

2969
- (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info;
3070

3171
@property(nonatomic, readonly) NSString *name;
3272
@property(nonatomic, readonly) NSDictionary<NSString *, NSString *> *parameters;
73+
@property(nonatomic, readonly) NSArray<NSNumber *> *scalabilityModes;
3374

3475
@end
3576

‎sdk/objc/base/RTCVideoCodecInfo.m‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,29 @@ @implementation RTC_OBJC_TYPE (RTCVideoCodecInfo)
1414

1515
@synthesize name = _name;
1616
@synthesize parameters = _parameters;
17+
@synthesize scalabilityModes = _scalabilityModes;
1718

1819
- (instancetype)initWithName:(NSString *)name {
1920
return [self initWithName:name parameters:nil];
2021
}
2122

2223
- (instancetype)initWithName:(NSString *)name
2324
parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters {
25+
return [self initWithName:name parameters:parameters scalabilityModes:nil];
26+
}
27+
28+
- (instancetype)initWithName:(NSString *)name
29+
parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters
30+
scalabilityModes:(nullable NSArray<NSNumber *> *)scalabilityModes {
2431
if (self = [super init]) {
2532
_name = name;
2633
_parameters = (parameters ? parameters : @{});
34+
_scalabilityModes = (scalabilityModes ? scalabilityModes : @[]);
2735
}
28-
2936
return self;
3037
}
3138

39+
3240
- (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info {
3341
if (!info ||
3442
![self.name isEqualToString:info.name] ||

‎sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#import <Foundation/Foundation.h>
2+
3+
#import "RTCMacros.h"
4+
#import "RTCVideoEncoderFactory.h"
5+
6+
NS_ASSUME_NONNULL_BEGIN
7+
8+
RTC_OBJC_EXPORT
9+
@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoderFactory)>
10+
11+
- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
12+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback;
13+
14+
@end
15+
16+
NS_ASSUME_NONNULL_END

‎sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#import <Foundation/Foundation.h>
2+
3+
#import "RTCMacros.h"
4+
#import "RTCVideoCodecInfo.h"
5+
#import "RTCVideoEncoderFactorySimulcast.h"
6+
#import "api/video_codec/RTCVideoEncoderSimulcast.h"
7+
#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
8+
9+
#include "absl/container/inlined_vector.h"
10+
#include "api/video_codecs/video_codec.h"
11+
#include "api/video_codecs/sdp_video_format.h"
12+
#include "api/video_codecs/video_codec.h"
13+
#include "modules/video_coding/codecs/av1/av1_svc_config.h"
14+
#include "modules/video_coding/codecs/vp9/include/vp9.h"
15+
#include "media/base/media_constants.h"
16+
17+
@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) ()
18+
19+
@property id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> primary;
20+
@property id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> fallback;
21+
22+
@end
23+
24+
25+
@implementation RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast)
26+
27+
@synthesize primary = _primary;
28+
@synthesize fallback = _fallback;
29+
30+
- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
31+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback {
32+
if (self = [super init]) {
33+
_primary = primary;
34+
_fallback = fallback;
35+
}
36+
return self;
37+
}
38+
39+
- (nullable id<RTC_OBJC_TYPE(RTCVideoEncoder)>)createEncoder: (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info {
40+
return [RTCVideoEncoderSimulcast simulcastEncoderWithPrimary: _primary fallback: _fallback videoCodecInfo: info];
41+
}
42+
43+
- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs {
44+
NSArray *supportedCodecs = [[_primary supportedCodecs] arrayByAddingObjectsFromArray: [_fallback supportedCodecs]];
45+
46+
NSMutableArray<RTCVideoCodecInfo *> *addingCodecs = [[NSMutableArray alloc] init];
47+
48+
for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs(true)) {
49+
RTCVideoCodecInfo *codec = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat: format];
50+
[addingCodecs addObject: codec];
51+
}
52+
53+
auto av1Format = webrtc::SdpVideoFormat(
54+
cricket::kAv1CodecName, webrtc::SdpVideoFormat::Parameters(),
55+
webrtc::LibaomAv1EncoderSupportedScalabilityModes());
56+
RTCVideoCodecInfo *av1Codec = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat: av1Format];
57+
[addingCodecs addObject: av1Codec];
58+
59+
return [supportedCodecs arrayByAddingObjectsFromArray: addingCodecs];
60+
}
61+
62+
63+
@end

0 commit comments

Comments
 (0)