Allow custom audio processing by exposing AudioProcessingModule (#85)

* apm

* progress

* expose raw buffer

* config

* runtime delegate update

* always create audio processing adapter

* delegate nullable

* dynamic delegate update thread safety

* fix delegate life cycle when swapped

* refactor

* avoid crash when no apm is passed in

* format

* make delegate weak & docs

* fix memory issue

Author: hiroshihorie

sdk/BUILD.gn

@@ -209,6 +209,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto
   NSParameterAssert(type == RTCMediaStreamTrackTypeAudio);
   if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) {
     RTC_LOG(LS_INFO) << "RTCAudioTrack init";
+    _lock = OS_UNFAIR_LOCK_INIT;
     _renderers = [NSHashTable weakObjectsHashTable];
     _audioConverter = new rtc::RefCountedObject<webrtc::AudioSinkConverter>(self, &_lock);
   }

sdk/objc/api/peerconnection/RTCAudioTrack.mm

@@ -73,12 +73,6 @@ NS_ASSUME_NONNULL_BEGIN
     initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
             decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
 
-- (instancetype)
-    initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
-                   encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
-
 /** Initialize an RTCPeerConnection with a configuration, constraints, and
  *  dependencies.
  */

sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h

@@ -38,6 +38,8 @@ typedef NS_ENUM(NSInteger, RTCRtpMediaType);
 (RTCSSLCertificateVerifier);
 @protocol RTC_OBJC_TYPE
 (RTCAudioDevice);
+@protocol RTC_OBJC_TYPE
+(RTCAudioProcessingModule);
 
 RTC_OBJC_EXPORT
 @interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) : NSObject
@@ -60,8 +62,9 @@ RTC_OBJC_EXPORT
 - (instancetype)
     initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
                    encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
+                   decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioProcessingModule:
+                (nullable id<RTC_OBJC_TYPE(RTCAudioProcessingModule)>)audioProcessingModule;
 
 @property(nonatomic, readonly) RTCAudioDeviceModule *audioDeviceModule;
 

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h

@@ -53,6 +53,9 @@
 #include "sdk/objc/native/src/objc_video_decoder_factory.h"
 #include "sdk/objc/native/src/objc_video_encoder_factory.h"
 
+#import "components/audio/RTCAudioProcessingModule.h"
+#import "components/audio/RTCDefaultAudioProcessingModule+Private.h"
+
 #if defined(WEBRTC_IOS)
 #import "sdk/objc/native/api/audio_device_module.h"
 #endif
@@ -62,6 +65,7 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) {
   std::unique_ptr<rtc::Thread> _workerThread;
   std::unique_ptr<rtc::Thread> _signalingThread;
   rtc::scoped_refptr<webrtc::AudioDeviceModule> _nativeAudioDeviceModule;
+  RTCDefaultAudioProcessingModule *_defaultAudioProcessingModule;
 
   BOOL _hasStartedAecDump;
 }
@@ -144,8 +148,9 @@ - (instancetype)init {
 - (instancetype)
     initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
                    encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory {
+                   decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioProcessingModule:
+                (nullable id<RTC_OBJC_TYPE(RTCAudioProcessingModule)>)audioProcessingModule {
 #ifdef HAVE_NO_MEDIA
   return [self initWithNoMedia];
 #else
@@ -158,12 +163,21 @@ - (instancetype)init {
     native_decoder_factory = webrtc::ObjCToNativeVideoDecoderFactory(decoderFactory);
   }
   rtc::scoped_refptr<webrtc::AudioDeviceModule> audio_device_module = [self createAudioDeviceModule:bypassVoiceProcessing];
+
+  if ([audioProcessingModule isKindOfClass:[RTCDefaultAudioProcessingModule class]]) {
+    _defaultAudioProcessingModule = (RTCDefaultAudioProcessingModule *)audioProcessingModule;
+  } else {
+    _defaultAudioProcessingModule = [[RTCDefaultAudioProcessingModule alloc] init];
+  }
+
+  NSLog(@"AudioProcessingModule: %@", _defaultAudioProcessingModule);
+  
   return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory()
                        nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory()
                        nativeVideoEncoderFactory:std::move(native_encoder_factory)
                        nativeVideoDecoderFactory:std::move(native_decoder_factory)
                                audioDeviceModule:audio_device_module.get()
-                           audioProcessingModule:nullptr
+                           audioProcessingModule:_defaultAudioProcessingModule.nativeAudioProcessingModule
                            bypassVoiceProcessing:bypassVoiceProcessing];
 #endif
 }

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioBuffer.h"
+
+#include "modules/audio_processing/audio_buffer.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTC_OBJC_TYPE (RTCAudioBuffer)()
+
+- (instancetype)initWithNativeType: (webrtc::AudioBuffer *) audioBuffer;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioBuffer+Private.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCAudioBuffer) : NSObject
+
+@property(nonatomic, readonly) size_t channels;
+@property(nonatomic, readonly) size_t frames;
+@property(nonatomic, readonly) size_t framesPerBand;
+@property(nonatomic, readonly) size_t bands;
+
+// Returns pointer arrays. Index range from 0 to `frames`.
+- (float* _Nonnull)rawBufferForChannel:(size_t)channel;
+
+// TODO: More convenience methods...
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioBuffer.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioBuffer.h"
+
+#include "modules/audio_processing/audio_buffer.h"
+
+@implementation RTC_OBJC_TYPE (RTCAudioBuffer) {
+  // Raw
+  webrtc::AudioBuffer *_audioBuffer;
+}
+
+- (size_t)channels {
+  return _audioBuffer->num_channels();
+}
+
+- (size_t)frames {
+  return _audioBuffer->num_frames();
+}
+
+- (size_t)framesPerBand {
+  return _audioBuffer->num_frames_per_band();
+}
+
+- (size_t)bands {
+  return _audioBuffer->num_bands();
+}
+
+- (float *)rawBufferForChannel:(size_t)channel {
+  return _audioBuffer->channels()[channel];
+}
+
+#pragma mark - Private
+
+- (instancetype)initWithNativeType:(webrtc::AudioBuffer *)audioBuffer {
+  if (self = [super init]) {
+    _audioBuffer = audioBuffer;
+  }
+  return self;
+}
+
+@end

sdk/objc/components/audio/RTCAudioBuffer.mm

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioCustomProcessingAdapter.h"
+#import "RTCAudioCustomProcessingDelegate.h"
+#import "RTCMacros.h"
+
+#include "modules/audio_processing/include/audio_processing.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTCAudioCustomProcessingAdapter ()
+
+// Thread safe set/get with os_unfair_lock.
+@property(nonatomic, weak, nullable) id<RTCAudioCustomProcessingDelegate>
+    audioCustomProcessingDelegate;
+
+// Direct read access without lock.
+@property(nonatomic, readonly, weak, nullable) id<RTCAudioCustomProcessingDelegate>
+    rawAudioCustomProcessingDelegate;
+
+@property(nonatomic, readonly) std::unique_ptr<webrtc::CustomProcessing>
+    nativeAudioCustomProcessingModule;
+
+- (instancetype)initWithDelegate:
+    (nullable id<RTCAudioCustomProcessingDelegate>)audioCustomProcessingDelegate;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+#import "RTCMacros.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTCAudioCustomProcessingAdapter : NSObject
+
+- (instancetype)init NS_UNAVAILABLE;
+
+@end
+
+NS_ASSUME_NONNULL_END