Audio Device Optimization (#29)

* Audio Device Optimization

allow listen-only mode in AudioUnit, adjust when category changes (#2)

release mic when category changes (#5)

Change defaults to iOS defaults (#7)

Sync audio session config (#8)

feat: support bypass voice processing for iOS. (#15)

Remove MacBookPro audio pan right code (#22)

fix: Fix can't open mic alone when built-in AEC is enabled. (#29)

feat: add audio device changes detect for windows. (#41)

fix Linux compile (#47)

AudioUnit: Don't rely on category switch for mic indicator to turn off (#52)

Stop recording on mute (turn off mic indicator) (#55)

Cherry pick audio selection from m97 release (#35)

[Mac] Allow audio device selection (#21)

RTCAudioDeviceModule.outputDevice / inputDevice getter and setter (#80)

Co-authored-by: Hiroshi Horie <[email protected]>
Co-authored-by: David Zhao <[email protected]>

* fix compilation errors

---------

Co-authored-by: CloudWebRTC <[email protected]>
Co-authored-by: Hiroshi Horie <[email protected]>
Co-authored-by: David Zhao <[email protected]>

# Conflicts:
#	audio/audio_state.cc
#	call/audio_state.h
#	media/engine/webrtc_voice_engine.h
#	modules/audio_device/audio_device_impl.cc
#	modules/audio_device/include/audio_device.h
#	modules/audio_device/mac/audio_device_mac.cc
#	modules/audio_device/mac/audio_device_mac.h
#	sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h
#	sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h
#	sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm
#	sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm
#	sdk/objc/components/audio/RTCAudioSessionConfiguration.m
#	sdk/objc/native/src/audio/audio_device_ios.mm
#	sdk/objc/native/src/audio/audio_device_module_ios.mm
#	sdk/objc/native/src/audio/voice_processing_audio_unit.mm

Author: ipavlidakis

api/audio/audio_device.h

@@ -22,6 +22,15 @@ namespace webrtc {
 
 class AudioDeviceModuleForTest;
 
+// Sink for callbacks related to a audio device.
+class AudioDeviceSink {
+ public:
+  virtual ~AudioDeviceSink() = default;
+
+  // input/output devices updated or default device changed
+  virtual void OnDevicesUpdated() = 0;
+};
+
 class AudioDeviceModule : public webrtc::RefCountInterface {
  public:
   enum AudioLayer {
@@ -62,11 +71,13 @@ class AudioDeviceModule : public webrtc::RefCountInterface {
  public:
   // Creates a default ADM for usage in production code.
   static scoped_refptr<AudioDeviceModule> Create(
-      AudioLayer audio_layer, TaskQueueFactory* task_queue_factory);
+      AudioLayer audio_layer, TaskQueueFactory* task_queue_factory,
+      bool bypass_voice_processing = false);
   // Creates an ADM with support for extra test methods. Don't use this factory
   // in production code.
   static scoped_refptr<AudioDeviceModuleForTest> CreateForTest(
-      AudioLayer audio_layer, TaskQueueFactory* task_queue_factory);
+      AudioLayer audio_layer, TaskQueueFactory* task_queue_factory,
+      bool bypass_voice_processing = false);
 
   // Retrieve the currently utilized audio layer
   virtual int32_t ActiveAudioLayer(AudioLayer* audioLayer) const = 0;
@@ -176,6 +187,10 @@ class AudioDeviceModule : public webrtc::RefCountInterface {
   virtual int GetRecordAudioParameters(AudioParameters* params) const = 0;
 #endif  // WEBRTC_IOS
 
+  virtual int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const { return -1; }
+  virtual int32_t GetPlayoutDevice() const { return -1; }
+  virtual int32_t GetRecordingDevice() const { return -1; }
+
  protected:
   ~AudioDeviceModule() override {}
 };

audio/audio_send_stream.cc

@@ -400,6 +400,11 @@ void AudioSendStream::SetMuted(bool muted) {
   channel_send_->SetInputMute(muted);
 }
 
+bool AudioSendStream::GetMuted() {
+  RTC_DCHECK_RUN_ON(&worker_thread_checker_);
+  return channel_send_->InputMute();
+}
+
 webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
   return GetStats(true);
 }

audio/audio_send_stream.h

@@ -88,6 +88,7 @@ class AudioSendStream final : public webrtc::AudioSendStream,
                           int payload_frequency,
                           int event,
                           int duration_ms) override;
+  bool GetMuted() override;
   void SetMuted(bool muted) override;
   webrtc::AudioSendStream::Stats GetStats() const override;
   webrtc::AudioSendStream::Stats GetStats(

audio/audio_state.cc

@@ -71,6 +71,25 @@ void AudioState::SetPlayout(bool enabled) {
   UpdateNullAudioPollerState();
 }
 
+void AudioState::SetRecording(bool enabled) {
+  RTC_LOG(LS_INFO) << "SetRecording(" << enabled << ")";
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  auto* adm = config_.audio_device_module.get();
+  if (enabled) {
+    if (ShouldRecord()) {
+      if (!adm->Recording()) {
+        if (adm->InitRecording() == 0) {
+          adm->StartRecording();
+        }
+      }
+    }
+  } else {
+    // Disable recording.
+    adm->StopRecording();
+  }
+  recording_enabled_ = enabled;
+}
+
 void AudioState::AddReceivingStream(
     webrtc::AudioReceiveStreamInterface* stream) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
@@ -106,25 +125,6 @@ void AudioState::RemoveReceivingStream(
   UpdateNullAudioPollerState();
 }
 
-void AudioState::SetRecording(bool enabled) {
-  RTC_LOG(LS_INFO) << "SetRecording(" << enabled << ")";
-  RTC_DCHECK_RUN_ON(&thread_checker_);
-  auto* adm = config_.audio_device_module.get();
-  if (enabled) {
-    if (!sending_streams_.empty()) {
-      if (!adm->Recording()) {
-        if (adm->InitRecording() == 0) {
-          adm->StartRecording();
-        }
-      }
-    }
-  } else {
-    // Disable recording.
-    adm->StopRecording();
-  }
-  recording_enabled_ = enabled;
-}
-
 void AudioState::AddSendingStream(webrtc::AudioSendStream* stream,
                                   int sample_rate_hz,
                                   size_t num_channels) {
@@ -135,13 +135,13 @@ void AudioState::AddSendingStream(webrtc::AudioSendStream* stream,
   UpdateAudioTransportWithSendingStreams();
 
   // Make sure recording is initialized; start recording if enabled.
-  auto* adm = config_.audio_device_module.get();
-  if (recording_enabled_) {
-    if (!adm->Recording()) {
-      if (adm->InitRecording() == 0) {
-        adm->StartRecording();
-      } else {
-        RTC_DLOG_F(LS_ERROR) << "Failed to initialize recording.";
+  if (ShouldRecord()) {
+    auto* adm = config_.audio_device_module.get();
+    if (recording_enabled_) {
+      if (!adm->Recording()) {
+        if (adm->InitRecording() == 0) {
+          adm->StartRecording();
+        }
       }
     }
   }
@@ -152,7 +152,8 @@ void AudioState::RemoveSendingStream(webrtc::AudioSendStream* stream) {
   auto count = sending_streams_.erase(stream);
   RTC_DCHECK_EQ(1, count);
   UpdateAudioTransportWithSendingStreams();
-  if (sending_streams_.empty()) {
+
+  if (!ShouldRecord()) {
     config_.audio_device_module->StopRecording();
   }
 }
@@ -208,6 +209,39 @@ void AudioState::UpdateNullAudioPollerState() {
     null_audio_poller_.Stop();
   }
 }
+
+void AudioState::OnMuteStreamChanged() {
+
+  auto* adm = config_.audio_device_module.get();
+  bool should_record = ShouldRecord();
+
+  if (should_record && !adm->Recording()) {
+    if (adm->InitRecording() == 0) {
+      adm->StartRecording();
+    }
+  } else if (!should_record && adm->Recording()) {
+    adm->StopRecording();
+  }
+}
+
+bool AudioState::ShouldRecord() {
+  // no streams to send
+  if (sending_streams_.empty()) {
+    return false;
+  }
+
+  int stream_count = sending_streams_.size();
+
+  int muted_count = 0;
+  for (const auto& kv : sending_streams_) {
+    if (kv.first->GetMuted()) {
+      muted_count++;
+    }
+  }
+
+  return muted_count != stream_count;
+}
+
 }  // namespace internal
 
 scoped_refptr<AudioState> AudioState::Create(const AudioState::Config& config) {

audio/audio_state.h

@@ -47,6 +47,8 @@ class AudioState : public webrtc::AudioState {
 
   void SetStereoChannelSwapping(bool enable) override;
 
+  void OnMuteStreamChanged() override;
+
   AudioDeviceModule* audio_device_module() {
     RTC_DCHECK(config_.audio_device_module);
     return config_.audio_device_module.get();
@@ -64,6 +66,9 @@ class AudioState : public webrtc::AudioState {
   void UpdateAudioTransportWithSendingStreams();
   void UpdateNullAudioPollerState() RTC_RUN_ON(&thread_checker_);
 
+  // Returns true when at least 1 stream exists and all streams are not muted.
+  bool ShouldRecord();
+
   SequenceChecker thread_checker_;
   SequenceChecker process_thread_checker_{SequenceChecker::kDetached};
   const webrtc::AudioState::Config config_;

audio/channel_send.cc

@@ -235,7 +235,7 @@ class ChannelSend : public ChannelSendInterface,
                    size_t payloadSize,
                    int64_t absolute_capture_timestamp_ms) override;
 
-  bool InputMute() const;
+  bool InputMute() const override;
 
   int32_t SendRtpAudio(AudioFrameType frameType,
                        uint8_t payloadType,

audio/channel_send.h

@@ -89,6 +89,8 @@ class ChannelSendInterface {
   virtual bool SendTelephoneEventOutband(int event, int duration_ms) = 0;
   virtual void OnBitrateAllocation(BitrateAllocationUpdate update) = 0;
   virtual int GetTargetBitrate() const = 0;
+
+  virtual bool InputMute() const = 0;
   virtual void SetInputMute(bool muted) = 0;
 
   virtual void ProcessAndEncodeAudio(

audio/mock_voe_channel_proxy.h

@@ -173,6 +173,7 @@ class MockChannelSend : public voe::ChannelSendInterface {
               OnBitrateAllocation,
               (BitrateAllocationUpdate update),
               (override));
+  MOCK_METHOD(bool, InputMute, (), (const, override));
   MOCK_METHOD(void, SetInputMute, (bool muted), (override));
   MOCK_METHOD(void,
               ReceivedRTCPPacket,

call/audio_send_stream.h

@@ -193,6 +193,7 @@ class AudioSendStream : public AudioSender {
                                   int event,
                                   int duration_ms) = 0;
 
+  virtual bool GetMuted() = 0;
   virtual void SetMuted(bool muted) = 0;
 
   virtual Stats GetStats() const = 0;

call/audio_state.h

@@ -59,6 +59,9 @@ class AudioState : public RefCountInterface {
   virtual void SetStereoChannelSwapping(bool enable) = 0;
 
   static scoped_refptr<AudioState> Create(const AudioState::Config& config);
+  
+  // Notify the AudioState that a stream updated it's mute state.
+  virtual void OnMuteStreamChanged() = 0;
 
   ~AudioState() override {}
 };