Add a way to intercept the audio samples before processing (#22)

* Add a way to intercept the audio samples before processing

* fix BUILD.gn

Author: kanat

sdk/android/BUILD.gn

@@ -427,6 +427,7 @@ if (is_android) {
       "src/java/org/webrtc/audio/WebRtcAudioRecord.java",
       "src/java/org/webrtc/audio/WebRtcAudioTrack.java",
       "src/java/org/webrtc/audio/WebRtcAudioUtils.java",
+      "src/java/org/webrtc/audio/AudioRecordDataCallback.java",
     ]
 
     deps = [

sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java

@@ -51,6 +51,7 @@ public static class Builder {
     private AudioAttributes audioAttributes;
     private boolean useLowLatency;
     private boolean enableVolumeLogger;
+    private AudioRecordDataCallback audioRecordDataCallback;
 
     private Builder(Context context) {
       this.context = context;
@@ -221,6 +222,16 @@ public Builder setEnableVolumeLogger(boolean enableVolumeLogger) {
       return this;
     }
 
+    /**
+     * Can be used to gain access to the raw ByteBuffer from the recording device before it's
+     * fed into WebRTC. You can use this to manipulate the ByteBuffer (e.g. audio filters).
+     * Make sure that the operation is fast.
+     */
+    public Builder setAudioRecordDataCallback(AudioRecordDataCallback audioRecordDataCallback) {
+      this.audioRecordDataCallback = audioRecordDataCallback;
+      return this;
+    }
+
     /**
      * Construct an AudioDeviceModule based on the supplied arguments. The caller takes ownership
      * and is responsible for calling release().
@@ -255,7 +266,7 @@ public JavaAudioDeviceModule createAudioDeviceModule() {
       }
       final WebRtcAudioRecord audioInput = new WebRtcAudioRecord(context, executor, audioManager,
           audioSource, audioFormat, audioRecordErrorCallback, audioRecordStateCallback,
-          samplesReadyCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor);
+          samplesReadyCallback, audioRecordDataCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor);
       final WebRtcAudioTrack audioOutput =
           new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback,
               audioTrackStateCallback, useLowLatency, enableVolumeLogger);

sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java

@@ -0,0 +1,16 @@
+package org.webrtc.audio;
+
+import androidx.annotation.NonNull;
+
+import java.nio.ByteBuffer;
+
+public interface AudioRecordDataCallback {
+  /**
+   * Invoked after an audio sample is recorded. Can be used to manipulate
+   * the ByteBuffer before it's fed into WebRTC. Currently the audio in the
+   * ByteBuffer is always PCM 16bit and the buffer sample size is ~10ms.
+   *
+   * @param audioFormat format in android.media.AudioFormat
+   */
+  void onAudioDataRecorded(int audioFormat, int channelCount, int sampleRate, @NonNull ByteBuffer audioBuffer);
+}

sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java

@@ -104,6 +104,7 @@ class WebRtcAudioRecord {
 
   private final @Nullable AudioRecordErrorCallback errorCallback;
   private final @Nullable AudioRecordStateCallback stateCallback;
+  private final @Nullable AudioRecordDataCallback audioRecordDataCallback;
   private final @Nullable SamplesReadyCallback audioSamplesReadyCallback;
   private final boolean isAcousticEchoCancelerSupported;
   private final boolean isNoiseSuppressorSupported;
@@ -153,6 +154,13 @@ public void run() {
                 captureTimeNs = audioTimestamp.nanoTime;
               }
             }
+
+            // Allow the client to intercept the ByteBuffer (to modify it)
+            if (audioRecordDataCallback != null) {
+              audioRecordDataCallback.onAudioDataRecorded(audioRecord.getAudioFormat(),
+                audioRecord.getChannelCount(), audioRecord.getSampleRate(), byteBuffer);
+            }
+
             nativeDataIsRecorded(nativeAudioRecord, bytesRead, captureTimeNs);
           }
           if (audioSamplesReadyCallback != null) {
@@ -196,7 +204,8 @@ public void stopThread() {
   WebRtcAudioRecord(Context context, AudioManager audioManager) {
     this(context, newDefaultScheduler() /* scheduler */, audioManager, DEFAULT_AUDIO_SOURCE,
         DEFAULT_AUDIO_FORMAT, null /* errorCallback */, null /* stateCallback */,
-        null /* audioSamplesReadyCallback */, WebRtcAudioEffects.isAcousticEchoCancelerSupported(),
+        null /* audioSamplesReadyCallback */, null /* audioRecordCallback */,
+        WebRtcAudioEffects.isAcousticEchoCancelerSupported(),
         WebRtcAudioEffects.isNoiseSuppressorSupported());
   }
 
@@ -205,6 +214,7 @@ public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler,
       @Nullable AudioRecordErrorCallback errorCallback,
       @Nullable AudioRecordStateCallback stateCallback,
       @Nullable SamplesReadyCallback audioSamplesReadyCallback,
+      @Nullable AudioRecordDataCallback audioRecordDataCallback,
       boolean isAcousticEchoCancelerSupported, boolean isNoiseSuppressorSupported) {
     if (isAcousticEchoCancelerSupported && !WebRtcAudioEffects.isAcousticEchoCancelerSupported()) {
       throw new IllegalArgumentException("HW AEC not supported");
@@ -220,6 +230,7 @@ public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler,
     this.errorCallback = errorCallback;
     this.stateCallback = stateCallback;
     this.audioSamplesReadyCallback = audioSamplesReadyCallback;
+    this.audioRecordDataCallback = audioRecordDataCallback;
     this.isAcousticEchoCancelerSupported = isAcousticEchoCancelerSupported;
     this.isNoiseSuppressorSupported = isNoiseSuppressorSupported;
     Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo());