feat: Implement internal audio/video recording for devices >= Android Q

Signed-off-by: sirmordred <[email protected]>

Author: sirmordred

README.md

@@ -321,6 +321,36 @@ and add them to the media library if their MIME types are supported. If the
 file/folder in _path_ does not exist/is not readable or is not provided then an
 error will be returned and the corresponding log message would be written into logs.
 
+## Internal Audio & Video Recording
+
+Required steps to activate recording:
+
+```bash
+adb shell pm grant io.appium.settings android.permission.RECORD_AUDIO
+adb shell appops set io.appium.settings PROJECT_MEDIA allow
+```
+
+Start Recording:
+```bash
+adb shell am start -n "io.appium.settings/io.appium.settings.Settings" -a io.appium.settings.recording.ACTION_START --es filename abc.mp4 --es priority high --es max_duration_sec 900 --es resolution 1920x1080
+```
+
+### Arguments (see above start command as an example for giving arguments)
+- filename (Mandatory) - You can type recording video file name as you want, but recording currently supports only "mp4" format so your filename must end with ".mp4"
+- priority (Optional) - Default value: "high" which means recording thread priority is maximum however if you face performance drops during testing with recording enabled, you can reduce recording priority to "normal" or "low"
+- max_duration_sec (Optional) (in seconds) - Default value: 900 seconds which means maximum allowed duration is 15 minute, you can increase it if your test takes longer than that
+- resolution (Optional) - Default value: maximum supported resolution on-device(Detected automatically on app itself), which usually equals to Full HD 1920x1080 on most phones however you can change it to following supported resolutions as well: "1920x1080", "1280x720", "720x480", "320x240", "176x144"
+
+Stop Recording:
+```bash
+adb shell am start -n "io.appium.settings/io.appium.settings.Settings" -a io.appium.settings.recording.ACTION_STOP
+```
+
+Obtain Recording Output File:
+```bash
+adb pull /storage/emulated/0/Android/data/io.appium.settings/files/abc.mp4 abc.mp4
+```
+
 
 ## Notes:
 

app/src/main/AndroidManifest.xml

@@ -28,6 +28,8 @@
     <uses-permission android:name="android.permission.BLUETOOTH_ADMIN" />
     <uses-permission android:name="android.permission.READ_SMS" />
 
+    <uses-permission android:name="android.permission.RECORD_AUDIO"/>
+
     <uses-feature android:name="android.hardware.wifi" />
 
     <application
@@ -66,6 +68,12 @@
             android:exported="true">
         </service>
 
+        <service
+            android:foregroundServiceType="mediaProjection"
+            android:name=".recorder.RecorderService"
+            android:exported="true">
+        </service>
+
         <service
             android:name=".AppiumIME"
             android:label="Appium IME"

app/src/main/java/io/appium/settings/Settings.java

@@ -18,12 +18,17 @@
 
 import android.app.Activity;
 import android.content.BroadcastReceiver;
+import android.content.Context;
+import android.content.Intent;
 import android.content.IntentFilter;
+import android.media.projection.MediaProjectionManager;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Handler;
 import android.util.Log;
 
+import java.io.File;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.List;
 
@@ -39,10 +44,34 @@
 import io.appium.settings.receivers.SmsReader;
 import io.appium.settings.receivers.UnpairBluetoothDevicesReceiver;
 import io.appium.settings.receivers.WiFiConnectionSettingReceiver;
+import io.appium.settings.recorder.RecorderService;
+import io.appium.settings.recorder.RecorderUtil;
+
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_BASE;
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_FILENAME;
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_MAX_DURATION;
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_PRIORITY;
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_RESOLUTION;
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_RESULT_CODE;
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_ROTATION;
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_START;
+import static io.appium.settings.recorder.RecorderConstant.ACTION_RECORDING_STOP;
+import static io.appium.settings.recorder.RecorderConstant.NO_PATH_SET;
+import static io.appium.settings.recorder.RecorderConstant.NO_RESOLUTION_MODE_SET;
+import static io.appium.settings.recorder.RecorderConstant.RECORDING_MAX_DURATION_DEFAULT_MS;
+import static io.appium.settings.recorder.RecorderConstant.RECORDING_PRIORITY_DEFAULT;
+import static io.appium.settings.recorder.RecorderConstant.RECORDING_ROTATION_DEFAULT_DEGREE;
+import static io.appium.settings.recorder.RecorderConstant.REQUEST_CODE_SCREEN_CAPTURE;
 
 public class Settings extends Activity {
     private static final String TAG = "APPIUM SETTINGS";
 
+    private String recordingOutputPath = NO_PATH_SET;
+    private int recordingRotation = RECORDING_ROTATION_DEFAULT_DEGREE;
+    private int recordingPriority = RECORDING_PRIORITY_DEFAULT;
+    private int recordingMaxDuration = RECORDING_MAX_DURATION_DEFAULT_MS;
+    private String recordingResolutionMode = NO_RESOLUTION_MODE_SET;
+
     @Override
     public void onCreate(Bundle savedInstanceState) {
         super.onCreate(savedInstanceState);
@@ -70,12 +99,152 @@ public void onCreate(Bundle savedInstanceState) {
             LocationTracker.getInstance().start(this);
         }
 
+        handleRecording(getIntent());
+    }
+
+    private void handleRecording(Intent intent) {
+        if (intent == null) {
+            Log.e(TAG, "handleRecording: Unable to retrieve intent instance");
+            finishActivity();
+            return;
+        }
+
+        String recordingAction = intent.getAction();
+        if (recordingAction == null) {
+            Log.e(TAG, "handleRecording: Unable to retrieve intent.action instance");
+            finishActivity();
+            return;
+        }
+
+        if (!recordingAction.startsWith(ACTION_RECORDING_BASE)) {
+            Log.i(TAG, "handleRecording: Received different intent with action: "
+                    + recordingAction);
+            finishActivity();
+            return;
+        }
+
+        if (RecorderUtil.isLowerThanQ()) {
+            Log.e(TAG, "handleRecording: Current Android OS Version is lower than Q");
+            finishActivity();
+            return;
+        }
+
+        if (!RecorderUtil.areRecordingPermissionsGranted(getApplicationContext())) {
+            Log.e(TAG, "handleRecording: Required permissions are not granted");
+            finishActivity();
+            return;
+        }
+
+        if (recordingAction.equals(ACTION_RECORDING_START)) {
+            String recordingFilename = intent.getStringExtra(ACTION_RECORDING_FILENAME);
+            if (!RecorderUtil.isValidFileName(recordingFilename)) {
+                Log.e(TAG, "handleRecording: Invalid filename passed by user: "
+                        + recordingFilename);
+                finishActivity();
+                return;
+            }
+
+            /*
+             External Storage File Directory for app
+             (i.e /storage/emulated/0/Android/data/io.appium.settings/files) may not be created
+             so we need to call getExternalFilesDir() method twice
+             source:https://www.androidbugfix.com/2021/10/getexternalfilesdirnull-returns-null-in.html
+            */
+            File externalStorageFile = getExternalFilesDir(null);
+            if (externalStorageFile == null) {
+                externalStorageFile = getExternalFilesDir(null);
+            }
+            // if path is still null despite calling method twice, early exit
+            if (externalStorageFile == null) {
+                Log.e(TAG, "handleRecording: Unable to retrieve external storage file path");
+                finishActivity();
+                return;
+            }
+
+            recordingOutputPath = Paths
+                    .get(externalStorageFile.getAbsolutePath(), recordingFilename)
+                    .toAbsolutePath()
+                    .toString();
+
+            recordingRotation = RecorderUtil.getDeviceRotationInDegree(getApplicationContext());
+
+            recordingPriority = RecorderUtil.getRecordingPriority(intent);
+
+            recordingMaxDuration = RecorderUtil.getRecordingMaxDuration(intent);
+
+            recordingResolutionMode = RecorderUtil.getRecordingResolutionMode(intent);
+
+            // start record
+            final MediaProjectionManager manager
+                    = (MediaProjectionManager) getSystemService(
+                    Context.MEDIA_PROJECTION_SERVICE);
+
+            if (manager == null) {
+                Log.e(TAG, "handleRecording: " +
+                        "Unable to retrieve MediaProjectionManager instance");
+                finishActivity();
+                return;
+            }
+
+            final Intent permissionIntent = manager.createScreenCaptureIntent();
+
+            startActivityForResult(permissionIntent, REQUEST_CODE_SCREEN_CAPTURE);
+        } else if (recordingAction.equals(ACTION_RECORDING_STOP)) {
+            // stop record
+            final Intent recorderIntent = new Intent(this, RecorderService.class);
+            recorderIntent.setAction(ACTION_RECORDING_STOP);
+            startService(recorderIntent);
+
+            finishActivity();
+        } else {
+            Log.e(TAG, "handleRecording: Unknown recording intent with action:"
+                    + recordingAction);
+            finishActivity();
+        }
+    }
+
+    private void finishActivity() {
         Log.d(TAG, "Closing the app");
         Handler handler = new Handler();
         handler.postDelayed(Settings.this::finish, 1000);
     }
 
-    private void registerSettingsReceivers(List<Class<? extends BroadcastReceiver>> receiverClasses) {
+    @Override
+    protected void onActivityResult(final int requestCode, final int resultCode, final Intent data)
+    {
+        super.onActivityResult(requestCode, resultCode, data);
+        if (REQUEST_CODE_SCREEN_CAPTURE != requestCode) {
+            Log.e(TAG, "handleRecording: onActivityResult: " +
+                    "Received unknown request with code: " + requestCode);
+            finishActivity();
+            return;
+        }
+
+        if (resultCode != Activity.RESULT_OK) {
+            Log.e(TAG, "handleRecording: onActivityResult: " +
+                    "MediaProjection permission is not granted, " +
+                    "Did you apply appops adb command?");
+            finishActivity();
+            return;
+        }
+
+        final Intent intent = new Intent(this, RecorderService.class);
+        intent.setAction(ACTION_RECORDING_START);
+        intent.putExtra(ACTION_RECORDING_RESULT_CODE, resultCode);
+        intent.putExtra(ACTION_RECORDING_FILENAME, recordingOutputPath);
+        intent.putExtra(ACTION_RECORDING_ROTATION, recordingRotation);
+        intent.putExtra(ACTION_RECORDING_PRIORITY, recordingPriority);
+        intent.putExtra(ACTION_RECORDING_MAX_DURATION, recordingMaxDuration);
+        intent.putExtra(ACTION_RECORDING_RESOLUTION, recordingResolutionMode);
+        intent.putExtras(data);
+
+        startService(intent);
+
+        finishActivity();
+    }
+
+    private void registerSettingsReceivers(List<Class<? extends BroadcastReceiver>> receiverClasses)
+    {
         for (Class<? extends BroadcastReceiver> receiverClass: receiverClasses) {
             try {
                 final BroadcastReceiver receiver = receiverClass.newInstance();

app/src/main/java/io/appium/settings/recorder/RecorderConstant.java

@@ -0,0 +1,94 @@
+/*
+  Copyright 2012-present Appium Committers
+  <p>
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+  <p>
+  http://www.apache.org/licenses/LICENSE-2.0
+  <p>
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+package io.appium.settings.recorder;
+
+import android.media.MediaFormat;
+import android.os.Build;
+import android.util.Size;
+
+import java.util.Arrays;
+import java.util.List;
+
+import androidx.annotation.RequiresApi;
+import io.appium.settings.BuildConfig;
+
+public class RecorderConstant {
+    public static final int REQUEST_CODE_SCREEN_CAPTURE = 123;
+    public static final String ACTION_RECORDING_BASE = BuildConfig.APPLICATION_ID + ".recording";
+    public static final String ACTION_RECORDING_START = ACTION_RECORDING_BASE + ".ACTION_START";
+    public static final String ACTION_RECORDING_STOP = ACTION_RECORDING_BASE + ".ACTION_STOP";
+    public static final String ACTION_RECORDING_RESULT_CODE = "result_code";
+    public static final String ACTION_RECORDING_ROTATION = "recording_rotation";
+    public static final String ACTION_RECORDING_FILENAME = "filename";
+    public static final String ACTION_RECORDING_PRIORITY = "priority";
+    public static final String ACTION_RECORDING_MAX_DURATION = "max_duration_sec";
+    public static final String ACTION_RECORDING_RESOLUTION = "resolution";
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+    public static final String RECORDING_DEFAULT_VIDEO_MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC;
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+    public static final Size RECORDING_RESOLUTION_FULL_HD = new Size(1920, 1080);
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+    public static final Size RECORDING_RESOLUTION_HD = new Size(1280, 720);
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+    public static final Size RECORDING_RESOLUTION_480P = new Size(720, 480);
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+    public static final Size RECORDING_RESOLUTION_QVGA = new Size(320, 240);
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+    public static final Size RECORDING_RESOLUTION_QCIF = new Size(176, 144);
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+    public static final Size RECORDING_RESOLUTION_DEFAULT = new Size(1920, 1080);
+    public static final float BITRATE_MULTIPLIER = 0.25f;
+    public static final int AUDIO_CODEC_SAMPLE_RATE_HZ = 44100;
+    public static final int AUDIO_CODEC_CHANNEL_COUNT = 1;
+    public static final int AUDIO_CODEC_REPEAT_PREV_FRAME_AFTER_MS = 1000000;
+    public static final int AUDIO_CODEC_I_FRAME_INTERVAL_MS = 5;
+    public static final int AUDIO_CODEC_DEFAULT_BITRATE = 64000;
+    public static final int VIDEO_CODEC_DEFAULT_FRAME_RATE = 30;
+    public static final long MEDIA_QUEUE_BUFFERING_DEFAULT_TIMEOUT_MS = 10000;
+    public static final long NANOSECONDS_IN_MICROSECOND = 1000;
+    public static final String NO_PATH_SET = "";
+    public static final long NO_TIMESTAMP_SET = -1;
+    // Assume 0 degree == portrait as default
+    public static final int RECORDING_ROTATION_DEFAULT_DEGREE = 0;
+    public static final int NO_TRACK_INDEX_SET = -1;
+    public static final String NO_RESOLUTION_MODE_SET = "";
+    public static final String RECORDING_PRIORITY_MAX = "high";
+    public static final String RECORDING_PRIORITY_NORM = "normal";
+    public static final String RECORDING_PRIORITY_MIN = "low";
+    public static final int RECORDING_PRIORITY_DEFAULT = Thread.MAX_PRIORITY;
+    public static final int RECORDING_MAX_DURATION_DEFAULT_MS = 15 * 60 * 1000; // 15 Minutes, in milliseconds
+    /*
+    * Note: Reason we limit recording to following resolution list is that
+    * android's AVC/H264 video encoder capabilities varies device-to-device (OEM modifications)
+    * and with values larger than 1920x1080, isSizeSupported(width, height) method (see https://developer.android.com/reference/android/media/MediaCodecInfo.VideoCapabilities#isSizeSupported(int,%20int))
+    * returns false on tested devices and also with arbitrary values between supported range,
+    * sometimes MediaEncoder.configure() method crashes with an exception on some phones
+    * also default supported resolutions as per CTS tests are limited to following values (values between 176x144 and 1920x1080)
+    * see https://android.googlesource.com/platform/cts/+/refs/heads/android12-qpr1-release/tests/tests/media/src/android/media/cts/VideoEncoderTest.java#1766
+    * because of these reasons, to support wide variety of devices, we pre-limit resolution modes to the following values
+     */
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
+    public static final List<Size> RECORDING_RESOLUTION_LIST =
+            Arrays.asList(
+                    new Size(1920, 1080),
+                    new Size(1280, 720),
+                    new Size(720, 480),
+                    new Size(320, 240),
+                    new Size(176, 144));
+    // 1048576 Bps == 1 Mbps (1024*1024)
+    public static final float BPS_IN_MBPS = 1048576f;
+}