io.appium.settings: recording: Handle orientation and make resolution dynamic

Signed-off-by: sirmordred <[email protected]>

Author: sirmordred

app/src/main/java/io/appium/settings/RecorderService.java

@@ -89,29 +89,25 @@ private void startScreenRecord(MediaProjectionManager mediaProjectionManager,
             if (projection != null) {
                 Log.v(TAG, "projection different from null");
                 final DisplayMetrics metrics = getResources().getDisplayMetrics();
-                int width = metrics.widthPixels;
-                int height = metrics.heightPixels;
-                float scale;
-                final float scaleX;
-                final float scaleY;
-                if (width > height) {
-                    // Horizontal
-                    scaleX = width / 1920f;
-                    scaleY = height / 1080f;
-                } else {
-                    // Vertical
-                    scaleX = width / 1080f;
-                    scaleY = height / 1920f;
+                int rawWidth = metrics.widthPixels;
+                int rawHeight = metrics.heightPixels;
+                boolean isRotated = false;
+                if (rawWidth > rawHeight) {
+                    // landscape mode
+                    isRotated = true;
+                    /* TODO we need to rotate frames that comes from virtual screen before writing to file via muxer,
+                    *  for handling landscape mode properly, rotateYuvImages somehow fast and reliable
+                    *  for now, just flip width and height to fake and force portrait mode instead of landscape
+                    * */
+                    rawWidth = metrics.heightPixels;
+                    rawHeight = metrics.widthPixels;
                 }
-                scale = Math.max(scaleX,  scaleY);
-                width = (int)(width / scale);
-                height = (int)(height / scale);
                 Log.v(TAG, String.format("startRecording:(%d,%d)(%d,%d)",
-                        metrics.widthPixels, metrics.heightPixels, width, height));
+                        metrics.widthPixels, metrics.heightPixels, rawWidth, rawHeight));
                 String outputFilePath = intent.getStringExtra(ACTION_RECORDING_FILENAME);
                 if (outputFilePath != null) {
-                    recorderThread = new RecorderThread(projection, outputFilePath, width, height,
-                            calcBitRate(width, height));
+                    recorderThread = new RecorderThread(projection, outputFilePath,
+                            rawWidth, rawHeight, isRotated);
                     recorderThread.startRecording();
                 } else {
                     Log.i(TAG, "outputFilePath == null");
@@ -120,13 +116,6 @@ private void startScreenRecord(MediaProjectionManager mediaProjectionManager,
         }
     }
 
-    protected int calcBitRate(int width, int height) {
-        final int bitrate = (int) (RecorderConstant.BITRATE_MULTIPLIER *
-                RecorderConstant.VIDEO_CODEC_FRAME_RATE * width * height);
-        Log.i(TAG, String.format("bitrate=%5.2f[Mbps]", bitrate / 1024f / 1024f));
-        return bitrate;
-    }
-
     /**
      * stop screen recording
      */

app/src/main/java/io/appium/settings/RecorderThread.java

@@ -50,6 +50,7 @@ public class RecorderThread implements Runnable {
     private int videoBitrate;
     private int sampleRate;
     private Thread recordingThread;
+    private boolean isRotated;
 
     private volatile boolean stopped = false;
     private volatile boolean audioStopped;
@@ -73,14 +74,14 @@ public void onStopped() {
     };
 
     public RecorderThread(MediaProjection mediaProjection, String outputFilePath,
-                          int videoWidth, int videoHeight, int videoBitrate) {
+                          int videoWidth, int videoHeight, boolean isRotated) {
         this.mediaProjection = mediaProjection;
         handler = new Handler();
         this.outputFilePath = outputFilePath;
         this.videoWidth = videoWidth;
         this.videoHeight = videoHeight;
-        this.videoBitrate = videoBitrate;
         this.sampleRate = RecorderConstant.AUDIO_CODEC_SAMPLE_RATE;
+        this.isRotated = isRotated;
         videoMime = MediaFormat.MIMETYPE_VIDEO_AVC;
     }
 
@@ -98,8 +99,20 @@ public boolean isRecordingContinue() {
         return stopped;
     }
 
+    @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
     private void setupVideoCodec() throws IOException {
-        // Encoded video resolution matches virtual display.
+        // Encoded video resolution MUST match virtual display.
+        videoEncoder = MediaCodec.createEncoderByType(videoMime);
+
+        /* Clamp raw width/height of device screen with video encoder's capabilities
+        *  to avoid crash.
+        * */
+        MediaCodecInfo.VideoCapabilities videoEncoderCapabilities = videoEncoder.getCodecInfo().
+                getCapabilitiesForType(videoMime).getVideoCapabilities();
+        this.videoWidth = videoEncoderCapabilities.getSupportedWidths().clamp(this.videoWidth);
+        this.videoHeight = videoEncoderCapabilities.getSupportedHeights().clamp(this.videoHeight);
+        this.videoBitrate = calcBitRate(videoWidth, videoHeight);
+
         MediaFormat encoderFormat = MediaFormat.createVideoFormat(videoMime, videoWidth,
                 videoHeight);
         encoderFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
@@ -112,7 +125,6 @@ private void setupVideoCodec() throws IOException {
         encoderFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL,
                 RecorderConstant.AUDIO_CODEC_I_FRAME_INTERVAL);
 
-        videoEncoder = MediaCodec.createEncoderByType(videoMime);
         videoEncoder.configure(encoderFormat, null, null,
                 MediaCodec.CONFIGURE_FLAG_ENCODE);
         surface = videoEncoder.createInputSurface();
@@ -227,6 +239,13 @@ private long getPresentationTimeUs() {
                         - startTimestampUs);
     }
 
+    private int calcBitRate(int width, int height) {
+        final int bitrate = (int) (RecorderConstant.BITRATE_MULTIPLIER *
+                RecorderConstant.VIDEO_CODEC_FRAME_RATE * width * height);
+        Log.i(TAG, String.format("bitrate=%5.2f[Mbps]", bitrate / 1024f / 1024f));
+        return bitrate;
+    }
+
     private void startMuxerIfSetUp() {
         if (audioTrackIndex >= 0 && videoTrackIndex >= 0) {
             muxer.start();
@@ -247,6 +266,12 @@ public void run() {
             muxer = new MediaMuxer(this.outputFilePath,
                     MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
 
+            // set output file orientation info if it is in landscape mode (default is 0==portrait)
+            // note: this method must be run before muxer.start()
+            if (isRotated) {
+                muxer.setOrientationHint(90);
+            }
+
             startAudioRecord();
 
             MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();