[SR] Fix Session Replay crashes

CHANGELOG.md

@@ -2,10 +2,18 @@
 
 ## Unreleased
 
+### Features
+
+- Session Replay: Gesture/touch support for Flutter ([#3623](https://github.com/getsentry/sentry-java/pull/3623))
+
 ### Fixes
 
 - Avoid ArrayIndexOutOfBoundsException on Android cpu data collection ([#3598](https://github.com/getsentry/sentry-java/pull/3598))
 - Fix lazy select queries instrumentation ([#3604](https://github.com/getsentry/sentry-java/pull/3604))
+- Session Replay: buffer mode improvements ([#3622](https://github.com/getsentry/sentry-java/pull/3622))
+  - Align next segment timestamp with the end of the buffered segment when converting from buffer mode to session mode
+  - Persist `buffer` replay type for the entire replay when converting from buffer mode to session mode
+  - Properly store screen names for `buffer` mode
 
 ### Chores
 

sentry-android-replay/api/sentry-android-replay.api

@@ -36,19 +36,20 @@ public abstract interface class io/sentry/android/replay/Recorder : java/io/Clos
 public final class io/sentry/android/replay/ReplayCache : java/io/Closeable {
 	public static final field Companion Lio/sentry/android/replay/ReplayCache$Companion;
 	public fun <init> (Lio/sentry/SentryOptions;Lio/sentry/protocol/SentryId;Lio/sentry/android/replay/ScreenshotRecorderConfig;)V
-	public final fun addFrame (Ljava/io/File;J)V
+	public final fun addFrame (Ljava/io/File;JLjava/lang/String;)V
+	public static synthetic fun addFrame$default (Lio/sentry/android/replay/ReplayCache;Ljava/io/File;JLjava/lang/String;ILjava/lang/Object;)V
 	public fun close ()V
 	public final fun createVideoOf (JJIIILjava/io/File;)Lio/sentry/android/replay/GeneratedVideo;
 	public static synthetic fun createVideoOf$default (Lio/sentry/android/replay/ReplayCache;JJIIILjava/io/File;ILjava/lang/Object;)Lio/sentry/android/replay/GeneratedVideo;
 	public final fun persistSegmentValues (Ljava/lang/String;Ljava/lang/String;)V
-	public final fun rotate (J)V
+	public final fun rotate (J)Ljava/lang/String;
 }
 
 public final class io/sentry/android/replay/ReplayCache$Companion {
 	public final fun makeReplayCacheDir (Lio/sentry/SentryOptions;Lio/sentry/protocol/SentryId;)Ljava/io/File;
 }
 
-public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/TouchRecorderCallback, java/io/Closeable {
+public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/gestures/TouchRecorderCallback, java/io/Closeable {
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function2;)V
 	public synthetic fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function2;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
@@ -102,7 +103,18 @@ public final class io/sentry/android/replay/ScreenshotRecorderConfig$Companion {
 	public final fun from (Landroid/content/Context;Lio/sentry/SentryReplayOptions;)Lio/sentry/android/replay/ScreenshotRecorderConfig;
 }
 
-public abstract interface class io/sentry/android/replay/TouchRecorderCallback {
+public final class io/sentry/android/replay/gestures/GestureRecorder : io/sentry/android/replay/OnRootViewsChangedListener {
+	public fun <init> (Lio/sentry/SentryOptions;Lio/sentry/android/replay/gestures/TouchRecorderCallback;)V
+	public fun onRootViewsChanged (Landroid/view/View;Z)V
+	public final fun stop ()V
+}
+
+public final class io/sentry/android/replay/gestures/ReplayGestureConverter {
+	public fun <init> (Lio/sentry/transport/ICurrentDateProvider;)V
+	public final fun convert (Landroid/view/MotionEvent;Lio/sentry/android/replay/ScreenshotRecorderConfig;)Ljava/util/List;
+}
+
+public abstract interface class io/sentry/android/replay/gestures/TouchRecorderCallback {
 	public abstract fun onTouchEvent (Landroid/view/MotionEvent;)V
 }
 

sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt

@@ -76,7 +76,7 @@ public class ReplayCache(
      * @param bitmap the frame screenshot
      * @param frameTimestamp the timestamp when the frame screenshot was taken
      */
-    internal fun addFrame(bitmap: Bitmap, frameTimestamp: Long) {
+    internal fun addFrame(bitmap: Bitmap, frameTimestamp: Long, screen: String? = null) {
         if (replayCacheDir == null || bitmap.isRecycled) {
             return
         }
@@ -89,7 +89,7 @@ public class ReplayCache(
             it.flush()
         }
 
-        addFrame(screenshot, frameTimestamp)
+        addFrame(screenshot, frameTimestamp, screen)
     }
 
     /**
@@ -101,8 +101,8 @@ public class ReplayCache(
      * @param screenshot file containing the frame screenshot
      * @param frameTimestamp the timestamp when the frame screenshot was taken
      */
-    public fun addFrame(screenshot: File, frameTimestamp: Long) {
-        val frame = ReplayFrame(screenshot, frameTimestamp)
+    public fun addFrame(screenshot: File, frameTimestamp: Long, screen: String? = null) {
+        val frame = ReplayFrame(screenshot, frameTimestamp, screen)
         frames += frame
     }
 
@@ -233,15 +233,20 @@ public class ReplayCache(
      * Removes frames from the in-memory and disk cache from start to [until].
      *
      * @param until value until whose the frames should be removed, represented as unix timestamp
+     * @return the first screen in the rotated buffer, if any
      */
-    fun rotate(until: Long) {
+    fun rotate(until: Long): String? {
+        var screen: String? = null
         frames.removeAll {
             if (it.timestamp < until) {
                 deleteFile(it.screenshot)
                 return@removeAll true
+            } else if (screen == null) {
+                screen = it.screen
             }
             return@removeAll false
         }
+        return screen
     }
 
     override fun close() {
@@ -426,7 +431,8 @@ internal data class LastSegmentData(
 
 internal data class ReplayFrame(
     val screenshot: File,
-    val timestamp: Long
+    val timestamp: Long,
+    val screen: String? = null
 )
 
 public data class GeneratedVideo(

sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt

@@ -12,7 +12,6 @@ import io.sentry.Integration
 import io.sentry.NoOpReplayBreadcrumbConverter
 import io.sentry.ReplayBreadcrumbConverter
 import io.sentry.ReplayController
-import io.sentry.ScopeObserverAdapter
 import io.sentry.SentryIntegrationPackageStorage
 import io.sentry.SentryLevel.DEBUG
 import io.sentry.SentryLevel.INFO
@@ -21,14 +20,15 @@ import io.sentry.android.replay.capture.BufferCaptureStrategy
 import io.sentry.android.replay.capture.CaptureStrategy
 import io.sentry.android.replay.capture.CaptureStrategy.ReplaySegment
 import io.sentry.android.replay.capture.SessionCaptureStrategy
+import io.sentry.android.replay.gestures.GestureRecorder
+import io.sentry.android.replay.gestures.TouchRecorderCallback
 import io.sentry.android.replay.util.MainLooperHandler
 import io.sentry.android.replay.util.sample
 import io.sentry.android.replay.util.submitSafely
 import io.sentry.cache.PersistingScopeObserver
 import io.sentry.cache.PersistingScopeObserver.BREADCRUMBS_FILENAME
 import io.sentry.cache.PersistingScopeObserver.REPLAY_FILENAME
 import io.sentry.hints.Backfillable
-import io.sentry.protocol.Contexts
 import io.sentry.protocol.SentryId
 import io.sentry.transport.ICurrentDateProvider
 import io.sentry.util.FileUtils
@@ -39,6 +39,7 @@ import java.io.File
 import java.security.SecureRandom
 import java.util.LinkedList
 import java.util.concurrent.atomic.AtomicBoolean
+import kotlin.LazyThreadSafetyMode.NONE
 
 public class ReplayIntegration(
     private val context: Context,
@@ -64,16 +65,20 @@ public class ReplayIntegration(
         recorderConfigProvider: ((configChanged: Boolean) -> ScreenshotRecorderConfig)?,
         replayCacheProvider: ((replayId: SentryId, recorderConfig: ScreenshotRecorderConfig) -> ReplayCache)?,
         replayCaptureStrategyProvider: ((isFullSession: Boolean) -> CaptureStrategy)? = null,
-        mainLooperHandler: MainLooperHandler? = null
+        mainLooperHandler: MainLooperHandler? = null,
+        gestureRecorderProvider: (() -> GestureRecorder)? = null
     ) : this(context, dateProvider, recorderProvider, recorderConfigProvider, replayCacheProvider) {
         this.replayCaptureStrategyProvider = replayCaptureStrategyProvider
         this.mainLooperHandler = mainLooperHandler ?: MainLooperHandler()
+        this.gestureRecorderProvider = gestureRecorderProvider
     }
 
     private lateinit var options: SentryOptions
     private var hub: IHub? = null
     private var recorder: Recorder? = null
+    private var gestureRecorder: GestureRecorder? = null
     private val random by lazy { SecureRandom() }
+    private val rootViewsSpy by lazy(NONE) { RootViewsSpy.install() }
 
     // TODO: probably not everything has to be thread-safe here
     internal val isEnabled = AtomicBoolean(false)
@@ -83,6 +88,7 @@ public class ReplayIntegration(
     private var replayBreadcrumbConverter: ReplayBreadcrumbConverter = NoOpReplayBreadcrumbConverter.getInstance()
     private var replayCaptureStrategyProvider: ((isFullSession: Boolean) -> CaptureStrategy)? = null
     private var mainLooperHandler: MainLooperHandler = MainLooperHandler()
+    private var gestureRecorderProvider: (() -> GestureRecorder)? = null
 
     private lateinit var recorderConfig: ScreenshotRecorderConfig
 
@@ -102,13 +108,8 @@ public class ReplayIntegration(
         }
 
         this.hub = hub
-        this.options.addScopeObserver(object : ScopeObserverAdapter() {
-            override fun setContexts(contexts: Contexts) {
-                // scope screen has fully-qualified name
-                captureStrategy?.onScreenChanged(contexts.app?.viewNames?.lastOrNull()?.substringAfterLast('.'))
-            }
-        })
-        recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, this, mainLooperHandler)
+        recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, mainLooperHandler)
+        gestureRecorder = gestureRecorderProvider?.invoke() ?: GestureRecorder(options, this)
         isEnabled.set(true)
 
         try {
@@ -155,6 +156,7 @@ public class ReplayIntegration(
 
         captureStrategy?.start(recorderConfig)
         recorder?.start(recorderConfig)
+        registerRootViewListeners()
     }
 
     override fun resume() {
@@ -176,8 +178,9 @@ public class ReplayIntegration(
             return
         }
 
-        captureStrategy?.captureReplay(isTerminating == true, onSegmentSent = {
+        captureStrategy?.captureReplay(isTerminating == true, onSegmentSent = { newTimestamp ->
             captureStrategy?.currentSegment = captureStrategy?.currentSegment!! + 1
+            captureStrategy?.segmentTimestamp = newTimestamp
         })
         captureStrategy = captureStrategy?.convert()
     }
@@ -204,16 +207,20 @@ public class ReplayIntegration(
             return
         }
 
+        unregisterRootViewListeners()
         recorder?.stop()
+        gestureRecorder?.stop()
         captureStrategy?.stop()
         isRecording.set(false)
         captureStrategy?.close()
         captureStrategy = null
     }
 
     override fun onScreenshotRecorded(bitmap: Bitmap) {
+        var screen: String? = null
+        hub?.configureScope { screen = it.screen?.substringAfterLast('.') }
         captureStrategy?.onScreenshotRecorded(bitmap) { frameTimeStamp ->
-            addFrame(bitmap, frameTimeStamp)
+            addFrame(bitmap, frameTimeStamp, screen)
         }
     }
 
@@ -257,6 +264,20 @@ public class ReplayIntegration(
         captureStrategy?.onTouchEvent(event)
     }
 
+    private fun registerRootViewListeners() {
+        if (recorder is OnRootViewsChangedListener) {
+            rootViewsSpy.listeners += (recorder as OnRootViewsChangedListener)
+        }
+        rootViewsSpy.listeners += gestureRecorder
+    }
+
+    private fun unregisterRootViewListeners() {
+        if (recorder is OnRootViewsChangedListener) {
+            rootViewsSpy.listeners -= (recorder as OnRootViewsChangedListener)
+        }
+        rootViewsSpy.listeners -= gestureRecorder
+    }
+
     private fun cleanupReplays(unfinishedReplayId: String = "") {
         // clean up old replays
         options.cacheDirPath?.let { cacheDir ->

sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt

@@ -26,6 +26,7 @@ import io.sentry.SentryReplayOptions
 import io.sentry.android.replay.util.MainLooperHandler
 import io.sentry.android.replay.util.getVisibleRects
 import io.sentry.android.replay.util.gracefullyShutdown
+import io.sentry.android.replay.util.submitSafely
 import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode
 import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode.ImageViewHierarchyNode
 import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode.TextViewHierarchyNode
@@ -122,7 +123,7 @@ internal class ScreenshotRecorder(
                         val viewHierarchy = ViewHierarchyNode.fromView(root, null, 0, options)
                         root.traverse(viewHierarchy)
 
-                        recorder.submit {
+                        recorder.submitSafely(options, "screenshot_recorder.redact") {
                             val canvas = Canvas(bitmap)
                             canvas.setMatrix(prescaledMatrix)
                             viewHierarchy.traverse { node ->
@@ -288,6 +289,18 @@ public data class ScreenshotRecorderConfig(
     val frameRate: Int,
     val bitRate: Int
 ) {
+    internal constructor(
+        scaleFactorX: Float,
+        scaleFactorY: Float
+    ) : this(
+        recordingWidth = 0,
+        recordingHeight = 0,
+        scaleFactorX = scaleFactorX,
+        scaleFactorY = scaleFactorY,
+        frameRate = 0,
+        bitRate = 0
+    )
+
     companion object {
         /**
          * Since codec block size is 16, so we have to adjust the width and height to it, otherwise