feat(replay): Capture SurfaceView content (experimental)

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## Unreleased
+
+### Features
+
+- Session Replay: experimental support for capturing `SurfaceView` content (e.g. Unity, video players, maps) ([#5333](https://github.com/getsentry/sentry-java/pull/5333))
+  - To enable, set `options.sessionReplay.isCaptureSurfaceViews = true`
+  - Or via manifest: `<meta-data android:name="io.sentry.session-replay.capture-surface-views" android:value="true" />`
+
 ## 8.40.0
 
 ### Fixes

sentry-android-core/src/main/java/io/sentry/android/core/ManifestMetadataReader.java

@@ -120,6 +120,8 @@ final class ManifestMetadataReader {
 
   static final String REPLAYS_DEBUG = "io.sentry.session-replay.debug";
   static final String REPLAYS_SCREENSHOT_STRATEGY = "io.sentry.session-replay.screenshot-strategy";
+  static final String REPLAYS_CAPTURE_SURFACE_VIEWS =
+      "io.sentry.session-replay.capture-surface-views";
 
   static final String REPLAYS_NETWORK_DETAIL_ALLOW_URLS =
       "io.sentry.session-replay.network-detail-allow-urls";
@@ -547,6 +549,15 @@ static void applyMetadata(
           }
         }
 
+        options
+            .getSessionReplay()
+            .setCaptureSurfaceViews(
+                readBool(
+                    metadata,
+                    logger,
+                    REPLAYS_CAPTURE_SURFACE_VIEWS,
+                    options.getSessionReplay().isCaptureSurfaceViews()));
+
         // Network Details Configuration
         if (options.getSessionReplay().getNetworkDetailAllowUrls().isEmpty()) {
           final @Nullable List<String> allowUrls =

sentry-android-core/src/main/java/io/sentry/android/core/ScreenshotEventProcessor.java

@@ -201,7 +201,7 @@ private boolean isMaskingEnabled() {
 
       final ViewHierarchyNode rootNode =
           ViewHierarchyNode.Companion.fromView(rootView, null, 0, options.getScreenshot());
-      ViewsKt.traverse(rootView, rootNode, options.getScreenshot(), options.getLogger());
+      ViewsKt.traverse(rootView, rootNode, options.getScreenshot(), options.getLogger(), null);
       return rootNode;
     } catch (Throwable e) {
       options.getLogger().log(SentryLevel.ERROR, "Failed to build view hierarchy", e);

sentry-android-core/src/test/java/io/sentry/android/core/ManifestMetadataReaderTest.kt

@@ -2022,6 +2022,31 @@ class ManifestMetadataReaderTest {
     )
   }
 
+  @Test
+  fun `applyMetadata reads capture-surface-views to options`() {
+    // Arrange
+    val bundle = bundleOf(ManifestMetadataReader.REPLAYS_CAPTURE_SURFACE_VIEWS to true)
+    val context = fixture.getContext(metaData = bundle)
+
+    // Act
+    ManifestMetadataReader.applyMetadata(context, fixture.options, fixture.buildInfoProvider)
+
+    // Assert
+    assertTrue(fixture.options.sessionReplay.isCaptureSurfaceViews)
+  }
+
+  @Test
+  fun `applyMetadata reads capture-surface-views and keeps default if not found`() {
+    // Arrange
+    val context = fixture.getContext()
+
+    // Act
+    ManifestMetadataReader.applyMetadata(context, fixture.options, fixture.buildInfoProvider)
+
+    // Assert
+    assertFalse(fixture.options.sessionReplay.isCaptureSurfaceViews)
+  }
+
   @Test
   fun `applyMetadata reads anrProfilingSampleRate to options`() {
     // Arrange

sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt

@@ -70,7 +70,7 @@ internal class ScreenshotRecorder(
       )
     }
 
-    if (!contentChanged.get()) {
+    if (!contentChanged.get() && !screenshotStrategy.hasSurfaceViews()) {
       screenshotStrategy.emitLastScreenshot()
       return
     }

sentry-android-replay/src/main/java/io/sentry/android/replay/screenshot/PixelCopyStrategy.kt

@@ -2,7 +2,13 @@ package io.sentry.android.replay.screenshot
 
 import android.annotation.SuppressLint
 import android.graphics.Bitmap
+import android.graphics.Canvas
 import android.graphics.Matrix
+import android.graphics.Paint
+import android.graphics.PorterDuff
+import android.graphics.PorterDuffXfermode
+import android.graphics.Rect
+import android.graphics.RectF
 import android.view.PixelCopy
 import android.view.View
 import io.sentry.SentryLevel.DEBUG
@@ -19,6 +25,7 @@ import io.sentry.android.replay.util.ReplayRunnable
 import io.sentry.android.replay.util.traverse
 import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode
 import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicInteger
 import kotlin.LazyThreadSafetyMode.NONE
 
 @SuppressLint("UseKtx")
@@ -40,6 +47,16 @@ internal class PixelCopyStrategy(
   private val maskRenderer = MaskRenderer()
   private val contentChanged = AtomicBoolean(false)
   private val isClosed = AtomicBoolean(false)
+  private val hasSurfaceViews = AtomicBoolean(false)
+  private val dstOverPaint by
+    lazy(NONE) { Paint().apply { xfermode = PorterDuffXfermode(PorterDuff.Mode.DST_OVER) } }
+  private val screenshotCanvas by lazy(NONE) { Canvas(screenshot) }
+  private val tmpSrcRect = Rect()
+  private val tmpDstRect = RectF()
+  private val windowLocation = IntArray(2)
+  private val svLocation = IntArray(2)
+
+  private class SurfaceViewCapture(val bitmap: Bitmap, val x: Int, val y: Int)
 
   @SuppressLint("NewApi")
   override fun capture(root: View) {
@@ -81,31 +98,21 @@ internal class PixelCopyStrategy(
 
           // TODO: disableAllMasking here and dont traverse?
           val viewHierarchy = ViewHierarchyNode.fromView(root, null, 0, options.sessionReplay)
-          root.traverse(viewHierarchy, options.sessionReplay, options.logger)
-
-          executor.submit(
-            ReplayRunnable("screenshot_recorder.mask") {
-              if (isClosed.get() || screenshot.isRecycled) {
-                options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping masking")
-                return@ReplayRunnable
-              }
+          val surfaceViewNodes =
+            if (options.sessionReplay.isCaptureSurfaceViews) {
+              mutableListOf<ViewHierarchyNode.SurfaceViewHierarchyNode>()
+            } else {
+              null
+            }
+          root.traverse(viewHierarchy, options.sessionReplay, options.logger, surfaceViewNodes)
 
-              val debugMasks = maskRenderer.renderMasks(screenshot, viewHierarchy, prescaledMatrix)
+          hasSurfaceViews.set(surfaceViewNodes?.isNotEmpty() == true)
 
-              if (options.replayController.isDebugMaskingOverlayEnabled()) {
-                mainLooperHandler.post {
-                  if (debugOverlayDrawable.callback == null) {
-                    root.overlay.add(debugOverlayDrawable)
-                  }
-                  debugOverlayDrawable.updateMasks(debugMasks)
-                  root.postInvalidate()
-                }
-              }
-              screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
-              lastCaptureSuccessful.set(true)
-              contentChanged.set(false)
-            }
-          )
+          if (surfaceViewNodes.isNullOrEmpty()) {
+            submitMaskingAndCallback(root, viewHierarchy)
+          } else {
+            captureSurfaceViews(root, surfaceViewNodes, viewHierarchy)
+          }
         },
         mainLooperHandler.handler,
       )
@@ -115,6 +122,126 @@ internal class PixelCopyStrategy(
     }
   }
 
+  private fun submitMaskingAndCallback(root: View, viewHierarchy: ViewHierarchyNode) {
+    executor.submit(
+      ReplayRunnable("screenshot_recorder.mask") { applyMaskingAndNotify(root, viewHierarchy) }
+    )
+  }
+
+  private fun applyMaskingAndNotify(root: View, viewHierarchy: ViewHierarchyNode) {
+    if (isClosed.get() || screenshot.isRecycled) {
+      options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping masking")
+      return
+    }
+
+    val debugMasks = maskRenderer.renderMasks(screenshot, viewHierarchy, prescaledMatrix)
+
+    if (options.replayController.isDebugMaskingOverlayEnabled()) {
+      mainLooperHandler.post {
+        if (debugOverlayDrawable.callback == null) {
+          root.overlay.add(debugOverlayDrawable)
+        }
+        debugOverlayDrawable.updateMasks(debugMasks)
+        root.postInvalidate()
+      }
+    }
+    screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
+    lastCaptureSuccessful.set(true)
+    contentChanged.set(false)
+  }
+
+  @SuppressLint("NewApi")
+  private fun captureSurfaceViews(
+    root: View,
+    surfaceViewNodes: List<ViewHierarchyNode.SurfaceViewHierarchyNode>,
+    viewHierarchy: ViewHierarchyNode,
+  ) {
+    root.getLocationOnScreen(windowLocation)
+
+    val captures = arrayOfNulls<SurfaceViewCapture>(surfaceViewNodes.size)
+    val remaining = AtomicInteger(surfaceViewNodes.size)
+
+    fun onCaptureComplete() {
+      if (remaining.decrementAndGet() == 0) {
+        compositeSurfaceViewsAndMask(root, captures, viewHierarchy)
+      }
+    }
+
+    for ((index, node) in surfaceViewNodes.withIndex()) {
+      val surfaceView = node.surfaceViewRef.get()
+      // holder.surface can be null before the surface is created — guard against NPE.
+      val surface = surfaceView?.holder?.surface
+      if (surfaceView == null || surface == null || !surface.isValid) {
+        onCaptureComplete()
+        continue
+      }
+
+      try {
+        val svBitmap =
+          Bitmap.createBitmap(surfaceView.width, surfaceView.height, Bitmap.Config.ARGB_8888)
+
+        surfaceView.getLocationOnScreen(svLocation)
+        val capturedX = svLocation[0]
+        val capturedY = svLocation[1]
+
+        PixelCopy.request(
+          surfaceView,
+          svBitmap,
+          { copyResult: Int ->
+            if (copyResult == PixelCopy.SUCCESS) {
+              captures[index] = SurfaceViewCapture(svBitmap, capturedX, capturedY)
+            } else {
+              svBitmap.recycle()
+              options.logger.log(INFO, "Failed to capture SurfaceView: %d", copyResult)
+            }
+            onCaptureComplete()
+          },
+          mainLooperHandler.handler,
+        )
+      } catch (e: Throwable) {
+        options.logger.log(WARNING, "Failed to capture SurfaceView", e)
+        onCaptureComplete()
+      }
+    }
+  }
+
+  private fun compositeSurfaceViewsAndMask(
+    root: View,
+    captures: Array<SurfaceViewCapture?>,
+    viewHierarchy: ViewHierarchyNode,
+  ) {
+    executor.submit(
+      ReplayRunnable("screenshot_recorder.composite") {
+        if (isClosed.get() || screenshot.isRecycled) {
+          options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping compositing")
+          return@ReplayRunnable
+        }
+
+        for (capture in captures) {
+          if (capture == null) continue
+          if (capture.bitmap.isRecycled) continue
+
+          compositeSurfaceViewInto(
+            screenshotCanvas,
+            dstOverPaint,
+            tmpSrcRect,
+            tmpDstRect,
+            capture.bitmap,
+            capture.x,
+            capture.y,
+            windowLocation[0],
+            windowLocation[1],
+            config.scaleFactorX,
+            config.scaleFactorY,
+          )
+          capture.bitmap.recycle()
+        }
+
+        applyMaskingAndNotify(root, viewHierarchy)
+      }
+    )
+  }
+
   override fun onContentChanged() {
     contentChanged.set(true)
   }
@@ -123,6 +250,10 @@ internal class PixelCopyStrategy(
     return lastCaptureSuccessful.get()
   }
 
+  override fun hasSurfaceViews(): Boolean {
+    return hasSurfaceViews.get()
+  }
+
   override fun emitLastScreenshot() {
     if (lastCaptureSuccessful() && !screenshot.isRecycled) {
       screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
@@ -148,3 +279,38 @@ internal class PixelCopyStrategy(
     )
   }
 }
+
+/**
+ * Composites [sourceBitmap] (a SurfaceView capture) onto [destCanvas] (wrapping the recording
+ * screenshot) using [destPaint] (expected to have DST_OVER xfermode), so the SurfaceView content
+ * draws _behind_ existing Window content — filling the transparent holes the Window PixelCopy
+ * leaves where SurfaceViews are.
+ *
+ * Extracted for testability — the compositing is pure drawing logic that can be driven with
+ * hand-built bitmaps, while the surrounding [PixelCopyStrategy.captureSurfaceViews] flow depends on
+ * a real SurfaceView producer that Robolectric cannot provide.
+ */
+internal fun compositeSurfaceViewInto(
+  destCanvas: Canvas,
+  destPaint: Paint,
+  tmpSrc: Rect,
+  tmpDst: RectF,
+  sourceBitmap: Bitmap,
+  sourceX: Int,
+  sourceY: Int,
+  windowX: Int,
+  windowY: Int,
+  scaleFactorX: Float,
+  scaleFactorY: Float,
+) {
+  val left = (sourceX - windowX) * scaleFactorX
+  val top = (sourceY - windowY) * scaleFactorY
+  tmpSrc.set(0, 0, sourceBitmap.width, sourceBitmap.height)
+  tmpDst.set(
+    left,
+    top,
+    left + sourceBitmap.width * scaleFactorX,
+    top + sourceBitmap.height * scaleFactorY,
+  )
+  destCanvas.drawBitmap(sourceBitmap, tmpSrc, tmpDst, destPaint)
+}

sentry-android-replay/src/main/java/io/sentry/android/replay/screenshot/ScreenshotStrategy.kt

@@ -12,4 +12,11 @@ internal interface ScreenshotStrategy {
   fun lastCaptureSuccessful(): Boolean
 
   fun emitLastScreenshot()
+
+  /**
+   * Whether the last capture detected SurfaceViews that render independently of the View tree. When
+   * true, the recorder bypasses the contentChanged guard since SurfaceView redraws don't trigger
+   * ViewTreeObserver.OnDrawListener.
+   */
+  fun hasSurfaceViews(): Boolean = false
 }

sentry-android-replay/src/main/java/io/sentry/android/replay/util/Views.kt

@@ -38,6 +38,7 @@ internal fun View.traverse(
   parentNode: ViewHierarchyNode,
   options: SentryMaskingOptions,
   logger: ILogger,
+  surfaceViewNodes: MutableList<ViewHierarchyNode.SurfaceViewHierarchyNode>? = null,
 ) {
   if (this !is ViewGroup) {
     return
@@ -59,7 +60,14 @@ internal fun View.traverse(
     if (child != null) {
       val childNode = ViewHierarchyNode.fromView(child, parentNode, indexOfChild(child), options)
       childNodes.add(childNode)
-      child.traverse(childNode, options, logger)
+      if (
+        surfaceViewNodes != null &&
+          childNode is ViewHierarchyNode.SurfaceViewHierarchyNode &&
+          childNode.isVisible
+      ) {
+        surfaceViewNodes.add(childNode)
+      }
+      child.traverse(childNode, options, logger, surfaceViewNodes)
     }
   }
   parentNode.children = childNodes