fix(mobile): handle image drops in the Android composer

apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorView.kt

@@ -1,5 +1,6 @@
 package expo.modules.t3composereditor
 
+import android.app.Activity
 import android.content.ClipData
 import android.content.ClipboardManager
 import android.content.Context
@@ -16,6 +17,7 @@ import android.text.TextUtils
 import android.text.TextWatcher
 import android.text.style.ReplacementSpan
 import android.util.TypedValue
+import android.view.DragEvent
 import android.view.Gravity
 import android.view.GestureDetector
 import android.view.MotionEvent
@@ -39,7 +41,7 @@ class T3ComposerEditorView(context: Context, appContext: AppContext) : ExpoView(
   // React Native ignores the editor's requestLayout(), which makes TextView defer caret scrolls.
   override val shouldUseAndroidLayout = true
 
-  private val editor = SelectionAwareEditText(context)
+  private val editor = SelectionAwareEditText(context) { appContext.currentActivity }
   private val defaultHighlightColor = editor.highlightColor
   private val defaultSelectionColor = context.resolveThemeColor(
     android.R.attr.colorAccent,
@@ -574,7 +576,10 @@ private fun parseTokens(value: String): List<ComposerToken> = try {
   emptyList()
 }
 
-internal class SelectionAwareEditText(context: Context) : EditText(context) {
+internal class SelectionAwareEditText(
+  context: Context,
+  private val currentActivity: () -> Activity? = { context as? Activity }
+) : EditText(context) {
   var readOnly = false
   var selectionListener: ((Int, Int) -> Unit)? = null
   var pasteImagesListener: ((List<String>) -> Unit)? = null
@@ -669,6 +674,39 @@ internal class SelectionAwareEditText(context: Context) : EditText(context) {
     selectionListener?.invoke(selStart, selEnd)
   }
 
+  override fun onDragEvent(event: DragEvent): Boolean = when {
+    readOnly || !isEnabled -> false
+    event.action == DragEvent.ACTION_DROP && event.clipDescription?.hasMimeType(
+      "image/*"
+    ) == true ->
+      dropImages(event)
+    else -> super.onDragEvent(event)
+  }
+
+  private fun dropImages(event: DragEvent): Boolean {
+    val clip = event.clipData
+    val activity = currentActivity()
+    val listener = pasteImagesListener
+    if (clip == null || activity == null || listener == null) return false
+    val imageUris = try {
+      // Keep the activity-scoped grant while JavaScript asynchronously reads the dropped images.
+      activity.requestDragAndDropPermissions(event)
+      (0 until clip.itemCount).mapNotNull { index ->
+        clip.getItemAt(index).uri?.takeIf { uri ->
+          context.contentResolver.getType(uri)?.startsWith("image/") == true
+        }?.toString()
+      }
+    } catch (_: SecurityException) {
+      emptyList()
+    }
+    return if (imageUris.size == clip.itemCount) {
+      listener(imageUris)
+      true
+    } else {
+      false
+    }
+  }
+
   override fun onTextContextMenuItem(id: Int): Boolean {
     val pasting = id == android.R.id.paste || id == android.R.id.pasteAsPlainText
     if (readOnly && (id == android.R.id.cut || pasting)) {

apps/mobile/modules/t3-composer-editor/android/src/test/java/expo/modules/t3composereditor/ComposerPasteTest.kt

@@ -1,17 +1,27 @@
 package expo.modules.t3composereditor
 
+import android.app.Activity
+import android.content.ContentProvider
+import android.content.ContentValues
+import android.database.Cursor
 import android.content.ClipData
 import android.content.ClipDescription
 import android.content.ClipboardManager
 import android.content.Context
+import android.content.pm.ProviderInfo
 import android.net.Uri
+import android.view.DragEvent
+import android.view.View
 import android.view.KeyEvent
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertFalse
 import org.junit.Assert.assertNull
 import org.junit.Assert.assertTrue
 import org.junit.Test
 import org.junit.runner.RunWith
+import org.robolectric.Robolectric
+import org.robolectric.shadows.DragEventBuilder
+import org.robolectric.shadows.ShadowContentResolver
 import org.robolectric.RobolectricTestRunner
 import org.robolectric.RuntimeEnvironment
 import org.robolectric.annotation.Config
@@ -169,4 +179,140 @@ class ComposerPasteTest {
     assertEquals("unchanged", editor.text.toString())
     assertNull(intercepted)
   }
+
+  private fun dropEditor(): SelectionAwareEditText {
+    val activity = Robolectric.buildActivity(Activity::class.java).setup().get()
+    return SelectionAwareEditText(activity).apply {
+      activity.setContentView(this)
+      isFocusableInTouchMode = true
+      setText("before")
+      setSelection(length())
+      measure(
+        View.MeasureSpec.makeMeasureSpec(500, View.MeasureSpec.EXACTLY),
+        View.MeasureSpec.makeMeasureSpec(200, View.MeasureSpec.EXACTLY),
+      )
+      layout(0, 0, 500, 200)
+      requestFocus()
+    }
+  }
+
+  private fun imageClip(vararg names: String): ClipData {
+    val provider = DropImageProvider().apply {
+      attachInfo(this@ComposerPasteTest.context, ProviderInfo().apply { authority = "images" })
+    }
+    ShadowContentResolver.registerProviderInternal("images", provider)
+    return ClipData(
+      "images",
+      arrayOf("image/png", "image/jpeg"),
+      ClipData.Item(Uri.parse("content://images/${names.first()}")),
+    ).apply {
+      names.drop(1).forEach { addItem(ClipData.Item(Uri.parse("content://images/$it"))) }
+    }
+  }
+
+  private fun dragEvent(clip: ClipData, action: Int = DragEvent.ACTION_DROP): DragEvent =
+    DragEventBuilder.newBuilder()
+      .setAction(action)
+      .setX(400f)
+      .setY(10f)
+      .setClipDescription(clip.description)
+      .setClipData(if (action == DragEvent.ACTION_DROP) clip else null)
+      .build()
+
+  @Test
+  @Config(sdk = [24, 36])
+  fun imageDropAttachesEveryImageOnceWithoutInsertingUriText() {
+    val target = dropEditor()
+    val received = mutableListOf<List<String>>()
+    target.pasteImagesListener = { received.add(it) }
+    val clip = imageClip("screenshot.png", "photo.jpg")
+
+    assertTrue(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
+    assertTrue(target.onDragEvent(dragEvent(clip)))
+    assertEquals(
+      listOf(listOf("content://images/screenshot.png", "content://images/photo.jpg")),
+      received
+    )
+    assertEquals("before", target.text.toString())
+  }
+
+  @Test
+  @Config(sdk = [24, 36])
+  fun imageDropIsRejectedWhenTheComposerLocksDuringTheDrag() {
+    val target = dropEditor()
+    target.pasteImagesListener = { error("A locked composer must not receive images") }
+    val clip = imageClip("screenshot.png")
+    assertTrue(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
+    target.readOnly = true
+
+    assertFalse(target.onDragEvent(dragEvent(clip)))
+    assertEquals("before", target.text.toString())
+  }
+
+  @Test
+  @Config(sdk = [24, 36])
+  fun disabledComposerRejectsImageDrags() {
+    val target = dropEditor()
+    target.isEnabled = false
+    target.pasteImagesListener = { error("A disabled composer must not receive images") }
+    val clip = imageClip("screenshot.png")
+
+    assertFalse(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
+    assertFalse(target.onDragEvent(dragEvent(clip)))
+    assertEquals("before", target.text.toString())
+  }
+
+  @Test
+  @Config(sdk = [24, 36])
+  fun mixedImageAndTextDropDoesNotSilentlyDiscardItems() {
+    val target = dropEditor()
+    target.pasteImagesListener = { error("Mixed drops must not partially attach") }
+    val clip = imageClip("screenshot.png").apply { addItem(ClipData.Item("hello")) }
+
+    assertFalse(target.onDragEvent(dragEvent(clip)))
+    assertEquals("before", target.text.toString())
+  }
+
+  @Test
+  @Config(sdk = [24, 36])
+  fun unreadableImageDropDoesNotCrashOrInsertUriText() {
+    val target = dropEditor()
+    target.pasteImagesListener = { error("Unreadable images must not attach") }
+
+    assertFalse(target.onDragEvent(dragEvent(imageClip("denied.png"))))
+    assertEquals("before", target.text.toString())
+  }
+
+  @Test
+  @Config(sdk = [24, 36])
+  fun ordinaryTextDropsKeepNativeInsertion() {
+    val target = dropEditor()
+    target.pasteImagesListener = { error("Text drops must stay native") }
+
+    assertTrue(target.onDragEvent(dragEvent(ClipData.newPlainText("text", "hello"))))
+    assertEquals("beforehello", target.text.toString())
+  }
+
+  private class DropImageProvider : ContentProvider() {
+    override fun onCreate() = true
+    override fun getType(uri: Uri): String {
+      if (uri.lastPathSegment == "denied.png") throw SecurityException("No image read grant")
+      return if (uri.lastPathSegment?.endsWith(".jpg") == true) "image/jpeg" else "image/png"
+    }
+    override fun query(
+      uri: Uri,
+      projection: Array<out String>?,
+      selection: String?,
+      selectionArgs: Array<out String>?,
+      sortOrder: String?
+    ): Cursor? = null
+    override fun insert(uri: Uri, values: ContentValues?): Uri? = null
+    override fun delete(uri: Uri, selection: String?, selectionArgs: Array<out String>?) = 0
+    override fun update(
+      uri: Uri,
+      values: ContentValues?,
+      selection: String?,
+      selectionArgs: Array<out String>?
+    ) = 0
+  }
 }

apps/mobile/src/lib/composerImages.test.ts

@@ -4,7 +4,10 @@ import {
   PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
 } from "@t3tools/contracts";
 
-const files = new Map<string, { base64: string; deleted: boolean; text?: string }>();
+const files = new Map<
+  string,
+  { base64: string; deleted: boolean; text?: string; mimeType?: string }
+>();
 
 const clipboard = vi.hoisted(() => ({
   hasImageAsync: vi.fn(),
@@ -60,6 +63,10 @@ vi.mock("expo-file-system", () => ({
       return files.has(this.uri) && files.get(this.uri)?.deleted === false;
     }
 
+    get type(): string {
+      return files.get(this.uri)?.mimeType ?? "";
+    }
+
     async base64(): Promise<string> {
       const entry = files.get(this.uri);
       if (!entry || entry.deleted) {
@@ -193,6 +200,40 @@ describe("native pasted image cleanup", () => {
     files.clear();
   });
 
+  it("preserves the reported image type of extensionless Android content URIs", async () => {
+    const uri = "content://media/external/images/media/42";
+    files.set(uri, { base64: "aGVsbG8=", deleted: false, mimeType: "image/jpeg" });
+
+    const attachments = await convertPastedImagesToAttachments({ uris: [uri], existingCount: 0 });
+
+    expect(attachments).toEqual([
+      expect.objectContaining({
+        name: "pasted-image.jpeg",
+        mimeType: "image/jpeg",
+        dataUrl: "data:image/jpeg;base64,aGVsbG8=",
+        previewUri: uri,
+      }),
+    ]);
+    expect(files.get(uri)?.deleted).toBe(false);
+  });
+
+  it("renders an unsupported Android image format to JPEG before attaching it", async () => {
+    const uri = "content://media/external/images/media/43";
+    files.set(uri, { base64: "b3JpZ2luYWw=", deleted: false, mimeType: "image/heic" });
+    manipulator.state.sourceSize = { width: 100, height: 100 };
+
+    const attachments = await convertPastedImagesToAttachments({ uris: [uri], existingCount: 0 });
+
+    expect(attachments).toEqual([
+      expect.objectContaining({
+        name: "pasted-image.jpeg",
+        mimeType: "image/jpeg",
+        dataUrl: "data:image/jpeg;base64,cmVuZGVyZWQ=",
+      }),
+    ]);
+    expect(files.get(uri)?.deleted).toBe(false);
+  });
+
   it("converts owned files to data-backed previews and deletes the source", async () => {
     const uri =
       "file:///private/var/mobile/Containers/Data/Application/app/tmp/t3-composer-paste/id.png";

apps/mobile/src/lib/composerImages.ts

@@ -685,8 +685,15 @@ export async function convertPastedImagesToAttachments(input: {
         continue;
       }
       const file = new File(uri);
-      const mimeType = mimeTypeFromUri(uri);
-      const pastedBase64 = await file.base64();
+      const reportedMimeType = file.type;
+      const sourceMimeType = reportedMimeType?.startsWith("image/")
+        ? reportedMimeType
+        : mimeTypeFromUri(uri);
+      const mimeType = isProviderSendTurnSupportedImageMimeType(sourceMimeType)
+        ? sourceMimeType
+        : "image/jpeg";
+      const pastedBase64 =
+        mimeType === sourceMimeType ? await file.base64() : (await renderPhoto(uri)).base64;
       const fitted = await fitComposerImage({ uri, mimeType, base64: pastedBase64 });
       const base64 = fitted?.base64 ?? pastedBase64;
       const sizeBytes = estimateBase64ByteSize(base64);

docs/user/composer.md

@@ -46,6 +46,8 @@ You can drag or paste images into the web or desktop composer. HEIC and HEIF
 photos are converted to JPEG there and when selected from the mobile photo
 library; photos over the image limit are also resized to fit. On mobile, you can
 also send files to T3 Code through another app's system share sheet.
+On Android, you can also drag images from another app into the message input
+in split screen.
 
 See [images and videos](#images-and-videos-in-messages) for previewing and saving media.