feat(mobile): add support for encoded image requests in local/remote image APIs

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImages.g.kt

@@ -59,7 +59,7 @@ private open class LocalImagesPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface LocalImageApi {
-  fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, isVideo: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
+  fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, isVideo: Boolean, preferEncoded: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
   fun cancelRequest(requestId: Long)
   fun getThumbhash(thumbhash: String, callback: (Result<Map<String, Long>>) -> Unit)
 
@@ -82,7 +82,8 @@ interface LocalImageApi {
             val widthArg = args[2] as Long
             val heightArg = args[3] as Long
             val isVideoArg = args[4] as Boolean
-            api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg, isVideoArg) { result: Result<Map<String, Long>?> ->
+            val preferEncodedArg = args[5] as Boolean
+            api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg, isVideoArg, preferEncodedArg) { result: Result<Map<String, Long>?> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(LocalImagesPigeonUtils.wrapError(error))

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt

@@ -14,6 +14,7 @@ import android.util.Size
 import androidx.annotation.RequiresApi
 import app.alextran.immich.NativeBuffer
 import kotlin.math.*
+import java.io.IOException
 import java.util.concurrent.Executors
 import com.bumptech.glide.Glide
 import com.bumptech.glide.Priority
@@ -99,12 +100,17 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
     width: Long,
     height: Long,
     isVideo: Boolean,
+    preferEncoded: Boolean,
     callback: (Result<Map<String, Long>?>) -> Unit
   ) {
     val signal = CancellationSignal()
     val task = threadPool.submit {
       try {
-        getThumbnailBufferInternal(assetId, width, height, isVideo, callback, signal)
+        if (preferEncoded) {
+          getEncodedImageInternal(assetId, callback, signal)
+        } else {
+          getThumbnailBufferInternal(assetId, width, height, isVideo, callback, signal)
+        }
       } catch (e: Exception) {
         when (e) {
           is OperationCanceledException -> callback(CANCELLED)
@@ -133,6 +139,35 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
     }
   }
 
+  private fun getEncodedImageInternal(
+    assetId: String,
+    callback: (Result<Map<String, Long>?>) -> Unit,
+    signal: CancellationSignal
+  ) {
+    signal.throwIfCanceled()
+    val id = assetId.toLong()
+    val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id)
+
+    signal.throwIfCanceled()
+    val bytes = resolver.openInputStream(uri)?.use { it.readBytes() }
+      ?: throw IOException("Could not read image data for $assetId")
+
+    signal.throwIfCanceled()
+    val pointer = NativeBuffer.allocate(bytes.size)
+    try {
+      val buffer = NativeBuffer.wrap(pointer, bytes.size)
+      buffer.put(bytes)
+      signal.throwIfCanceled()
+      callback(Result.success(mapOf(
+        "pointer" to pointer,
+        "length" to bytes.size.toLong()
+      )))
+    } catch (e: Exception) {
+      NativeBuffer.free(pointer)
+      throw e
+    }
+  }
+
   private fun getThumbnailBufferInternal(
     assetId: String,
     width: Long,

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt

@@ -47,7 +47,7 @@ private open class RemoteImagesPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface RemoteImageApi {
-  fun requestImage(url: String, headers: Map<String, String>, requestId: Long, callback: (Result<Map<String, Long>?>) -> Unit)
+  fun requestImage(url: String, headers: Map<String, String>, requestId: Long, preferEncoded: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
   fun cancelRequest(requestId: Long)
   fun clearCache(callback: (Result<Long>) -> Unit)
 
@@ -68,7 +68,8 @@ interface RemoteImageApi {
             val urlArg = args[0] as String
             val headersArg = args[1] as Map<String, String>
             val requestIdArg = args[2] as Long
-            api.requestImage(urlArg, headersArg, requestIdArg) { result: Result<Map<String, Long>?> ->
+            val preferEncodedArg = args[3] as Boolean
+            api.requestImage(urlArg, headersArg, requestIdArg, preferEncodedArg) { result: Result<Map<String, Long>?> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(RemoteImagesPigeonUtils.wrapError(error))

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt

@@ -51,6 +51,7 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
     url: String,
     headers: Map<String, String>,
     requestId: Long,
+    @Suppress("UNUSED_PARAMETER") preferEncoded: Boolean, // always returns encoded; setting has no effect on Android
     callback: (Result<Map<String, Long>?>) -> Unit
   ) {
     val signal = CancellationSignal()

mobile/ios/Runner/Images/LocalImages.g.swift

@@ -70,7 +70,7 @@ class LocalImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
 
 /// Generated protocol from Pigeon that represents a handler of messages from Flutter.
 protocol LocalImageApi {
-  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
+  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, preferEncoded: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
   func cancelRequest(requestId: Int64) throws
   func getThumbhash(thumbhash: String, completion: @escaping (Result<[String: Int64], Error>) -> Void)
 }
@@ -90,7 +90,8 @@ class LocalImageApiSetup {
         let widthArg = args[2] as! Int64
         let heightArg = args[3] as! Int64
         let isVideoArg = args[4] as! Bool
-        api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg, isVideo: isVideoArg) { result in
+        let preferEncodedArg = args[5] as! Bool
+        api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg, isVideo: isVideoArg, preferEncoded: preferEncodedArg) { result in
           switch result {
           case .success(let res):
             reply(wrapResult(res))

mobile/ios/Runner/Images/LocalImagesImpl.swift

@@ -7,7 +7,7 @@ class LocalImageRequest {
   weak var workItem: DispatchWorkItem?
   var isCancelled = false
   let callback: (Result<[String: Int64]?, any Error>) -> Void
-  
+
   init(callback: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
     self.callback = callback
   }
@@ -30,11 +30,11 @@ class LocalImageApiImpl: LocalImageApi {
     requestOptions.version = .current
     return requestOptions
   }()
-  
+
   private static let assetQueue = DispatchQueue(label: "thumbnail.assets", qos: .userInitiated)
   private static let requestQueue = DispatchQueue(label: "thumbnail.requests", qos: .userInitiated)
   private static let cancelQueue = DispatchQueue(label: "thumbnail.cancellation", qos: .default)
-  
+
   private static var rgbaFormat = vImage_CGImageFormat(
     bitsPerComponent: 8,
     bitsPerPixel: 32,
@@ -48,12 +48,12 @@ class LocalImageApiImpl: LocalImageApi {
     assetCache.countLimit = 10000
     return assetCache
   }()
-  
+
   func getThumbhash(thumbhash: String, completion: @escaping (Result<[String : Int64], any Error>) -> Void) {
     ImageProcessing.queue.async {
       guard let data = Data(base64Encoded: thumbhash)
       else { return completion(.failure(PigeonError(code: "", message: "Invalid base64 string: \(thumbhash)", details: nil)))}
-      
+
       let (width, height, pointer) = thumbHashToRGBA(hash: data)
       completion(.success([
         "pointer": Int64(Int(bitPattern: pointer.baseAddress)),
@@ -63,34 +63,77 @@ class LocalImageApiImpl: LocalImageApi {
       ]))
     }
   }
-  
-  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
+
+  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, preferEncoded: Bool, completion: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
     let request = LocalImageRequest(callback: completion)
     let item = DispatchWorkItem {
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-      
+
       ImageProcessing.semaphore.wait()
       defer {
         ImageProcessing.semaphore.signal()
       }
-      
+
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-      
+
       guard let asset = Self.requestAsset(assetId: assetId)
       else {
         Self.remove(requestId: requestId)
         completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil)))
         return
       }
-      
+
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-
+
+      if preferEncoded {
+        let dataOptions = PHImageRequestOptions()
+        dataOptions.isNetworkAccessAllowed = true
+        dataOptions.isSynchronous = true
+        dataOptions.version = .current
+
+        var imageData: Data?
+        Self.imageManager.requestImageDataAndOrientation(
+          for: asset,
+          options: dataOptions,
+          resultHandler: { (data, _, _, _) in
+            imageData = data
+          }
+        )
+
+        if request.isCancelled {
+          Self.remove(requestId: requestId)
+          return completion(ImageProcessing.cancelledResult)
+        }
+
+        guard let data = imageData else {
+          Self.remove(requestId: requestId)
+          return completion(.failure(PigeonError(code: "", message: "Could not get image data for \(assetId)", details: nil)))
+        }
+
+        let length = data.count
+        let pointer = malloc(length)!
+        data.copyBytes(to: pointer.assumingMemoryBound(to: UInt8.self), count: length)
+
+        if request.isCancelled {
+          free(pointer)
+          Self.remove(requestId: requestId)
+          return completion(ImageProcessing.cancelledResult)
+        }
+
+        request.callback(.success([
+          "pointer": Int64(Int(bitPattern: pointer)),
+          "length": Int64(length),
+        ]))
+        Self.remove(requestId: requestId)
+        return
+      }
+
       var image: UIImage?
       Self.imageManager.requestImage(
         for: asset,
@@ -101,29 +144,29 @@ class LocalImageApiImpl: LocalImageApi {
           image = _image
         }
       )
-      
+
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-      
+
       guard let image = image,
             let cgImage = image.cgImage else {
         Self.remove(requestId: requestId)
         return completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil)))
       }
-      
+
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-      
+
       do {
         let buffer = try vImage_Buffer(cgImage: cgImage, format: Self.rgbaFormat)
-        
+
         if request.isCancelled {
           buffer.free()
           return completion(ImageProcessing.cancelledResult)
         }
-        
+
         request.callback(.success([
           "pointer": Int64(Int(bitPattern: buffer.data)),
           "width": Int64(buffer.width),
@@ -136,24 +179,24 @@ class LocalImageApiImpl: LocalImageApi {
         return completion(.failure(PigeonError(code: "", message: "Failed to convert image for \(assetId): \(error)", details: nil)))
       }
     }
-    
+
     request.workItem = item
     Self.add(requestId: requestId, request: request)
     ImageProcessing.queue.async(execute: item)
   }
-  
+
   func cancelRequest(requestId: Int64) {
     Self.cancel(requestId: requestId)
   }
-  
+
   private static func add(requestId: Int64, request: LocalImageRequest) -> Void {
     requestQueue.sync { requests[requestId] = request }
   }
-  
+
   private static func remove(requestId: Int64) -> Void {
     requestQueue.sync { requests[requestId] = nil }
   }
-  
+
   private static func cancel(requestId: Int64) -> Void {
     requestQueue.async {
       guard let request = requests.removeValue(forKey: requestId) else { return }
@@ -164,12 +207,12 @@ class LocalImageApiImpl: LocalImageApi {
       }
     }
   }
-  
+
   private static func requestAsset(assetId: String) -> PHAsset? {
     var asset: PHAsset?
     assetQueue.sync { asset = assetCache.object(forKey: assetId as NSString) }
     if asset != nil { return asset }
-    
+
     guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions).firstObject
     else { return nil }
     assetQueue.async { assetCache.setObject(asset, forKey: assetId as NSString) }

mobile/ios/Runner/Images/RemoteImages.g.swift

@@ -70,7 +70,7 @@ class RemoteImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable
 
 /// Generated protocol from Pigeon that represents a handler of messages from Flutter.
 protocol RemoteImageApi {
-  func requestImage(url: String, headers: [String: String], requestId: Int64, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
+  func requestImage(url: String, headers: [String: String], requestId: Int64, preferEncoded: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
   func cancelRequest(requestId: Int64) throws
   func clearCache(completion: @escaping (Result<Int64, Error>) -> Void)
 }
@@ -88,7 +88,8 @@ class RemoteImageApiSetup {
         let urlArg = args[0] as! String
         let headersArg = args[1] as! [String: String]
         let requestIdArg = args[2] as! Int64
-        api.requestImage(url: urlArg, headers: headersArg, requestId: requestIdArg) { result in
+        let preferEncodedArg = args[3] as! Bool
+        api.requestImage(url: urlArg, headers: headersArg, requestId: requestIdArg, preferEncoded: preferEncodedArg) { result in
           switch result {
           case .success(let res):
             reply(wrapResult(res))