feat(mobile): add support for animated images (GIFs) in the asset viewer

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImages.g.kt

@@ -59,7 +59,7 @@ private open class LocalImagesPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface LocalImageApi {
-  fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, isVideo: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
+  fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, isVideo: Boolean, encoded: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
   fun cancelRequest(requestId: Long)
   fun getThumbhash(thumbhash: String, callback: (Result<Map<String, Long>>) -> Unit)
 
@@ -82,7 +82,8 @@ interface LocalImageApi {
             val widthArg = args[2] as Long
             val heightArg = args[3] as Long
             val isVideoArg = args[4] as Boolean
-            api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg, isVideoArg) { result: Result<Map<String, Long>?> ->
+            val encodedArg = args[5] as Boolean
+            api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg, isVideoArg, encodedArg) { result: Result<Map<String, Long>?> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(LocalImagesPigeonUtils.wrapError(error))

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt

@@ -14,6 +14,7 @@ import android.util.Size
 import androidx.annotation.RequiresApi
 import app.alextran.immich.NativeBuffer
 import kotlin.math.*
+import java.io.IOException
 import java.util.concurrent.Executors
 import com.bumptech.glide.Glide
 import com.bumptech.glide.Priority
@@ -99,12 +100,17 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
     width: Long,
     height: Long,
     isVideo: Boolean,
+    encoded: Boolean,
     callback: (Result<Map<String, Long>?>) -> Unit
   ) {
     val signal = CancellationSignal()
     val task = threadPool.submit {
       try {
-        getThumbnailBufferInternal(assetId, width, height, isVideo, callback, signal)
+        if (encoded) {
+          getEncodedImageInternal(assetId, callback, signal)
+        } else {
+          getThumbnailBufferInternal(assetId, width, height, isVideo, callback, signal)
+        }
       } catch (e: Exception) {
         when (e) {
           is OperationCanceledException -> callback(CANCELLED)
@@ -133,6 +139,35 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
     }
   }
 
+  private fun getEncodedImageInternal(
+    assetId: String,
+    callback: (Result<Map<String, Long>?>) -> Unit,
+    signal: CancellationSignal
+  ) {
+    signal.throwIfCanceled()
+    val id = assetId.toLong()
+    val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id)
+
+    signal.throwIfCanceled()
+    val bytes = resolver.openInputStream(uri)?.use { it.readBytes() }
+      ?: throw IOException("Could not read image data for $assetId")
+
+    signal.throwIfCanceled()
+    val pointer = NativeBuffer.allocate(bytes.size)
+    try {
+      val buffer = NativeBuffer.wrap(pointer, bytes.size)
+      buffer.put(bytes)
+      signal.throwIfCanceled()
+      callback(Result.success(mapOf(
+        "pointer" to pointer,
+        "length" to bytes.size.toLong()
+      )))
+    } catch (e: Exception) {
+      NativeBuffer.free(pointer)
+      throw e
+    }
+  }
+
   private fun getThumbnailBufferInternal(
     assetId: String,
     width: Long,

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt

@@ -47,7 +47,7 @@ private open class RemoteImagesPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface RemoteImageApi {
-  fun requestImage(url: String, headers: Map<String, String>, requestId: Long, callback: (Result<Map<String, Long>?>) -> Unit)
+  fun requestImage(url: String, headers: Map<String, String>, requestId: Long, encoded: Boolean, callback: (Result<Map<String, Long>?>) -> Unit)
   fun cancelRequest(requestId: Long)
   fun clearCache(callback: (Result<Long>) -> Unit)
 
@@ -68,7 +68,8 @@ interface RemoteImageApi {
             val urlArg = args[0] as String
             val headersArg = args[1] as Map<String, String>
             val requestIdArg = args[2] as Long
-            api.requestImage(urlArg, headersArg, requestIdArg) { result: Result<Map<String, Long>?> ->
+            val encodedArg = args[3] as Boolean
+            api.requestImage(urlArg, headersArg, requestIdArg, encodedArg) { result: Result<Map<String, Long>?> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(RemoteImagesPigeonUtils.wrapError(error))

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt

@@ -51,6 +51,7 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
     url: String,
     headers: Map<String, String>,
     requestId: Long,
+    encoded: Boolean,
     callback: (Result<Map<String, Long>?>) -> Unit
   ) {
     val signal = CancellationSignal()

mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt

@@ -92,7 +92,8 @@ data class PlatformAsset (
   val isFavorite: Boolean,
   val adjustmentTime: Long? = null,
   val latitude: Double? = null,
-  val longitude: Double? = null
+  val longitude: Double? = null,
+  val playbackStyle: Long
 )
  {
   companion object {
@@ -110,7 +111,8 @@ data class PlatformAsset (
       val adjustmentTime = pigeonVar_list[10] as Long?
       val latitude = pigeonVar_list[11] as Double?
       val longitude = pigeonVar_list[12] as Double?
-      return PlatformAsset(id, name, type, createdAt, updatedAt, width, height, durationInSeconds, orientation, isFavorite, adjustmentTime, latitude, longitude)
+      val playbackStyle = pigeonVar_list[13] as Long
+      return PlatformAsset(id, name, type, createdAt, updatedAt, width, height, durationInSeconds, orientation, isFavorite, adjustmentTime, latitude, longitude, playbackStyle)
     }
   }
   fun toList(): List<Any?> {
@@ -128,6 +130,7 @@ data class PlatformAsset (
       adjustmentTime,
       latitude,
       longitude,
+      playbackStyle,
     )
   }
   override fun equals(other: Any?): Boolean {

mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt

@@ -59,6 +59,7 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
       add(MediaStore.MediaColumns.HEIGHT)
       add(MediaStore.MediaColumns.DURATION)
       add(MediaStore.MediaColumns.ORIENTATION)
+      add(MediaStore.MediaColumns.MIME_TYPE)
       // IS_FAVORITE is only available on Android 11 and above
       if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) {
         add(MediaStore.MediaColumns.IS_FAVORITE)
@@ -108,6 +109,7 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
         val durationColumn = c.getColumnIndexOrThrow(MediaStore.MediaColumns.DURATION)
         val orientationColumn =
           c.getColumnIndexOrThrow(MediaStore.MediaColumns.ORIENTATION)
+        val mimeTypeColumn = c.getColumnIndexOrThrow(MediaStore.MediaColumns.MIME_TYPE)
         val favoriteColumn = c.getColumnIndex(MediaStore.MediaColumns.IS_FAVORITE)
 
         while (c.moveToNext()) {
@@ -141,8 +143,15 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
           val duration = if (mediaType == MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE) 0
           else c.getLong(durationColumn) / 1000
           val orientation = c.getInt(orientationColumn)
+          val mimeType = c.getStringOrNull(mimeTypeColumn)
           val isFavorite = if (favoriteColumn == -1) false else c.getInt(favoriteColumn) != 0
 
+          val playbackStyle: Long = when {
+            mediaType == 2 -> 1L           // video
+            mimeType == "image/gif" -> 2L  // animated
+            else -> 0L                     // static image
+          }
+
           val asset = PlatformAsset(
             id,
             name,
@@ -154,6 +163,7 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin() {
             duration,
             orientation.toLong(),
             isFavorite,
+            playbackStyle = playbackStyle,
           )
           yield(AssetResult.ValidAsset(asset, bucketId))
         }

mobile/ios/Runner/Images/LocalImages.g.swift

@@ -70,7 +70,7 @@ class LocalImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
 
 /// Generated protocol from Pigeon that represents a handler of messages from Flutter.
 protocol LocalImageApi {
-  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
+  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, encoded: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
   func cancelRequest(requestId: Int64) throws
   func getThumbhash(thumbhash: String, completion: @escaping (Result<[String: Int64], Error>) -> Void)
 }
@@ -90,7 +90,8 @@ class LocalImageApiSetup {
         let widthArg = args[2] as! Int64
         let heightArg = args[3] as! Int64
         let isVideoArg = args[4] as! Bool
-        api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg, isVideo: isVideoArg) { result in
+        let encodedArg = args[5] as! Bool
+        api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg, isVideo: isVideoArg, encoded: encodedArg) { result in
           switch result {
           case .success(let res):
             reply(wrapResult(res))

mobile/ios/Runner/Images/LocalImagesImpl.swift

@@ -7,7 +7,7 @@ class LocalImageRequest {
   weak var workItem: DispatchWorkItem?
   var isCancelled = false
   let callback: (Result<[String: Int64]?, any Error>) -> Void
-  
+
   init(callback: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
     self.callback = callback
   }
@@ -30,11 +30,11 @@ class LocalImageApiImpl: LocalImageApi {
     requestOptions.version = .current
     return requestOptions
   }()
-  
+
   private static let assetQueue = DispatchQueue(label: "thumbnail.assets", qos: .userInitiated)
   private static let requestQueue = DispatchQueue(label: "thumbnail.requests", qos: .userInitiated)
   private static let cancelQueue = DispatchQueue(label: "thumbnail.cancellation", qos: .default)
-  
+
   private static var rgbaFormat = vImage_CGImageFormat(
     bitsPerComponent: 8,
     bitsPerPixel: 32,
@@ -48,12 +48,12 @@ class LocalImageApiImpl: LocalImageApi {
     assetCache.countLimit = 10000
     return assetCache
   }()
-  
+
   func getThumbhash(thumbhash: String, completion: @escaping (Result<[String : Int64], any Error>) -> Void) {
     ImageProcessing.queue.async {
       guard let data = Data(base64Encoded: thumbhash)
       else { return completion(.failure(PigeonError(code: "", message: "Invalid base64 string: \(thumbhash)", details: nil)))}
-      
+
       let (width, height, pointer) = thumbHashToRGBA(hash: data)
       completion(.success([
         "pointer": Int64(Int(bitPattern: pointer.baseAddress)),
@@ -63,34 +63,77 @@ class LocalImageApiImpl: LocalImageApi {
       ]))
     }
   }
-  
-  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
+
+  func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, encoded: Bool, completion: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
     let request = LocalImageRequest(callback: completion)
     let item = DispatchWorkItem {
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-      
+
       ImageProcessing.semaphore.wait()
       defer {
         ImageProcessing.semaphore.signal()
       }
-      
+
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-      
+
       guard let asset = Self.requestAsset(assetId: assetId)
       else {
         Self.remove(requestId: requestId)
         completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil)))
         return
       }
-      
+
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-
+
+      if encoded {
+        let dataOptions = PHImageRequestOptions()
+        dataOptions.isNetworkAccessAllowed = true
+        dataOptions.isSynchronous = true
+        dataOptions.version = .current
+
+        var imageData: Data?
+        Self.imageManager.requestImageDataAndOrientation(
+          for: asset,
+          options: dataOptions,
+          resultHandler: { (data, _, _, _) in
+            imageData = data
+          }
+        )
+
+        if request.isCancelled {
+          Self.remove(requestId: requestId)
+          return completion(ImageProcessing.cancelledResult)
+        }
+
+        guard let data = imageData else {
+          Self.remove(requestId: requestId)
+          return completion(.failure(PigeonError(code: "", message: "Could not get image data for \(assetId)", details: nil)))
+        }
+
+        let length = data.count
+        let pointer = malloc(length)!
+        data.copyBytes(to: pointer.assumingMemoryBound(to: UInt8.self), count: length)
+
+        if request.isCancelled {
+          free(pointer)
+          Self.remove(requestId: requestId)
+          return completion(ImageProcessing.cancelledResult)
+        }
+
+        request.callback(.success([
+          "pointer": Int64(Int(bitPattern: pointer)),
+          "length": Int64(length),
+        ]))
+        Self.remove(requestId: requestId)
+        return
+      }
+
       var image: UIImage?
       Self.imageManager.requestImage(
         for: asset,
@@ -101,29 +144,29 @@ class LocalImageApiImpl: LocalImageApi {
           image = _image
         }
       )
-      
+
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-      
+
       guard let image = image,
             let cgImage = image.cgImage else {
         Self.remove(requestId: requestId)
         return completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil)))
       }
-      
+
       if request.isCancelled {
         return completion(ImageProcessing.cancelledResult)
       }
-      
+
       do {
         let buffer = try vImage_Buffer(cgImage: cgImage, format: Self.rgbaFormat)
-        
+
         if request.isCancelled {
           buffer.free()
           return completion(ImageProcessing.cancelledResult)
         }
-        
+
         request.callback(.success([
           "pointer": Int64(Int(bitPattern: buffer.data)),
           "width": Int64(buffer.width),
@@ -136,24 +179,24 @@ class LocalImageApiImpl: LocalImageApi {
         return completion(.failure(PigeonError(code: "", message: "Failed to convert image for \(assetId): \(error)", details: nil)))
       }
     }
-    
+
     request.workItem = item
     Self.add(requestId: requestId, request: request)
     ImageProcessing.queue.async(execute: item)
   }
-  
+
   func cancelRequest(requestId: Int64) {
     Self.cancel(requestId: requestId)
   }
-  
+
   private static func add(requestId: Int64, request: LocalImageRequest) -> Void {
     requestQueue.sync { requests[requestId] = request }
   }
-  
+
   private static func remove(requestId: Int64) -> Void {
     requestQueue.sync { requests[requestId] = nil }
   }
-  
+
   private static func cancel(requestId: Int64) -> Void {
     requestQueue.async {
       guard let request = requests.removeValue(forKey: requestId) else { return }
@@ -164,12 +207,12 @@ class LocalImageApiImpl: LocalImageApi {
       }
     }
   }
-  
+
   private static func requestAsset(assetId: String) -> PHAsset? {
     var asset: PHAsset?
     assetQueue.sync { asset = assetCache.object(forKey: assetId as NSString) }
     if asset != nil { return asset }
-    
+
     guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions).firstObject
     else { return nil }
     assetQueue.async { assetCache.setObject(asset, forKey: assetId as NSString) }