Support native image encoding on native image formats (#1120)

* Support native image encoding on native image formats * Try to better handle CG objects
korlibs · Nov 22, 2022 · 5656959 · 5656959
1 parent ced7467
commit 5656959
Show file tree

Hide file tree

Showing 12 changed files with 226 additions and 76 deletions.
diff --git a/korim/src/androidMain/kotlin/com/soywiz/korim/format/NativeImageFormatProviderAndroid.kt b/korim/src/androidMain/kotlin/com/soywiz/korim/format/NativeImageFormatProviderAndroid.kt
@@ -15,10 +15,11 @@ import com.soywiz.korim.bitmap.Bitmap
 import com.soywiz.korim.color.*
 import com.soywiz.korim.paint.*
 import com.soywiz.korim.vector.*
-import com.soywiz.korio.android.androidContext
+import com.soywiz.korio.android.*
 import com.soywiz.korma.geom.*
 import com.soywiz.korma.geom.vector.*
 import kotlinx.coroutines.*
+import java.io.ByteArrayOutputStream
 
 actual val nativeImageFormatProvider: NativeImageFormatProvider by lazy {
     try {
@@ -31,6 +32,23 @@ actual val nativeImageFormatProvider: NativeImageFormatProvider by lazy {
 }
 
 object AndroidNativeImageFormatProvider : NativeImageFormatProvider() {
+    override suspend fun encodeSuspend(image: ImageDataContainer, props: ImageEncodingProps): ByteArray {
+        val compressFormat = when (props.mimeType) {
+            "image/png" -> android.graphics.Bitmap.CompressFormat.PNG
+            "image/jpeg", "image/jpg" -> android.graphics.Bitmap.CompressFormat.JPEG
+            "image/webp" -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+                android.graphics.Bitmap.CompressFormat.WEBP_LOSSY
+            } else {
+                android.graphics.Bitmap.CompressFormat.PNG
+            }
+            else -> android.graphics.Bitmap.CompressFormat.PNG
+        }
+        return ByteArrayOutputStream().use { bao ->
+            image.mainBitmap.toAndroidBitmap().compress(compressFormat, (props.quality * 100).toInt(), bao)
+            bao.toByteArray()
+        }
+    }
+
     override suspend fun display(bitmap: Bitmap, kind: Int) {
         val ctx = androidContext()
         val androidBitmap = bitmap.toAndroidBitmap()

diff --git a/korim/src/commonMain/kotlin/com/soywiz/korim/format/ImageDataContainer.kt b/korim/src/commonMain/kotlin/com/soywiz/korim/format/ImageDataContainer.kt
@@ -1,10 +1,15 @@
 package com.soywiz.korim.format
 
+import com.soywiz.korim.bitmap.*
+
 open class ImageDataContainer(
     val imageDatas: List<ImageData>
 ) {
+    constructor(bitmap: Bitmap) : this(listOf(ImageData(bitmap)))
+
     val imageDatasByName = imageDatas.associateBy { it.name }
     val default = imageDatasByName[null] ?: imageDatas.first()
+    val mainBitmap get() = default.mainBitmap
 
     operator fun get(name: String?): ImageData? = imageDatasByName[name]
 }
diff --git a/korim/src/commonMain/kotlin/com/soywiz/korim/format/ImageFormat.kt b/korim/src/commonMain/kotlin/com/soywiz/korim/format/ImageFormat.kt
@@ -1,22 +1,12 @@
 package com.soywiz.korim.format
 
-import com.soywiz.kds.Extra
-import com.soywiz.kds.ExtraType
-import com.soywiz.kmem.UByteArrayInt
-import com.soywiz.korim.bitmap.Bitmap
-import com.soywiz.korio.async.runBlockingNoSuspensionsNullable
-import com.soywiz.korio.file.VfsFile
-import com.soywiz.korio.file.baseName
-import com.soywiz.korio.lang.runIgnoringExceptions
-import com.soywiz.korio.stream.AsyncStream
-import com.soywiz.korio.stream.MemorySyncStreamToByteArray
-import com.soywiz.korio.stream.SyncStream
-import com.soywiz.korio.stream.openSync
-import com.soywiz.korio.stream.readAll
-import com.soywiz.korio.stream.sliceHere
-import com.soywiz.korio.stream.toAsync
-import com.soywiz.korio.stream.toSyncOrNull
-import kotlin.math.max
+import com.soywiz.kds.*
+import com.soywiz.korim.bitmap.*
+import com.soywiz.korio.async.*
+import com.soywiz.korio.file.*
+import com.soywiz.korio.lang.*
+import com.soywiz.korio.stream.*
+import kotlin.math.*
 
 abstract class ImageFormatWithContainer(vararg exts: String) : ImageFormat(*exts) {
     override fun readImageContainer(s: SyncStream, props: ImageDecodingProps): ImageDataContainer = TODO()
@@ -35,15 +25,32 @@ interface ImageFormatDecoder {
     suspend fun decodeSuspend(data: ByteArray, props: ImageDecodingProps = ImageDecodingProps.DEFAULT): Bitmap
 }
 
-abstract class ImageFormat(vararg exts: String) : ImageFormatDecoder {
+interface ImageFormatEncoder {
+    suspend fun encodeSuspend(
+        image: ImageDataContainer,
+        props: ImageEncodingProps = ImageEncodingProps("unknown"),
+    ): ByteArray = throw UnsupportedOperationException()
+}
+
+suspend fun ImageFormatEncoder.encodeSuspend(
+    bitmap: Bitmap,
+    props: ImageEncodingProps = ImageEncodingProps("unknown"),
+): ByteArray = encodeSuspend(ImageDataContainer(bitmap), props)
+
+
+interface ImageFormatEncoderDecoder : ImageFormatEncoder, ImageFormatDecoder
+
+abstract class ImageFormat(vararg exts: String) : ImageFormatEncoderDecoder {
 	val extensions = exts.map { it.toLowerCase().trim() }.toSet()
     open fun readImageContainer(s: SyncStream, props: ImageDecodingProps = ImageDecodingProps.DEFAULT): ImageDataContainer = ImageDataContainer(listOf(readImage(s, props)))
 	open fun readImage(s: SyncStream, props: ImageDecodingProps = ImageDecodingProps.DEFAULT): ImageData = TODO()
-	open fun writeImage(
-		image: ImageData,
-		s: SyncStream,
-		props: ImageEncodingProps = ImageEncodingProps("unknown")
-	): Unit = throw UnsupportedOperationException()
+	open fun writeImage(image: ImageData, s: SyncStream, props: ImageEncodingProps): Unit = throw UnsupportedOperationException()
+
+    override suspend fun encodeSuspend(image: ImageDataContainer, props: ImageEncodingProps): ByteArray {
+        val out = MemorySyncStream()
+        writeImage(image.default, out, props)
+        return out.toByteArray()
+    }
 
     open suspend fun decodeHeaderSuspend(s: AsyncStream, props: ImageDecodingProps = ImageDecodingProps.DEFAULT): ImageInfo? {
         return decodeHeader(s.toSyncOrNull() ?: s.readAll().openSync())
@@ -169,6 +176,17 @@ data class ImageEncodingProps(
     override var extra: ExtraType = null,
     val init: (ImageEncodingProps.() -> Unit)? = null
 ) : Extra {
+    val extension: String get() = PathInfo(filename).extensionLC
+    val mimeType: String get() = when (extension) {
+        "jpg", "jpeg" -> "image/jpeg"
+        "png" -> "image/png"
+        "gif" -> "image/gif"
+        "webp" -> "image/webp"
+        "avif" -> "image/avif"
+        "heic" -> "image/heic"
+        else -> "image/png"
+    }
+
     init {
         init?.invoke(this)
     }

diff --git a/korim/src/commonMain/kotlin/com/soywiz/korim/format/NativeImageFormatProvider.kt b/korim/src/commonMain/kotlin/com/soywiz/korim/format/NativeImageFormatProvider.kt
@@ -1,23 +1,13 @@
 package com.soywiz.korim.format
 
-import com.soywiz.korim.bitmap.Bitmap
-import com.soywiz.korim.bitmap.Bitmap32
-import com.soywiz.korim.bitmap.BmpSlice
-import com.soywiz.korim.bitmap.NativeImage
-import com.soywiz.korim.bitmap.asumePremultiplied
-import com.soywiz.korim.bitmap.context2d
-import com.soywiz.korim.bitmap.ensureNative
-import com.soywiz.korim.bitmap.extract
-import com.soywiz.korim.color.RGBA
-import com.soywiz.korim.vector.Context2d
-import com.soywiz.korim.vector.SizedDrawable
-import com.soywiz.korim.vector.render
-import com.soywiz.korio.file.FinalVfsFile
-import com.soywiz.korio.file.Vfs
-import com.soywiz.korio.file.VfsFile
-import kotlinx.coroutines.CancellationException
-import kotlin.math.ceil
-import kotlin.native.concurrent.ThreadLocal
+import com.soywiz.korim.bitmap.*
+import com.soywiz.korim.color.*
+import com.soywiz.korim.vector.*
+import com.soywiz.korio.file.*
+import com.soywiz.korio.stream.*
+import kotlinx.coroutines.*
+import kotlin.math.*
+import kotlin.native.concurrent.*
 
 @ThreadLocal
 expect val nativeImageFormatProvider: NativeImageFormatProvider
@@ -28,7 +18,7 @@ data class NativeImageResult(
     val originalHeight: Int = image.height,
 )
 
-abstract class NativeImageFormatProvider : ImageFormatDecoder {
+abstract class NativeImageFormatProvider : ImageFormatEncoderDecoder {
     protected open suspend fun decodeHeaderInternal(data: ByteArray): ImageInfo {
         val result = decodeInternal(data, ImageDecodingProps.DEFAULT)
         return ImageInfo().also {
@@ -37,6 +27,8 @@ abstract class NativeImageFormatProvider : ImageFormatDecoder {
         }
     }
 
+    override suspend fun encodeSuspend(image: ImageDataContainer, props: ImageEncodingProps): ByteArray = throw UnsupportedOperationException()
+
     protected abstract suspend fun decodeInternal(data: ByteArray, props: ImageDecodingProps): NativeImageResult
     protected open suspend fun decodeInternal(vfs: Vfs, path: String, props: ImageDecodingProps): NativeImageResult = decodeInternal(vfs.file(path).readBytes(), props)
 
@@ -59,6 +51,7 @@ abstract class NativeImageFormatProvider : ImageFormatDecoder {
     suspend fun decode(data: ByteArray, props: ImageDecodingProps = ImageDecodingProps.DEFAULT): NativeImage = decodeInternal(data, props).image
     override suspend fun decodeSuspend(data: ByteArray, props: ImageDecodingProps): NativeImage = decodeInternal(data, props).image
     suspend fun decode(file: FinalVfsFile, props: ImageDecodingProps): Bitmap = decodeInternal(file.vfs, file.path, props).image
+
     override suspend fun decode(file: VfsFile, props: ImageDecodingProps): Bitmap = decode(file.getUnderlyingUnscapedFile(), props)
 
     suspend fun decode(vfs: Vfs, path: String, premultiplied: Boolean = true): NativeImage = decode(vfs, path, ImageDecodingProps.DEFAULT(premultiplied))

diff --git a/korim/src/commonTest/kotlin/com/soywiz/korim/format/NativeEncodingTest.kt b/korim/src/commonTest/kotlin/com/soywiz/korim/format/NativeEncodingTest.kt
@@ -0,0 +1,21 @@
+package com.soywiz.korim.format
+
+import com.soywiz.kmem.*
+import com.soywiz.korim.bitmap.*
+import com.soywiz.korim.color.*
+import com.soywiz.korio.async.*
+import com.soywiz.korio.stream.*
+import com.soywiz.korma.geom.*
+import kotlin.test.*
+
+class NativeEncodingTest {
+    @Test
+    fun test() = suspendTest {
+        if (Platform.isJsNodeJs) RegisteredImageFormats.register(PNG)
+        val bytes = nativeImageFormatProvider.encodeSuspend(Bitmap32(10, 10, Colors.RED), ImageEncodingProps("image.png"))
+        assertEquals(Size(10, 10), PNG.decodeHeader(bytes.openSync())!!.size)
+
+        val image = nativeImageFormatProvider.decodeSuspend(bytes)
+        assertEquals(Colors.RED, image.toBMP32()[0, 0])
+    }
+}
diff --git a/korim/src/darwinMain/kotlin/com/soywiz/korim/format/cg/CGImageExt.kt b/korim/src/darwinMain/kotlin/com/soywiz/korim/format/cg/CGImageExt.kt
@@ -0,0 +1,25 @@
+package com.soywiz.korim.format.cg
+
+import com.soywiz.korim.bitmap.*
+import kotlinx.cinterop.*
+import platform.CoreGraphics.*
+
+fun Bitmap32.toCGImage(): CGImageRef? {
+    return transferBitmap32ToCGImage(this, null)
+}
+
+fun CGImageRef.toBitmap32(): Bitmap32 {
+    val image = this
+    val width = CGImageGetWidth(image).toInt()
+    val height = CGImageGetHeight(image).toInt()
+    val colorSpace = CGColorSpaceCreateDeviceRGB()
+    val ctx = CGBitmapContextCreate(
+        null, width.convert(), height.convert(),
+        8.convert(), 0.convert(), colorSpace,
+        CGImageAlphaInfo.kCGImageAlphaPremultipliedLast.value
+    )
+    CGContextDrawImage(ctx, CGRectMake(0.cg, 0.cg, width.cg, height.cg), image)
+    val out = Bitmap32(width, height, premultiplied = true)
+    transferBitmap32CGContext(out, ctx, toBitmap = true)
+    return out
+}
diff --git a/korim/src/darwinMain/kotlin/com/soywiz/korim/format/cg/CGNativeImageFormatProvider.kt b/korim/src/darwinMain/kotlin/com/soywiz/korim/format/cg/CGNativeImageFormatProvider.kt
@@ -1,13 +1,18 @@
 package com.soywiz.korim.format.cg
 
+import cnames.structs.CGImage
 import com.soywiz.kmem.*
 import com.soywiz.korim.bitmap.*
 import com.soywiz.korim.format.*
 import kotlinx.cinterop.*
 import platform.CoreFoundation.*
 import platform.CoreGraphics.*
+import platform.CoreServices.*
 import platform.ImageIO.*
-import platform.posix.memcpy
+import platform.posix.*
+import kotlin.Boolean
+import kotlin.ByteArray
+import kotlin.Int
 import kotlin.native.concurrent.*
 
 open class CGBaseNativeImageFormatProvider : StbImageNativeImageFormatProvider() {
@@ -155,4 +160,50 @@ open class CGNativeImageFormatProvider : CGBaseNativeImageFormatProvider() {
             vvar.value
         }
     }
+
+    override suspend fun encodeSuspend(image: ImageDataContainer, props: ImageEncodingProps): ByteArray = memScoped {
+        val data = CFDataCreateMutable(null, 0)
+        val destination = CGImageDestinationCreateWithData(data, when (props.mimeType) {
+            "image/jpeg", "image/jpg" -> kUTTypeJPEG
+            //"image/heif", "image/heic" -> UTTypeHEIF
+            else -> kUTTypePNG
+        }, 1, null)
+            ?: error("Failed to create CGImageDestination")
+
+        val imageProperties = CFDictionaryCreateMutable(null, 0, null, null)
+        val ref = alloc<DoubleVar>()
+        ref.value = props.quality
+        val num = CFNumberCreate(null, kCFNumberDoubleType, ref.ptr)
+        CFDictionaryAddValue(imageProperties, kCGImageDestinationLossyCompressionQuality, num)
+
+        //println("CGNativeImageFormatProvider.encodeSuspend")
+        val cgImage = image.mainBitmap.toBMP32().toCGImage()
+
+        try {
+            CGImageDestinationAddImage(destination, cgImage, imageProperties)
+            if (!CGImageDestinationFinalize(destination)) error("Can't write image")
+        } finally {
+            CGImageRelease(cgImage)
+            CFRelease(imageProperties)
+            CFRelease(num)
+            CFRelease(destination)
+        }
+        val length: Int = CFDataGetLength(data).convert()
+        val bytes = CFDataGetMutableBytePtr(data)?.readBytes(length.convert())
+        CFRelease(data)
+        return bytes ?: error("Can't write image")
+    }
+}
+
+/*
+fun Map<*, *>.toCFDictionary(): CFDictionaryRef = memScoped {
+    val dict = CFDictionaryCreateMutable(null, 0, null, null)
+    for ((key, value) in this) {
+        val ref = alloc<DoubleVar>()
+        ref.value = value.todo
+        CFNumberCreate(null, kCFNumberDoubleType, null, ref.ptr)
+        CFDictionaryAddValue()
+    }
+    return dict
 }
+*/
diff --git a/korim/src/darwinMobileMain/kotlin/com/soywiz/korim/format/NativeImageFormatProviderIos.kt b/korim/src/darwinMobileMain/kotlin/com/soywiz/korim/format/NativeImageFormatProviderIos.kt
@@ -1,11 +1,26 @@
 package com.soywiz.korim.format
 
-import com.soywiz.korim.format.cg.CGNativeImageFormatProvider
+import com.soywiz.korim.format.cg.*
+import com.soywiz.korim.format.ui.*
+import kotlinx.cinterop.*
+import platform.UIKit.*
 
 //actual val nativeImageFormatProvider: NativeImageFormatProvider = UIImageNativeImageFormatProvider
-actual val nativeImageFormatProvider: NativeImageFormatProvider get() = CGNativeImageFormatProvider
+actual val nativeImageFormatProvider: NativeImageFormatProvider get() = UINativeImageFormatProvider
 //actual val nativeImageFormatProvider: NativeImageFormatProvider get() = CGBaseNativeImageFormatProvider
 
+object UINativeImageFormatProvider : CGNativeImageFormatProvider() {
+    @OptIn(UnsafeNumber::class)
+    override suspend fun encodeSuspend(image: ImageDataContainer, props: ImageEncodingProps): ByteArray {
+        val uiImage: UIImage = image.mainBitmap.toBMP32().toUIImage()
+        //println("Using UINativeImageFormatProvider.encodeSuspend")
+        return when (props.mimeType) {
+            "image/jpeg", "image/jpg" -> UIImageJPEGRepresentation(uiImage, compressionQuality = props.quality.cg)?.toByteArray()
+            else -> UIImagePNGRepresentation(uiImage)?.toByteArray()
+        } ?: error("Can't encode using UIImageRepresentation")
+    }
+}
+
 
 /*
 object UIImageNativeImageFormatProvider : BaseNativeImageFormatProvider() {

diff --git a/.../com/soywiz/korim/format/ui/UIImageExt.kt → .../com/soywiz/korim/format/ui/UIImageExt.kt b/.../com/soywiz/korim/format/ui/UIImageExt.kt → .../com/soywiz/korim/format/ui/UIImageExt.kt
@@ -1,5 +1,6 @@
 package com.soywiz.korim.format.ui
 
+import cnames.structs.CGContext
 import com.soywiz.korim.bitmap.*
 import com.soywiz.korim.format.cg.*
 import kotlinx.cinterop.*