Improve zip building: support compressing a folder into a file directly, support compression methods with levels, deflate & lzma in zip files, and filter a AsyncOutputStream.withChecksumUpdater

korge-core/src/korlibs/io/compression/CompressionMethod.kt

@@ -1,5 +1,6 @@
 package korlibs.io.compression
 
+import korlibs.datastructure.*
 import korlibs.io.async.*
 import korlibs.io.compression.util.BitReader
 import korlibs.io.experimental.KorioExperimentalApi
@@ -19,18 +20,28 @@ open class CompressionContext(var level: Int = 6) {
 	var custom: Any? = null
 }
 
+data class CompressionMethodWithConfig(val method: CompressionMethod, override val level: Int) : CompressionMethod by method, Extra by Extra.Mixin()
+
+fun CompressionMethod.withLevel(level: Int): CompressionMethodWithConfig = CompressionMethodWithConfig(this, level)
+
 interface CompressionMethod {
+    val name: String get() = "UNKNOWN"
+
+    val level: Int get() = 6
+
 	@KorioExperimentalApi
 	suspend fun uncompress(reader: BitReader, out: AsyncOutputStream): Unit = unsupported()
 
 	@KorioExperimentalApi
 	suspend fun compress(
 		i: BitReader,
 		o: AsyncOutputStream,
-		context: CompressionContext = CompressionContext()
+		context: CompressionContext = CompressionContext(level = this.level)
 	): Unit = unsupported()
 
 	object Uncompressed : CompressionMethod {
+        override val name: String get() = "STORE"
+
 		@OptIn(KorioExperimentalApi::class)
 		override suspend fun uncompress(reader: BitReader, out: AsyncOutputStream) { reader.copyTo(out) }
 		@OptIn(KorioExperimentalApi::class)
@@ -41,7 +52,7 @@ interface CompressionMethod {
 @OptIn(KorioExperimentalApi::class)
 suspend fun CompressionMethod.uncompress(i: AsyncInputStream, o: AsyncOutputStream): Unit = uncompress(BitReader.forInput(i), o)
 @OptIn(KorioExperimentalApi::class)
-suspend fun CompressionMethod.compress(i: AsyncInputStream, o: AsyncOutputStream, context: CompressionContext = CompressionContext()): Unit = compress(BitReader.forInput(i), o, context)
+suspend fun CompressionMethod.compress(i: AsyncInputStream, o: AsyncOutputStream, context: CompressionContext = CompressionContext(level = this.level)): Unit = compress(BitReader.forInput(i), o, context)
 
 suspend fun CompressionMethod.uncompressStream(input: AsyncInputStream, bufferSize: Int = AsyncRingBufferChunked.DEFAULT_MAX_SIZE): AsyncInputStream =
     asyncStreamWriter(bufferSize, name = "uncompress:$this") { output -> uncompress(input, output) }
@@ -55,19 +66,19 @@ fun CompressionMethod.uncompress(i: SyncInputStream, o: SyncOutputStream) = runB
 	uncompress(i.toAsyncInputStream(), o.toAsyncOutputStream())
 }
 
-fun CompressionMethod.compress(i: SyncInputStream, o: SyncOutputStream, context: CompressionContext = CompressionContext()) = runBlockingNoSuspensions {
+fun CompressionMethod.compress(i: SyncInputStream, o: SyncOutputStream, context: CompressionContext = CompressionContext(level = this.level)) = runBlockingNoSuspensions {
 	compress(i.toAsyncInputStream(), o.toAsyncOutputStream(), context)
 }
 
-fun CompressionMethod.compress(bytes: ByteArray, context: CompressionContext = CompressionContext(), outputSizeHint: Int = (bytes.size * 1.1).toInt()): ByteArray =
+fun CompressionMethod.compress(bytes: ByteArray, context: CompressionContext = CompressionContext(level = this.level), outputSizeHint: Int = (bytes.size * 1.1).toInt()): ByteArray =
     MemorySyncStreamToByteArray(outputSizeHint) { [email protected](bytes.openSync(), this, context) }
 fun CompressionMethod.uncompress(bytes: ByteArray, outputSizeHint: Int = bytes.size * 2): ByteArray =
     MemorySyncStreamToByteArray(outputSizeHint) { [email protected](bytes.openSync(), this) }
 
 fun ByteArray.uncompress(method: CompressionMethod, outputSizeHint: Int = this.size * 2): ByteArray =
     method.uncompress(this, outputSizeHint)
-fun ByteArray.compress(method: CompressionMethod, context: CompressionContext = CompressionContext(), outputSizeHint: Int = (this.size * 1.1).toInt()): ByteArray =
+fun ByteArray.compress(method: CompressionMethod, context: CompressionContext = CompressionContext(level = method.level), outputSizeHint: Int = (this.size * 1.1).toInt()): ByteArray =
 	method.compress(this, context, outputSizeHint)
 
 suspend fun AsyncInputStream.uncompressed(method: CompressionMethod, bufferSize: Int = AsyncRingBufferChunked.DEFAULT_MAX_SIZE): AsyncInputStream = method.uncompressStream(this, bufferSize)
-suspend fun AsyncInputStream.compressed(method: CompressionMethod, context: CompressionContext = CompressionContext(), bufferSize: Int = AsyncByteArrayDequeChunked.DEFAULT_MAX_SIZE): AsyncInputStream = method.compressStream(this, context, bufferSize)
+suspend fun AsyncInputStream.compressed(method: CompressionMethod, context: CompressionContext = CompressionContext(level = method.level), bufferSize: Int = AsyncByteArrayDequeChunked.DEFAULT_MAX_SIZE): AsyncInputStream = method.compressStream(this, context, bufferSize)

korge-core/src/korlibs/io/compression/deflate/Deflate.kt

@@ -23,7 +23,9 @@ val Deflate: CompressionMethod by lazy { Deflate(15) }
 
 @OptIn(KorioExperimentalApi::class)
 open class DeflatePortable(val windowBits: Int) : CompressionMethod {
-	override suspend fun compress(
+    override val name: String get() = "DEFLATE"
+
+    override suspend fun compress(
 		i: BitReader,
 		o: AsyncOutputStream,
 		context: CompressionContext

korge-core/src/korlibs/io/compression/deflate/GZIP.kt

@@ -25,6 +25,8 @@ open class GZIPNoCrc(deflater: () -> CompressionMethod) : GZIPBase(false, deflat
 
 @OptIn(KorioExperimentalApi::class)
 open class GZIPBase(val checkCrc: Boolean, val deflater: () -> CompressionMethod) : CompressionMethod {
+    override val name: String get() = "GZIP"
+
     override fun toString(): String = "GZIPBase($checkCrc, ${deflater})"
 
 	override suspend fun uncompress(reader: BitReader, out: AsyncOutputStream) {

korge-core/src/korlibs/io/compression/deflate/ZLib.kt

@@ -16,6 +16,8 @@ import korlibs.encoding.hex
 
 @OptIn(KorioExperimentalApi::class)
 open class ZLib(val deflater: (windowBits: Int) -> CompressionMethod) : CompressionMethod {
+    override val name: String get() = "ZLIB"
+
 	companion object : ZLib({ Deflate(it) })
 
     object Portable : ZLib({ DeflatePortable(it) })

korge-core/src/korlibs/io/compression/lzma/Lzma.kt

@@ -18,6 +18,8 @@ import korlibs.io.stream.writeBytes
  */
 @OptIn(KorioExperimentalApi::class)
 object Lzma : CompressionMethod {
+    override val name: String get() = "LZMA"
+
 	override suspend fun uncompress(reader: BitReader, out: AsyncOutputStream) {
 		val input = reader.readAll().openSync()
 		val properties = input.readBytesExact(5)

korge-core/src/korlibs/io/compression/lzo/LZO.kt

@@ -27,6 +27,8 @@ import korlibs.encoding.hex
 
 // @TODO: We might want to support a raw version without headers?
 open class LZO(val headerType: HeaderType = HeaderType.SHORT) : CompressionMethod {
+    override val name: String get() = "LZO"
+
     companion object : LZO(headerType = HeaderType.SHORT);
 
     enum class HeaderType { NONE, SHORT, LONG }

korge-core/src/korlibs/io/compression/zip/ZipBuilder.kt

@@ -1,39 +1,43 @@
 package korlibs.io.compression.zip
 
-import korlibs.memory.ByteArrayBuilder
-import korlibs.io.file.VfsFile
-import korlibs.io.file.fullName
-import korlibs.io.file.std.createZipFromTreeTo
-import korlibs.io.lang.UTF8
-import korlibs.io.lang.toByteArray
-import korlibs.io.stream.AsyncStream
-import korlibs.io.stream.MemorySyncStream
-import korlibs.io.stream.toAsync
-import korlibs.io.stream.write16LE
-import korlibs.io.stream.write32LE
-import korlibs.io.stream.writeBytes
-import korlibs.io.stream.writeFile
-import korlibs.io.stream.writeString
-import korlibs.io.stream.writeSync
-import korlibs.io.util.checksum.CRC32
-import korlibs.io.util.checksum.checksum
+import korlibs.io.compression.*
+import korlibs.io.file.*
+import korlibs.io.lang.*
+import korlibs.io.stream.*
+import korlibs.io.util.checksum.*
+import korlibs.memory.*
 
 class ZipBuilder {
     companion object {
-        suspend fun createZipFromTree(file: VfsFile): ByteArray {
-            val buf = ByteArrayBuilder()
-            val mem = MemorySyncStream(buf)
-            file.createZipFromTreeTo(mem.toAsync())
-            return buf.toByteArray()
+        suspend fun createZipFromTree(file: VfsFile, compression: CompressionMethod = CompressionMethod.Uncompressed, useFolderAsRoot: Boolean = false): ByteArray = buildByteArray {
+            createZipFromTreeTo(file, MemorySyncStream(this).toAsync(), compression, useFolderAsRoot)
         }
 
-        suspend fun createZipFromTreeTo(file: VfsFile, s: AsyncStream) {
+        suspend fun createZipFromTreeTo(
+            folder: VfsFile,
+            zipFile: VfsFile,
+            compression: CompressionMethod = CompressionMethod.Uncompressed,
+            useFolderAsRoot: Boolean = true
+        ): VfsFile {
+            zipFile.openUse(VfsOpenMode.CREATE_OR_TRUNCATE) {
+            //zipFile.openUse(VfsOpenMode.CREATE) {
+                createZipFromTreeTo(folder, this, compression, useFolderAsRoot)
+            }
+            return zipFile
+        }
+
+        suspend fun createZipFromTreeTo(
+            file: VfsFile,
+            s: AsyncStream,
+            compression: CompressionMethod = CompressionMethod.Uncompressed,
+            useFolderAsRoot: Boolean = false
+        ) {
             val entries = arrayListOf<ZipEntry>()
-            ZipBuilder.addZipFileEntryTree(s, file, entries)
+            addZipFileEntryTree(s, if (useFolderAsRoot) file.jail() else file, entries, compression)
             val directoryStart = s.position
 
             for (entry in entries) {
-                ZipBuilder.addDirEntry(s, entry)
+                addDirEntry(s, entry)
             }
             val directoryEnd = s.position
             val comment = byteArrayOf()
@@ -51,23 +55,35 @@ class ZipBuilder {
             }
         }
 
-        suspend fun addZipFileEntry(s: AsyncStream, entry: VfsFile): ZipEntry {
+        val CompressionMethod.zipId: Int get() = when (this.name) {
+            "STORE" -> 0
+            "DEFLATE" -> 8
+            "LZMA" -> 14
+            else -> TODO("Unknown '${this.name}' compression method for zip ($this)")
+        }
+
+        suspend fun addZipFileEntry(s: AsyncStream, entry: VfsFile, compression: CompressionMethod = CompressionMethod.Uncompressed): ZipEntry {
             val size = entry.size().toInt()
             val versionMadeBy = 0x314
             val extractVersion = 10
             val flags = 2048
-            //val compressionMethod = 8 // Deflate
-            val compressionMethod = 0 // Store
+            val compressionMethod = compression.zipId
+            val compressed = compressionMethod != 0
             val date = 0
             val time = 0
-            val crc32 = entry.checksum(CRC32)
+            //var crc32 = if (compressed) 0 else entry.checksum(CRC32)
+            var crc32 = entry.checksum(CRC32)
             val name = entry.fullName.trim('/')
             val nameBytes = name.toByteArray(UTF8)
             val extraBytes = byteArrayOf()
-            val compressedSize = size
+            var compressedSize = if (compressed) 0 else size
             val uncompressedSize = size
 
             val headerOffset = s.position
+
+            var compressedPos = 0L
+
+            compressedPos = s.position + 14
             s.writeSync {
                 writeString("PK\u0003\u0004")
                 write16LE(extractVersion)
@@ -83,7 +99,23 @@ class ZipBuilder {
                 writeBytes(nameBytes)
                 writeBytes(extraBytes)
             }
-            s.writeFile(entry)
+            if (compressed) {
+                val startPos = s.position
+                //val crcUpdater = CRC32.updater()
+                entry.openUseIt {
+                    //compression.compress(it, s.withChecksumUpdater(crcUpdater))
+                    compression.compress(it, s)
+                }
+                //crc32 = crcUpdater.current
+                val endPos = s.position
+                compressedSize = (endPos - startPos).toInt()
+                s.sliceWithSize(compressedPos, 8).also {
+                    it.write32LE(crc32)
+                    it.write32LE(compressedSize)
+                }
+            } else {
+                s.writeFile(entry)
+            }
 
             return ZipEntry(
                 versionMadeBy = versionMadeBy,
@@ -105,11 +137,11 @@ class ZipBuilder {
             )
         }
 
-        suspend fun addZipFileEntryTree(s: AsyncStream, entry: VfsFile, entries: MutableList<ZipEntry>) {
+        suspend fun addZipFileEntryTree(s: AsyncStream, entry: VfsFile, entries: MutableList<ZipEntry>, compression: CompressionMethod = CompressionMethod.Uncompressed) {
             if (entry.isDirectory()) {
-                entry.list().collect { addZipFileEntryTree(s, it, entries) }
+                entry.list().collect { addZipFileEntryTree(s, it, entries, compression) }
             } else {
-                entries += addZipFileEntry(s, entry)
+                entries += addZipFileEntry(s, entry, compression)
             }
         }
 

korge-core/src/korlibs/io/file/VfsFile.kt

@@ -76,6 +76,7 @@ data class VfsFile(
 	override suspend fun openRead(): AsyncStream = open(VfsOpenMode.READ)
 
 	suspend inline fun <T> openUse(mode: VfsOpenMode = VfsOpenMode.READ, callback: AsyncStream.() -> T): T = open(mode).use(callback)
+    suspend inline fun <T> openUseIt(mode: VfsOpenMode = VfsOpenMode.READ, callback: (AsyncStream) -> T): T = open(mode).use(callback)
 
 	suspend fun readRangeBytes(range: LongRange): ByteArray = vfs.readRange(this.path, range)
 	suspend fun readRangeBytes(range: IntRange): ByteArray = vfs.readRange(this.path, range.toLongRange())

korge-core/src/korlibs/io/file/std/ZipVfs.kt

@@ -1,8 +1,8 @@
 package korlibs.io.file.std
 
+import korlibs.io.compression.*
 import korlibs.io.compression.deflate.Deflate
 import korlibs.io.compression.deflate.DeflatePortable
-import korlibs.io.compression.uncompressed
 import korlibs.io.compression.zip.ZipBuilder
 import korlibs.io.compression.zip.ZipEntry2
 import korlibs.io.compression.zip.ZipFile
@@ -33,8 +33,14 @@ suspend fun ZipVfs(
     caseSensitive: Boolean = true,
     closeStream: Boolean = false,
     useNativeDecompression: Boolean = true,
-    fullName: String? = null
+    fullName: String? = null,
+    compressionMethods: List<CompressionMethod>? = null,
 ): VfsFile {
+    val compressionMethods = (compressionMethods ?: emptyList()) + listOf(CompressionMethod.Uncompressed, if (useNativeDecompression) Deflate else DeflatePortable)
+    val algosByName = compressionMethods.associateBy { it.name }
+
+    val algoIdToName = mapOf(0 to "STORE", 8 to "DEFLATE", 14 to "LZMA")
+
     //val s = zipFile.open(VfsOpenMode.READ)
     val zipFile = ZipFile(s, caseSensitive, fullName ?: vfsFile?.fullName)
 
@@ -78,10 +84,8 @@ suspend fun ZipVfs(
                     0 -> compressedData
                     else -> {
                         //println("[ZipVfs].open[5]")
-                        val method = when (entry.compressionMethod) {
-                            8 -> if (useNativeDecompression) Deflate else DeflatePortable
-                            else -> TODO("Not implemented zip method ${entry.compressionMethod}")
-                        }
+                        val algoName = algoIdToName[entry.compressionMethod]
+                        val method = algosByName[algoName] ?: TODO("Not implemented zip method ${entry.compressionMethod} with name '$algoName' not provided as compressionMethods")
                         compressedData.uncompressed(method).withLength(entry.uncompressedSize).toAsyncStream()
                         //val compressed = compressedData.uncompressed(method).readAll()
                         //val compressed = compressedData.readAll().uncompress(method)
@@ -112,14 +116,14 @@ suspend fun ZipVfs(
     return Impl().root
 }
 
-suspend fun VfsFile.openAsZip(caseSensitive: Boolean = true, useNativeDecompression: Boolean = true): VfsFile =
-    ZipVfs(this.open(VfsOpenMode.READ), this, caseSensitive = caseSensitive, closeStream = true, useNativeDecompression = useNativeDecompression)
+suspend fun VfsFile.openAsZip(caseSensitive: Boolean = true, useNativeDecompression: Boolean = true, compressionMethods: List<CompressionMethod>? = null): VfsFile =
+    ZipVfs(this.open(VfsOpenMode.READ), this, caseSensitive = caseSensitive, closeStream = true, useNativeDecompression = useNativeDecompression, compressionMethods = compressionMethods)
 
-suspend fun AsyncStream.openAsZip(caseSensitive: Boolean = true, useNativeDecompression: Boolean = true) =
-    ZipVfs(this, caseSensitive = caseSensitive, closeStream = false, useNativeDecompression = useNativeDecompression)
+suspend fun AsyncStream.openAsZip(caseSensitive: Boolean = true, useNativeDecompression: Boolean = true, compressionMethods: List<CompressionMethod>? = null) =
+    ZipVfs(this, caseSensitive = caseSensitive, closeStream = false, useNativeDecompression = useNativeDecompression, compressionMethods = compressionMethods)
 
-suspend fun <R> VfsFile.openAsZip(caseSensitive: Boolean = true, useNativeDecompression: Boolean = true, callback: suspend (VfsFile) -> R): R {
-    val file = openAsZip(caseSensitive, useNativeDecompression = useNativeDecompression)
+suspend fun <R> VfsFile.openAsZip(caseSensitive: Boolean = true, useNativeDecompression: Boolean = true, compressionMethods: List<CompressionMethod>? = null, callback: suspend (VfsFile) -> R): R {
+    val file = openAsZip(caseSensitive, useNativeDecompression = useNativeDecompression, compressionMethods = compressionMethods)
     try {
         return callback(file)
     } finally {
@@ -143,8 +147,24 @@ suspend fun <R> AsyncStream.openAsZip(caseSensitive: Boolean = true, useNativeDe
  * at Coroutine$await$lambda.doResume (korio.js:626:34)
  * at file:///Users/soywiz/projects/korlibs/korio/build/node_modules/korio.js:603:25
  */
-suspend fun VfsFile.createZipFromTree(): ByteArray = ZipBuilder.createZipFromTree(this)
-suspend fun VfsFile.createZipFromTreeTo(s: AsyncStream) = ZipBuilder.createZipFromTreeTo(this, s)
+suspend fun VfsFile.createZipFromTree(
+    useFolderAsRoot: Boolean = false, compression: CompressionMethod = CompressionMethod.Uncompressed): ByteArray = ZipBuilder.createZipFromTree(
+    this,
+    compression,
+    useFolderAsRoot
+)
+suspend fun VfsFile.createZipFromTreeTo(s: AsyncStream, compression: CompressionMethod = CompressionMethod.Uncompressed, useFolderAsRoot: Boolean = false) = ZipBuilder.createZipFromTreeTo(
+    this,
+    s,
+    compression,
+    useFolderAsRoot
+)
+suspend fun VfsFile.createZipFromTreeTo(zipFile: VfsFile, compression: CompressionMethod = CompressionMethod.Uncompressed, useFolderAsRoot: Boolean = true): VfsFile = ZipBuilder.createZipFromTreeTo(
+    this,
+    zipFile,
+    compression,
+    useFolderAsRoot
+)
 
 private fun ZipEntry2?.toStat(file: VfsFile): VfsStat {
     val vfs = file.vfs

korge-core/src/korlibs/io/stream/AsyncStream.kt

@@ -842,6 +842,9 @@ fun AsyncInputStream.withLength(length: Long): AsyncInputStream {
 	}
 }
 
+fun MemoryAsyncStream(data: korlibs.memory.ByteArrayBuilder): AsyncStream = MemoryAsyncStreamBase(data).toAsyncStream()
+fun MemoryAsyncStream(initialCapacity: Int = 4096): AsyncStream = MemoryAsyncStreamBase(initialCapacity).toAsyncStream()
+
 class MemoryAsyncStreamBase(var data: korlibs.memory.ByteArrayBuilder) : AsyncStreamBase() {
 	constructor(initialCapacity: Int = 4096) : this(ByteArrayBuilder(initialCapacity))