Some EXIF & AVIF/HEIC parsing fixes

soywiz-invideo · soywiz-invideo · commit 98ce0b5c74cb · 2022-03-04T13:14:47.000+01:00
diff --git a/korge-sandbox/src/commonMain/kotlin/Main.kt b/korge-sandbox/src/commonMain/kotlin/Main.kt
@@ -32,7 +32,8 @@ import kotlin.random.*
 suspend fun main() = Korge(bgcolor = Colors.DARKCYAN.mix(Colors.BLACK, 0.8), clipBorders = false
     //, debugAg = true
 ) {
-    mainVectorRendering()
+    mainExifTest()
+    //mainVectorRendering()
     //mainRenderText()
     //mainTextMetrics()
     //mainBitmapTexId()
diff --git a/korge-sandbox/src/commonMain/kotlin/MainExifTest.kt b/korge-sandbox/src/commonMain/kotlin/MainExifTest.kt
@@ -0,0 +1,11 @@
+import com.soywiz.korge.view.*
+import com.soywiz.korim.format.*
+import com.soywiz.korio.file.std.*
+
+suspend fun Stage.mainExifTest() {
+    //val info = resourcesVfs["IMG_5455.HEIC"].readImageInfo(HEICInfo)
+    //val info = localVfs("/tmp/IMG_5455.HEIC").readImageInfo(HEICInfo, ImageDecodingProps(debug = true))
+    val info = localVfs("/tmp/Exif5-2x.avif").readImageInfo(HEICInfo, ImageDecodingProps(debug = true))
+    println(info)
+}
+
diff --git a/korim/src/commonMain/kotlin/com/soywiz/korim/format/AVIFInfo.kt b/korim/src/commonMain/kotlin/com/soywiz/korim/format/AVIFInfo.kt
@@ -1,37 +1,165 @@
 package com.soywiz.korim.format
 
+import com.soywiz.kds.*
+import com.soywiz.klogger.*
+import com.soywiz.kmem.*
+import com.soywiz.korio.compression.util.*
+import com.soywiz.korio.dynamic.KDynamic.Companion.toInt
+import com.soywiz.korio.experimental.*
 import com.soywiz.korio.lang.*
 import com.soywiz.korio.stream.*
+import com.soywiz.krypto.encoding.*
 
 // AVIF & HEIC metadata extractor
-object AVIFInfo : BaseAVIFInfo("avif")
-object HEICInfo : BaseAVIFInfo("heic")
+object AVIFInfo : ISOBMFF("avif")
+object HEICInfo : ISOBMFF("heic")
 
-open class BaseAVIFInfo(vararg exts: String) : ImageFormatSuspend(*exts) {
+// ISOBMFF
+// https://gpac.github.io/mp4box.js/test/filereader.html
+// https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format
+// https://www.w3.org/TR/mse-byte-stream-format-isobmff/
+open class ISOBMFF(vararg exts: String) : ImageFormatSuspend(*exts) {
     override suspend fun decodeHeaderSuspend(s: AsyncStream, props: ImageDecodingProps): ImageInfo? {
         val ss = s.slice(4 until 8).readString(4, LATIN1)
         if (ss != "ftyp") return null
-        return StreamParser(props).also { it.decodeLevel(s, 0) }.info
+        return StreamParser(props).also { it.decode(s) }.info
     }
 
+    data class ItemExtent(val offset: Long, val size: Long)
+    data class ItemInfo(val id: Int) {
+        var type: String = ""
+        var extents: ArrayList<ItemExtent> = arrayListOf()
+        override fun toString(): String = "ItemInfo($id, $type, $extents)"
+    }
+
+    @OptIn(KorioExperimentalApi::class)
     class StreamParser(val props: ImageDecodingProps) {
+        val debug get() = props.debug
         var info = ImageInfo()
+        val items = IntMap<ItemInfo>()
 
-        suspend fun decodeLevel(s: AsyncStream, level: Int) {
+        suspend fun decode(s: AsyncStream) {
+            decodeLevel(s.sliceHere(), 0)
+            if (debug) Console.error("ITEMS")
+            items.fastValueForEach {
+                if (it.type == "Exif") {
+                    val extent = it.extents.first()
+                    val range = s.sliceWithSize(extent.offset + 4, extent.size - 4).readAllAsFastStream()
+                    EXIF.readExif(range.toAsyncStream(), info, debug = debug)
+                }
+            }
+        }
+
+        private suspend fun decodeLevel(s: AsyncStream, level: Int) {
             while (!s.eof()) {
                 val blockSize = s.readS32BE()
                 val blockType = s.readStringz(4, LATIN1)
                 //val blockSubtype = s.readStringz(4, LATIN1)
                 //val blockStream = s.readStream(blockSize - 12)
-                val blockStream = s.readStream(blockSize - 8)
+                val blockStream = if (blockSize < 8) {
+                    s.readStream(s.getAvailable())
+                } else {
+                    s.readStream(blockSize - 8)
+                }
                 //if (blockSize)
-                //Console.error("${"  ".repeat(level)}blockSize=$blockSize, blockType=$blockType")
+                if (debug) {
+                    Console.error("${"  ".repeat(level)}blockSize=$blockSize, blockType=$blockType")
+                }
                 when (blockType) {
                     "ftyp" -> Unit
                     "meta" -> {
                         blockStream.skip(4)
                         decodeLevel(blockStream, level + 1)
                     }
+                    /// See ISO 14496-12:2015 § 8.11.3
+                    // https://github.com/kornelski/avif-parse/blob/2174e0e15647918cbbcb965ba142c285a6ef457f/src/lib.rs#L1136
+                    "iloc" -> {
+                        val version = blockStream.readU8()
+                        val flags = blockStream.readU24BE()
+                        val sizePacked = ByteArrayBitReader(blockStream.readBytesExact(2))
+                        val offsetSize = sizePacked.readIntBits(4)
+                        val lengthSize = sizePacked.readIntBits(4)
+                        val baseOffsetSize = sizePacked.readIntBits(4)
+                        val indexSize = when (version) {
+                            1, 2 -> sizePacked.readIntBits(4)
+                            else -> 0
+                        }
+                        val count: Int = when (version) {
+                            0, 1 -> blockStream.readU16BE()
+                            2 -> blockStream.readS32BE()
+                            else -> TODO()
+                        }
+
+                        suspend fun AsyncStream.readSize(size: Int): Long {
+                            return when (size) {
+                                0 -> 0L
+                                4 -> blockStream.readS32BE().toLong()
+                                8 -> blockStream.readS64BE()
+                                else -> TODO("size=$size")
+                            }
+                        }
+
+                        if (debug) {
+                            Console.error("iloc version=$version, flags=$flags, count=$count, offsetSize=$offsetSize, lengthSize=$lengthSize, baseOffsetSize=$baseOffsetSize, indexSize=$indexSize")
+                        }
+                        for (n in 0 until count) {
+                            val itemID = when (version) {
+                                0, 1 -> blockStream.readU16BE()
+                                2 -> blockStream.readS32LE()
+                                else -> TODO()
+                            }
+                            val method = when (version) {
+                                0 -> 0
+                                1, 2 -> blockStream.readU16BE()
+                                else -> TODO()
+                            }
+                            val dataRefIndex = blockStream.readU16BE()
+
+                            val baseOffset: Long = blockStream.readSize(baseOffsetSize)
+                            val extentCount = blockStream.readU16BE()
+                            if (debug) {
+                                Console.error(" - itemID=$itemID, method=$method, dataRefIndex=$dataRefIndex, baseOffset=$baseOffset, extentCount=$extentCount")
+                            }
+                            val itemInfo = items.getOrPut(itemID) { ItemInfo(itemID) }
+                            for (m in 0 until extentCount) {
+                                val extentIndex = blockStream.readSize(indexSize)
+                                val extentOffset = blockStream.readSize(offsetSize)
+                                val extentLength = blockStream.readSize(lengthSize)
+                                val offset = baseOffset + extentOffset
+                                if (debug) {
+                                    Console.error("   - $extentIndex, $offset, $extentLength")
+                                }
+                                itemInfo.extents.add(ItemExtent(offset, extentLength))
+                            }
+                            //Console.error(" - $itemID, $unk2, $offset, $size")
+                        }
+                    }
+                    "iinf" -> {
+                        val version = blockStream.readU8()
+                        val flags = blockStream.readU24BE()
+                        val entryCount = when (version) {
+                            0 -> blockStream.readU16BE()
+                            1 -> blockStream.readS32BE()
+                            else -> TODO()
+                        }
+                        for (n in 0 until entryCount) {
+                            decodeLevel(blockStream, level + 1)
+                        }
+                    }
+                    "infe" -> {
+                        val version = blockStream.readU8()
+                        val flags = blockStream.readU24BE()
+                        val itemID: Int = when (version) {
+                            2 -> blockStream.readU16BE()
+                            3 -> blockStream.readS32BE()
+                            else -> TODO()
+                        }
+                        val protectionIndex = blockStream.readU16BE()
+                        val itemType = blockStream.readStringz(4, LATIN1)
+                        val itemInfo = items.getOrPut(itemID) { ItemInfo(itemID) }
+                        itemInfo.type = itemType
+                        if (debug) Console.error("infe: itemID=$itemID, protectionIndex=$protectionIndex, itemType=${itemType}")
+                    }
                     "iprp" -> decodeLevel(blockStream, level + 1)
                     "ipco" -> decodeLevel(blockStream, level + 1)
                     "ispe" -> {
@@ -41,13 +169,18 @@ open class BaseAVIFInfo(vararg exts: String) : ImageFormatSuspend(*exts) {
                     }
                     "mdat" -> {
                         blockStream.skip(4)
-                        if (blockStream.sliceHere().readStringz(4, LATIN1) == "Exif") {
-                            val exif = EXIF.readExif(blockStream.sliceHere())
-                            info.orientation = exif.orientation
-                        }
                     }
                 }
             }
         }
+
+        suspend fun checkExif(blockStream: AsyncStream): Boolean {
+            if (blockStream.sliceHere().readStringz(4, LATIN1) == "Exif") {
+                val exif = EXIF.readExif(blockStream.sliceHere())
+                info.orientation = exif.orientation
+                return true
+            }
+            return false
+        }
     }
 }
diff --git a/korim/src/commonMain/kotlin/com/soywiz/korim/format/EXIF.kt b/korim/src/commonMain/kotlin/com/soywiz/korim/format/EXIF.kt
@@ -1,6 +1,5 @@
 package com.soywiz.korim.format
 
-import com.soywiz.kds.mapDouble
 import com.soywiz.klogger.*
 import com.soywiz.kmem.*
 import com.soywiz.korio.async.*
@@ -10,12 +9,13 @@ import com.soywiz.korio.stream.*
 import com.soywiz.krypto.encoding.*
 
 // https://zpl.fi/exif-orientation-in-different-formats/
+// https://exiftool.org/TagNames/EXIF.html
 object EXIF {
-    suspend fun readExifFromJpeg(s: VfsFile, info: ImageInfo = ImageInfo()): ImageInfo = s.openUse(VfsOpenMode.READ) {
-        readExifFromJpeg(this, info)
+    suspend fun readExifFromJpeg(s: VfsFile, info: ImageInfo = ImageInfo(), debug: Boolean = false): ImageInfo = s.openUse(VfsOpenMode.READ) {
+        readExifFromJpeg(this, info, debug)
     }
 
-    suspend fun readExifFromJpeg(s: AsyncStream, info: ImageInfo = ImageInfo()): ImageInfo {
+    suspend fun readExifFromJpeg(s: AsyncStream, info: ImageInfo = ImageInfo(), debug: Boolean = false): ImageInfo {
         val jpegHeader = s.readU16BE()
         if (jpegHeader != 0xFFD8) error("Not a JPEG file ${jpegHeader.hex}")
         while (!s.eof()) {
@@ -26,7 +26,7 @@ object EXIF {
             val ss = s.readStream(sectionSize - 2)
             when (sectionType) {
                 0xFFE1 -> { // APP1
-                    readExif(ss.readAllAsFastStream(), info)
+                    readExif(ss.readAllAsFastStream(), info, debug)
                 }
                 0xFFC0 -> { // SOF0
                     val precision = ss.readU8()
@@ -43,43 +43,53 @@ object EXIF {
         error("Couldn't find EXIF information")
     }
 
-    fun readExif(s: FastByteArrayInputStream, info: ImageInfo = ImageInfo()): ImageInfo {
-        return runBlockingNoSuspensions { readExif(s.toAsyncStream(), info) }
+    fun readExif(s: FastByteArrayInputStream, info: ImageInfo = ImageInfo(), debug: Boolean = false): ImageInfo {
+        return runBlockingNoSuspensions { readExif(s.toAsyncStream(), info, debug) }
     }
 
-    suspend fun readExif(s: AsyncStream, info: ImageInfo = ImageInfo()): ImageInfo {
+    suspend fun readExif(s: AsyncStream, info: ImageInfo = ImageInfo(), debug: Boolean = false): ImageInfo {
         if (s.readString(4, Charsets.LATIN1) != "Exif") error("Not an Exif section")
         s.skip(2)
-        return readExifBase(s, info)
+        return readExifBase(s, info, debug)
     }
 
-    suspend fun readExifBase(ss: AsyncStream, info: ImageInfo = ImageInfo()): ImageInfo {
-        val endian = if (ss.readString(2, Charsets.LATIN1) == "MM") Endian.BIG_ENDIAN else Endian.LITTLE_ENDIAN
+    // @TODO: We are missing some data from other FIDs, but for orientation it should be on the first FID
+    suspend fun readExifBase(ss: AsyncStream, info: ImageInfo = ImageInfo(), debug: Boolean = false): ImageInfo {
         val s = ss.sliceHere()
+        val endian = when (val start = s.readString(2, Charsets.LATIN1)) {
+            "MM" -> Endian.BIG_ENDIAN
+            "II" -> Endian.LITTLE_ENDIAN
+            else -> error("Not Exif data (not starting with MM or II but '$start')")
+        }
         val tagMark = s.readU16(endian)
-        val offsetFirstFID = s.readS32(endian) // @TODO: do we need to use this somehow?
+        val offsetFirstIFD = s.readS32(endian) // @TODO: do we need to use this somehow?
 
         val nDirEntry = s.readU16(endian)
-        //Console.error("nDirEntry=$nDirEntry, tagMark=$tagMark, offsetFirstFID=$offsetFirstFID")
+        if (debug) {
+            Console.error("nDirEntry=$nDirEntry, tagMark=$tagMark, offsetFirstFID=$offsetFirstIFD")
+        }
         for (n in 0 until nDirEntry) {
+            val tagPos = s.position.toInt()
             val tagNumber = s.readU16(endian)
-            val dataFormat = s.readU16(endian)
+            val dataFormat = DataFormat[s.readU16(endian)]
             val nComponent = s.readS32(endian)
-            //Console.error("tagNumber=$tagNumber, dataFormat=$dataFormat, nComponent=$nComponent")
-            val values = when (dataFormat) {
-                DataFormat.UBYTE.id, DataFormat.SBYTE.id -> s.readBytesExact(nComponent).mapDouble { it.toDouble() }
-                DataFormat.STRING.id -> s.readBytesExact(nComponent).mapDouble { it.toDouble() }
-                DataFormat.UNDEFINED.id -> s.readBytesExact(nComponent).mapDouble { it.toDouble() }
-                DataFormat.USHORT.id, DataFormat.SSHORT.id -> s.readShortArray(nComponent, endian).mapDouble { it.toDouble() }
-                DataFormat.ULONG.id, DataFormat.SLONG.id -> s.readIntArray(nComponent, endian).mapDouble { it.toDouble() }
-                DataFormat.SFLOAT.id -> s.readFloatArray(nComponent, endian).mapDouble { it.toDouble() }
-                DataFormat.DFLOAT.id -> s.readIntArray(nComponent, endian).mapDouble { it.toDouble() } // These are offsets to the 8-byte structure (DWORD/DWORD)
-                DataFormat.URATIO.id, DataFormat.SRATIO.id -> s.readIntArray(nComponent, endian).mapDouble { it.toDouble() } // These are offsets to the 8-byte structure (DWORD/DWORD)
-                else -> error("Invalid data type: ${dataFormat.hex}")
+            if (debug) {
+                Console.error("tagPos=${tagPos.hex}, tagNumber=${tagNumber.hex}, dataFormat=$dataFormat, nComponent=$nComponent, size=${dataFormat.indexBytes(nComponent)}")
             }
+            val data: ByteArray = s.readBytesExact(dataFormat.indexBytes(nComponent))
+
+            fun readUShort(index: Int): Int = data.readU16(index * 2, little = endian.isLittle)
+            fun readInt(index: Int): Int = data.readS32(index * 4, little = endian.isLittle)
+
+            if (debug) {
+                if (dataFormat == DataFormat.STRING) {
+                    Console.error("  - '${s.sliceStart(readInt(0).toLong()).readStringz(nComponent)}'")
+                }
+            }
+
             when (tagNumber) {
                 0x112 -> { // Orientation
-                    info.orientation = when (values[0].toInt()) {
+                    info.orientation = when (readUShort(0)) {
                         1 -> ImageOrientation.ORIGINAL
                         2 -> ImageOrientation.MIRROR_HORIZONTAL
                         3 -> ImageOrientation.ROTATE_180
@@ -93,27 +103,33 @@ object EXIF {
                 }
             }
             //AsyncStream().skipToAlign()
-            s.skipToAlign(4)
+            s.skipToAlign(4, offset = 2)
         }
         return info
     }
 
     enum class DataFormat(
         val id: Int,
-        val nBytes: Int,
-        val rBytes: Int = nBytes
+        val indexBytes: (Int) -> Int,
     ) {
-        UBYTE(1, 1),
-        STRING(2, 1),
-        USHORT(3, 2),
-        ULONG(4, 3),
-        URATIO(5, 4, 8),
-        SBYTE(6, 1),
-        UNDEFINED(7, 1),
-        SSHORT(8, 2),
-        SLONG(9, 4),
-        SRATIO(10, 4, 8),
-        SFLOAT(11, 4),
-        DFLOAT(12, 8);
+        UBYTE(1, { it }),
+        STRING(2, { 4 }),
+        USHORT(3, { it * 2 }),
+        ULONG(4, { it * 4 }),
+        URATIO(5, { it * 4 }),
+        SBYTE(6, { it }),
+        UNDEFINED(7, { it }),
+        SSHORT(8, { it * 2 }),
+        SLONG(9, { it * 4 }),
+        SRATIO(10, { it * 4 }),
+        SFLOAT(11, { it * 4 }),
+        DFLOAT(12, { it * 8 }),
+        UNKNOWN(-1, { 0 });
+
+        companion object {
+            val BY_ID = values().associateBy { it.id }
+            operator fun get(index: Int): DataFormat =
+                BY_ID[index] ?: UNKNOWN
+        }
     }
 }
diff --git a/korim/src/commonMain/kotlin/com/soywiz/korim/format/ImageData.kt b/korim/src/commonMain/kotlin/com/soywiz/korim/format/ImageData.kt
@@ -44,6 +44,8 @@ open class ImageData constructor(
 
 data class ImageDataWithAtlas(val image: ImageData, val atlas: AtlasPacker.Result<ImageFrameLayer>)
 
+var ImageData.info: ImageInfo? by Extra.Property { null }
+
 //fun ImageData.packInAtlas(): ImageDataWithAtlas {
 //    val frameLayers = frames.flatMap { it.layerData }.filter { it.includeInAtlas }
 //    val atlasResult = AtlasPacker.pack(frameLayers.map { it to it.slice })
diff --git a/korim/src/commonMain/kotlin/com/soywiz/korim/format/ImageFormat.kt b/korim/src/commonMain/kotlin/com/soywiz/korim/format/ImageFormat.kt
@@ -90,6 +90,7 @@ data class ImageDecodingProps(
     val premultiplied: Boolean = true,
     // Requested but not enforced. Max width and max height
     val requestedMaxSize: Int? = null,
+    val debug: Boolean = false,
     override var extra: ExtraType = null
 ) : Extra {
 
diff --git a/korio/src/commonMain/kotlin/com/soywiz/korio/stream/AsyncStream.kt b/korio/src/commonMain/kotlin/com/soywiz/korio/stream/AsyncStream.kt
diff --git a/korio/src/commonMain/kotlin/com/soywiz/korio/stream/ByteArrayBitReader.kt b/korio/src/commonMain/kotlin/com/soywiz/korio/stream/ByteArrayBitReader.kt
diff --git a/korio/src/commonTest/kotlin/com/soywiz/korio/stream/ByteArrayBitReaderTest.kt b/korio/src/commonTest/kotlin/com/soywiz/korio/stream/ByteArrayBitReaderTest.kt
