Serialize nested collections in column major order

src/main/kotlin/dev/bright/vb6serializer/BinaryDecoder.kt

@@ -2,11 +2,14 @@ package dev.bright.vb6serializer
 
 import kotlinx.serialization.DeserializationStrategy
 import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.InternalSerializationApi
 import kotlinx.serialization.descriptors.SerialDescriptor
 import kotlinx.serialization.descriptors.StructureKind
 import kotlinx.serialization.encoding.CompositeDecoder
 import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.internal.AbstractCollectionSerializer
 import kotlinx.serialization.modules.SerializersModule
+import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
 import java.io.EOFException
 
@@ -183,17 +186,46 @@ internal open class BinaryDecoder(
         return decodeSerializableValue(deserializer)
     }
 
+    @OptIn(InternalSerializationApi::class)
     private fun <T> decodeSerializableList(
         descriptor: SerialDescriptor, index: Int, deserializer: DeserializationStrategy<T>
     ): T {
-        return if (deserializer is ConstByteSizeDeserializationStrategy<T>) {
-            decodeSerializableValue(deserializer)
+        val listDescriptor = descriptor.getElementDescriptor(index)
+        val listElementDescriptor = listDescriptor.getElementDescriptor(0)
+
+        val collectionSize = (deserializer as? ConstByteSizeCollectionDeserializationStrategy<*>)?.let { Size(it.collectionMaxSize) }
+            ?: configuration.sizeResolver.sizeFor(descriptor, index)
+            ?: descriptor.requireSizeOnElement(index)
+
+        val decoder = if (listElementDescriptor.kind == StructureKind.LIST) {
+            // nested list, handle VB6 native column-major order
+            val listElementSerializer =
+                CollectionLikeSerializer.elementSerializer(deserializer as AbstractCollectionSerializer<*, *, *>)
+
+            require(listElementSerializer is ConstByteSizeCollectionDeserializationStrategy<*>) {
+                "Required ConstByteSizeCollectionDeserializationStrategy got $listElementSerializer"
+            }
+
+            val nestedElementByteSize = listElementSerializer.elementByteSize
+            val columns = listElementSerializer.collectionMaxSize
+            val nestedListTotalBytes = listElementSerializer.totalByteSize
+
+            val serialized = input.withBytesLimitedTo(collectionSize.length * nestedListTotalBytes)
+                .readAllBytes()
+
+            transposeInPlace(serialized, rows = columns, cols = collectionSize.length, nestedElementByteSize)
+
+            BinaryDecoder(Input.create(ByteArrayInputStream(serialized)), configuration, serializersModule)
         } else {
-            val collectionSize =
-                configuration.sizeResolver.sizeFor(descriptor, index) ?: descriptor.requireSizeOnElement(index)
+            this
+        }
 
+        return if (deserializer is ConstByteSizeDeserializationStrategy<T>) {
+            deserializer.deserialize(decoder)
+        } else {
             val constSizeCollectionDecoder =
-                ConstSizeCollectionDecoder(collectionSize.length, input, configuration, serializersModule)
+                ConstSizeCollectionDecoder(collectionSize.length, decoder.input, decoder.configuration, decoder.serializersModule)
+
             deserializer.deserialize(
                 HasConstStructureDecoder(
                     input, configuration, constSizeCollectionDecoder

src/main/kotlin/dev/bright/vb6serializer/BinaryEncoder.kt

@@ -16,7 +16,7 @@ import kotlinx.serialization.modules.SerializersModule
 internal class BinaryEncoder(
     override val output: Output,
     override val serializersModule: SerializersModule,
-    private val configuration: VB6BinaryConfiguration
+    internal val configuration: VB6BinaryConfiguration
 ) :
     Encoder, CompositeEncoder, HasOutput {
     override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder {
@@ -207,6 +207,7 @@ internal class BinaryEncoder(
                     descriptor.getElementDescriptor(index).getElementDescriptor(0)
                 )
             } else {
+
                 val constSizeSerializer = ConstByteSizeCollectionSerializationStrategy(
                     serializer as SerializationStrategy<T>,
                     maxSize,
@@ -270,11 +271,6 @@ internal class BinaryEncoder(
         return javaClass.getMethod("collectionSize", Any::class.java).invoke(this, value) as Int
     }
 
-    @OptIn(InternalSerializationApi::class)
-    private fun <T> AbstractCollectionSerializer<T, *, *>.elementSerializer(
-    ): SerializationStrategy<*> {
-        return CollectionLikeSerializer.elementSerializer(this)
-    }
 }
 
 internal class FillZeroBytesSerializer<T>(override val descriptor: SerialDescriptor, override val totalByteSize: Int) :
@@ -284,22 +280,17 @@ internal class FillZeroBytesSerializer<T>(override val descriptor: SerialDescrip
     }
 }
 
-@OptIn(InternalSerializationApi::class)
-object CollectionLikeSerializer {
-    private val elementSerializer =
-        javaClass.classLoader.loadClass("kotlinx.serialization.internal.CollectionLikeSerializer")
-            .getDeclaredField("elementSerializer").apply { isAccessible = true }
-
-    fun <T> elementSerializer(serializer: AbstractCollectionSerializer<T, *, *>): SerializationStrategy<*> =
-        elementSerializer.get(serializer) as SerializationStrategy<*>
-}
-
 
 internal fun Encoder.requireHasOutputEncoder(): HasOutput {
     return this as? HasOutput
         ?: throw IllegalArgumentException("Only ${HasOutput::class} is supported got $this")
 }
 
+internal fun Encoder.requireBinaryEncoder(): BinaryEncoder {
+    return this as? BinaryEncoder
+        ?: throw IllegalArgumentException("Only ${BinaryEncoder::class} is supported got $this")
+}
+
 internal fun Decoder.requireBinaryDecoderBase(): BinaryDecoder {
     return this as? BinaryDecoder
         ?: throw IllegalArgumentException("Only ${BinaryDecoder::class} is supported got $this")

src/main/kotlin/dev/bright/vb6serializer/CollectionLikeSerializer.kt

@@ -0,0 +1,21 @@
+package dev.bright.vb6serializer
+
+import kotlinx.serialization.InternalSerializationApi
+import kotlinx.serialization.SerializationStrategy
+import kotlinx.serialization.internal.AbstractCollectionSerializer
+
+@OptIn(InternalSerializationApi::class)
+internal object CollectionLikeSerializer {
+    private val elementSerializer =
+        javaClass.classLoader.loadClass("kotlinx.serialization.internal.CollectionLikeSerializer")
+            .getDeclaredField("elementSerializer").apply { isAccessible = true }
+
+    fun <T> elementSerializer(serializer: AbstractCollectionSerializer<T, *, *>): SerializationStrategy<*> =
+        elementSerializer.get(serializer) as SerializationStrategy<*>
+}
+
+@OptIn(InternalSerializationApi::class)
+internal fun <T> AbstractCollectionSerializer<T, *, *>.elementSerializer(
+): SerializationStrategy<*> {
+    return CollectionLikeSerializer.elementSerializer(this)
+}

src/main/kotlin/dev/bright/vb6serializer/ConstByteSizeSerializer.kt

@@ -1,12 +1,14 @@
+@file:OptIn(ExperimentalSerializationApi::class, InternalSerializationApi::class)
+
 package dev.bright.vb6serializer
 
-import kotlinx.serialization.DeserializationStrategy
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.SerializationStrategy
+import kotlinx.serialization.*
 import kotlinx.serialization.builtins.serializer
 import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.descriptors.StructureKind
 import kotlinx.serialization.encoding.Decoder
 import kotlinx.serialization.encoding.Encoder
+import kotlinx.serialization.internal.AbstractCollectionSerializer
 
 internal open class ConstByteSizeCollectionSerializationStrategy<T>(
     private val inner: SerializationStrategy<T>,
@@ -19,9 +21,21 @@ internal open class ConstByteSizeCollectionSerializationStrategy<T>(
     override val descriptor: SerialDescriptor = inner.descriptor
 
     override fun serialize(encoder: Encoder, value: T) {
-        val binaryEncoder = encoder.requireHasOutputEncoder()
+        val binaryEncoder = encoder.requireBinaryEncoder()
         binaryEncoder.output.addPaddingWithRatio(collectionActualSize, collectionMaxSize) {
-            inner.serialize(encoder, value)
+            val elementDescriptor = inner.descriptor.getElementDescriptor(0)
+            if (elementDescriptor.kind == StructureKind.LIST) {
+                val serialized =
+                    serializedBytesOf(binaryEncoder.serializersModule, binaryEncoder.configuration, inner, value)
+                @Suppress("UNCHECKED_CAST") val actualSerializer = inner as AbstractCollectionSerializer<T, *, *>
+                val elementSerializer = actualSerializer.elementSerializer() as ConstByteSizeCollectionKSerializer<*>
+                val rows = collectionActualSize
+                val cols = elementSerializer.collectionMaxSize
+                transposeInPlace(serialized, rows, cols, elementSerializer.elementByteSize)
+                binaryEncoder.output.write(serialized)
+            } else {
+                inner.serialize(encoder, value)
+            }
         }
     }
 }
@@ -52,15 +66,23 @@ internal interface ConstByteSizeSerializationStrategy<T> : SerializationStrategy
     val totalByteSize: Int
 }
 
-internal interface ConstByteSizeDeserializationStrategy<T> : DeserializationStrategy<T>
-internal interface ConstByteSizeKSerializer<T> : KSerializer<T>, ConstByteSizeSerializationStrategy<T>,
+internal interface ConstByteSizeDeserializationStrategy<T> : DeserializationStrategy<T> {
+    val totalByteSize: Int
+}
+
+internal interface ConstByteSizeCollectionDeserializationStrategy<T> : ConstByteSizeDeserializationStrategy<T> {
+    val elementByteSize: Int
+    val collectionMaxSize: Int
+}
+internal interface ConstByteSizeKSerializer<T> : KSerializer<T>,
+    ConstByteSizeSerializationStrategy<T>,
     ConstByteSizeDeserializationStrategy<T>
 
 open class ConstByteSizeCollectionKSerializer<T>(
-    private val inner: KSerializer<T>, private val elementByteSize: Int, private val collectionMaxSize: Int
-) : KSerializer<T>, ConstByteSizeKSerializer<T> {
+    private val inner: KSerializer<T>, override val elementByteSize: Int, override val collectionMaxSize: Int
+) : KSerializer<T>, ConstByteSizeKSerializer<T>, ConstByteSizeCollectionDeserializationStrategy<T> {
     override val totalByteSize get() = elementByteSize * collectionMaxSize
-    override val descriptor: SerialDescriptor = inner.descriptor
+    override val descriptor: SerialDescriptor get() = inner.descriptor
 
     override fun serialize(encoder: Encoder, value: T) {
         val binaryEncoder = encoder.requireHasOutputEncoder()

src/main/kotlin/dev/bright/vb6serializer/Input.kt

@@ -35,6 +35,10 @@ internal class Input private constructor(
     // TODO: that's not accurate
     val isComplete: Boolean get() = stream.available() <= 0
 
+    fun readAllBytes(): ByteArray {
+        return stream.readAllBytes()
+    }
+
     companion object {
         fun create(stream: InputStream) = Input(ByteCountingInputStream(stream))
     }

src/main/kotlin/dev/bright/vb6serializer/TransposeInPlace.kt

@@ -0,0 +1,31 @@
+package dev.bright.vb6serializer
+
+internal fun transposeInPlace(serialized: ByteArray, rows: Int, cols: Int, elementByteSize: Int) {
+    fun swapElement(i: Int, j: Int) {
+        val iByteSizeIndex = i * elementByteSize
+        val jByteSizeIndex = j * elementByteSize
+
+        for (x in 0 until elementByteSize) {
+            val temp = serialized[iByteSizeIndex + x]
+            serialized[iByteSizeIndex + x] = serialized[jByteSizeIndex + x]
+            serialized[jByteSizeIndex + x] = temp
+        }
+    }
+    // https://stackoverflow.com/questions/9227747/in-place-transposition-of-a-matrix
+    val lastIndex = rows * cols - 1
+    val visited = BooleanArray(rows * cols)
+
+    for (ix in 1 until lastIndex) {// 0 and lastIndex shouldn't move
+        if (visited[ix]) {
+            continue
+        }
+
+        var cycleStartIndex = ix
+        do {
+            cycleStartIndex = if (cycleStartIndex == lastIndex) lastIndex else (rows * cycleStartIndex) % lastIndex
+            // Swap arr[a] and arr[cycleIndex]
+            swapElement(cycleStartIndex, ix)
+            visited[cycleStartIndex] = true
+        } while (cycleStartIndex != ix)
+    }
+}

src/main/kotlin/dev/bright/vb6serializer/VB6Binary.kt

@@ -38,11 +38,20 @@ open class VB6Binary(override val serializersModule: SerializersModule, private
     }
 
     override fun <T> encodeToByteArray(serializer: SerializationStrategy<T>, value: T): ByteArray {
-        val outputStream = ByteArrayOutputStream()
-        val dumper = BinaryEncoder(Output.create(outputStream), serializersModule, configuration)
-        dumper.encodeSerializableValue(serializer, value)
-        return outputStream.toByteArray()
+        return serializedBytesOf(serializersModule, configuration, serializer, value)
     }
 }
 
+internal fun <T> serializedBytesOf(
+    serializersModule: SerializersModule,
+    configuration: VB6BinaryConfiguration,
+    serializer: SerializationStrategy<T>,
+    value: T
+): ByteArray {
+    val outputStream = ByteArrayOutputStream()
+    val dumper = BinaryEncoder(Output.create(outputStream), serializersModule, configuration)
+    dumper.encodeSerializableValue(serializer, value)
+    return outputStream.toByteArray()
+}
+
 internal val defaultSerializingCharset = Charset.forName("ISO-8859-8")

src/test/kotlin/dev/bright/vb6serializer/VB6BinaryTests.kt

@@ -5,6 +5,7 @@ import io.kotest.matchers.shouldBe
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.builtins.IntArraySerializer
+import kotlinx.serialization.builtins.ShortArraySerializer
 import kotlinx.serialization.decodeFromByteArray
 import kotlinx.serialization.descriptors.SerialDescriptor
 import kotlinx.serialization.encodeToByteArray
@@ -240,11 +241,51 @@ class VB6BinaryTests {
     }
 
     @Test
-    fun `can serialize int array inside array with list of strings`() {
+    fun `can serialize complete short array inside array`() {
+        // given
+        val input = HasArrayOfShortArrays(
+            arrayOf(
+                ShortArray(5) { (it + 1).toShort() },
+                ShortArray(5) { (it + 1 + 10).toShort() },
+                ShortArray(5) { (it + 1 + 100).toShort() },
+            )
+        )
+
+        // when
+        val output = serde(input)
+
+        // then
+        output.items.shouldHaveSize(HasArrayOfShortArrays.HasArrayOfShortArraysItemSize)
+        output.items[0].shouldBe(input.items[0])
+        output.items[1].shouldBe(input.items[1])
+        output.items[2].shouldBe(input.items[2])
+    }
+
+    @Test
+    fun `can serialize complete short array inside array in vb6 compatible way - column major order`() {
+        // given
+        val input = HasArrayOfShortArrays(
+            arrayOf(
+                ShortArray(5) { (it + 1).toShort() },
+                ShortArray(5) { (it + 1 + 10).toShort() },
+                ShortArray(5) { (it + 1 + 100).toShort() },
+            )
+        )
+
+        // when
+        val output = VB6Binary.encodeToByteArray(input)
+
+        // then
+        output.shouldBe(byteArrayOf(1, 0, 11, 0, 101, 0, 2, 0, 12, 0, 102, 0, 3, 0, 13, 0, 103, 0, 4, 0, 14, 0, 104, 0, 5, 0, 15, 0, 105, 0))
+    }
+
+    @Test
+    fun `can serialize int array inside array with complete list of ints`() {
         // given
         val input = HasListOfIntArrays(
             listOf(
-                IntArray(3) { it + 1 }
+                IntArray(3) { it + 1 },
+                IntArray(3) { it + 1 + 10 }
             )
         )
 
@@ -254,7 +295,7 @@ class VB6BinaryTests {
         // then
         output.items.shouldHaveSize(HasListOfIntArrays.HasListOfIntArraysItemSize)
         output.items[0].shouldBe(input.items[0])
-        output.items[1].shouldBe(IntArray(3) { 0 })
+        output.items[1].shouldBe(input.items[1])
     }
 
     @Test
@@ -383,8 +424,22 @@ data class HasListOfIntArrays(
     override fun toString(): String {
         return "HasListOfIntArrays(items=${items.map { it.contentToString() }})"
     }
+}
 
 
+@Serializable
+data class HasArrayOfShortArrays(
+    @Size(HasArrayOfShortArraysItemSize)
+    val items: Array<@Serializable(with = ShortArrayWith5ElementsSerializer::class) ShortArray>,
+) {
+    companion object {
+        const val HasArrayOfShortArraysItemSize = 3
+    }
+
+    override fun toString(): String {
+        return "HasArrayOfShortArrays(items=${items.map { it.contentToString() }})"
+    }
+
 }
 
 
@@ -404,3 +459,10 @@ object IntArrayWith3ElementsSerializer : ConstByteSizeCollectionKSerializer<IntA
     collectionMaxSize = 3
 )
 
+
+object ShortArrayWith5ElementsSerializer : ConstByteSizeCollectionKSerializer<ShortArray>(
+    ShortArraySerializer(),
+    elementByteSize = Short.SIZE_BYTES,
+    collectionMaxSize = 5
+)
+