Merge pull request #187 from JetBrains-Research/update-test-environment

JBAI-4915 [all] Rework of errors printing and assertions
JetBrains-Research · Jun 18, 2024 · 3b60f3e · 3b60f3e
2 parents 1627e97 + c84525f
commit 3b60f3e
Show file tree

Hide file tree

Showing 37 changed files with 693 additions and 1,027 deletions.
diff --git a/.../kmath/adapter-kmath-core/src/commonTest/kotlin/io/kinference/kmath/KIKMathAdapterTest.kt b/.../kmath/adapter-kmath-core/src/commonTest/kotlin/io/kinference/kmath/KIKMathAdapterTest.kt
@@ -19,7 +19,6 @@ import io.kinference.utils.*
 import io.kinference.utils.inlines.InlineInt
 import space.kscience.kmath.nd.*
 import space.kscience.kmath.structures.Buffer
-import kotlin.math.abs
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -102,14 +101,12 @@ class KIKMathAdapterTest {
     companion object {
         fun assertEquals(expected: KITensor, actual: KITensor) {
             assertEquals(expected.data.type, actual.data.type, "Types of tensors ${expected.name} do not match")
-            ArrayAssertions.assertArrayEquals(expected.data.shape.toTypedArray(), actual.data.shape.toTypedArray(), "Shapes do not match")
-            ArrayAssertions.assertArrayEquals(
-                (expected.data as IntNDArray).array,
-                (actual.data as IntNDArray).array,
-                { l, r -> abs(l - r).toDouble() },
-                delta = 0.0,
-                ""
-            )
+            ArrayAssertions.assertArrayEquals(expected.data.shape.toTypedArray(), actual.data.shape.toTypedArray()) { "Shapes of tensors ${expected.name} do not match" }
+
+            val expectedArray = (expected.data as IntNDArray).array.blocks
+            val actualArray = (actual.data as IntNDArray).array.blocks
+
+            ArrayAssertions.assertArrayEquals(expectedArray, actualArray, delta = 0.0) { "Tensors ${expected.name} do not match" }
         }
 
         fun assertEquals(expected: KIONNXMap, actual: KIONNXMap) {

diff --git a/...kmath/adapter-kmath-ort-gpu/src/jvmTest/kotlin/io/kinference/kmath/ORTKMathAdapterTest.kt b/...kmath/adapter-kmath-ort-gpu/src/jvmTest/kotlin/io/kinference/kmath/ORTKMathAdapterTest.kt
@@ -7,7 +7,6 @@ import space.kscience.kmath.nd.*
 import space.kscience.kmath.structures.Buffer
 import java.nio.ByteBuffer
 import java.nio.IntBuffer
-import kotlin.math.abs
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -49,27 +48,13 @@ class ORTKMathAdapterTest {
         fun assertTensorEquals(expected: ORTTensor, actual: ORTTensor) {
             assertEquals(expected.data.info.type, actual.data.info.type, "Types of tensors ${expected.name} do not match")
             assertEquals(expected.name, actual.name, "Names of tensors do not match")
-            ArrayAssertions.assertArrayEquals(expected.data.info.shape.toTypedArray(), actual.data.info.shape.toTypedArray(), "Shapes do not match")
-            when {
-                expected.data.intBuffer != null -> assertIntEquals(expected, actual)
-                expected.data.byteBuffer != null -> assertByteEquals(expected, actual)
+            ArrayAssertions.assertArrayEquals(expected.data.info.shape.toTypedArray(), actual.data.info.shape.toTypedArray()) { "Shapes of tensors ${expected.name} do not match" }
+
+            when (expected.data.info.type) {
+                OnnxJavaType.INT32 -> ArrayAssertions.assertArrayEquals(expected.toIntArray(), actual.toIntArray(), delta = 0.0) { "Tensors ${expected.name} do not match" }
+                OnnxJavaType.UINT8 -> ArrayAssertions.assertArrayEquals(expected.toUByteArray(), actual.toUByteArray(), delta = 0.0) { "Tensors ${expected.name} do not match"}
+                else -> error("Unsupported data type: ${expected.data.info.type}")
             }
         }
-
-        fun assertIntEquals(expected: ORTTensor, actual: ORTTensor) = ArrayAssertions.assertArrayEquals(
-            expected.data.intBuffer.array(),
-            actual.data.intBuffer.array(),
-            { l, r -> abs(l - r).toDouble() },
-            delta = 0.0,
-            ""
-        )
-
-        fun assertByteEquals(expected: ORTTensor, actual: ORTTensor) = ArrayAssertions.assertArrayEquals(
-            expected.data.byteBuffer.array(),
-            actual.data.byteBuffer.array(),
-            { l, r -> abs(l - r).toDouble() },
-            delta = 0.0,
-            ""
-        )
     }
 }
diff --git a/...ers/kmath/adapter-kmath-ort/src/jvmTest/kotlin/io/kinference/kmath/ORTKMathAdapterTest.kt b/...ers/kmath/adapter-kmath-ort/src/jvmTest/kotlin/io/kinference/kmath/ORTKMathAdapterTest.kt
@@ -7,7 +7,6 @@ import space.kscience.kmath.nd.*
 import space.kscience.kmath.structures.Buffer
 import java.nio.ByteBuffer
 import java.nio.IntBuffer
-import kotlin.math.abs
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -49,27 +48,13 @@ class ORTKMathAdapterTest {
         fun assertTensorEquals(expected: ORTTensor, actual: ORTTensor) {
             assertEquals(expected.data.info.type, actual.data.info.type, "Types of tensors ${expected.name} do not match")
             assertEquals(expected.name, actual.name, "Names of tensors do not match")
-            ArrayAssertions.assertArrayEquals(expected.data.info.shape.toTypedArray(), actual.data.info.shape.toTypedArray(), "Shapes do not match")
-            when {
-                expected.data.intBuffer != null -> assertIntEquals(expected, actual)
-                expected.data.byteBuffer != null -> assertByteEquals(expected, actual)
+            ArrayAssertions.assertArrayEquals(expected.data.info.shape.toTypedArray(), actual.data.info.shape.toTypedArray()) { "Shapes of tensors ${expected.name} do not match" }
+
+            when (expected.data.info.type) {
+                OnnxJavaType.INT32 -> ArrayAssertions.assertArrayEquals(expected.toIntArray(), actual.toIntArray(), delta = 0.0) { "Tensors ${expected.name} do not match" }
+                OnnxJavaType.UINT8 -> ArrayAssertions.assertArrayEquals(expected.toUByteArray(), actual.toUByteArray(), delta = 0.0) { "Tensors ${expected.name} do not match"}
+                else -> error("Unsupported data type: ${expected.data.info.type}")
             }
         }
-
-        fun assertIntEquals(expected: ORTTensor, actual: ORTTensor) = ArrayAssertions.assertArrayEquals(
-            expected.data.intBuffer.array(),
-            actual.data.intBuffer.array(),
-            { l, r -> abs(l - r).toDouble() },
-            delta = 0.0,
-            ""
-        )
-
-        fun assertByteEquals(expected: ORTTensor, actual: ORTTensor) = ArrayAssertions.assertArrayEquals(
-            expected.data.byteBuffer.array(),
-            actual.data.byteBuffer.array(),
-            { l, r -> abs(l - r).toDouble() },
-            delta = 0.0,
-            ""
-        )
     }
 }
diff --git a/...tik/adapter-multik-core/src/commonTest/kotlin/io/kinference/multik/KIMultikAdapterTest.kt b/...tik/adapter-multik-core/src/commonTest/kotlin/io/kinference/multik/KIMultikAdapterTest.kt
@@ -6,12 +6,10 @@ import io.kinference.ndarray.arrays.IntNDArray
 import io.kinference.ndarray.extensions.createNDArray
 import io.kinference.ndarray.extensions.tiledFromPrimitiveArray
 import io.kinference.primitives.types.DataType
-import io.kinference.utils.ArrayAssertions
-import io.kinference.utils.TestRunner
+import io.kinference.utils.*
 import multik.KIMultikData
 import multik.KIMultikTensorAdapter
 import org.jetbrains.kotlinx.multik.ndarray.data.*
-import kotlin.math.abs
 import kotlin.test.*
 
 class KIMultikAdapterTest {
@@ -39,14 +37,12 @@ class KIMultikAdapterTest {
         fun assertTensorEquals(expected: KITensor, actual: KITensor) {
             assertEquals(expected.data.type, actual.data.type, "Types of tensors ${expected.name} do not match")
             assertEquals(expected.name, actual.name, "Names of tensors do not match")
-            ArrayAssertions.assertArrayEquals(expected.data.shape.toTypedArray(), actual.data.shape.toTypedArray(), "Shapes do not match")
-            ArrayAssertions.assertArrayEquals(
-                (expected.data as IntNDArray).array,
-                (actual.data as IntNDArray).array,
-                { l, r -> abs(l - r).toDouble() },
-                delta = 0.0,
-                ""
-            )
+            ArrayAssertions.assertArrayEquals(expected.data.shape.toTypedArray(), actual.data.shape.toTypedArray()) { "Shapes of tensors ${expected.name} do not match" }
+
+            val expectedArray = (expected.data as IntNDArray).array.blocks
+            val actualArray = (actual.data as IntNDArray).array.blocks
+
+            ArrayAssertions.assertArrayEquals(expectedArray, actualArray, delta = 0.0) { "Tensors ${expected.name} do not match" }
         }
     }
 }
diff --git a/...ik/adapter-multik-ort-gpu/src/jvmTest/kotlin/io/kinference/multik/ORTMultikAdapterTest.kt b/...ik/adapter-multik-ort-gpu/src/jvmTest/kotlin/io/kinference/multik/ORTMultikAdapterTest.kt
@@ -6,7 +6,6 @@ import io.kinference.utils.*
 import org.jetbrains.kotlinx.multik.ndarray.data.*
 import org.junit.jupiter.api.Test
 import java.nio.IntBuffer
-import kotlin.math.abs
 import kotlin.test.assertEquals
 import kotlin.test.assertTrue
 
@@ -57,25 +56,13 @@ class ORTMultikAdapterTest {
         fun assertTensorEquals(expected: ORTTensor, actual: ORTTensor) {
             assertEquals(expected.data.info.type, actual.data.info.type, "Types of tensors ${expected.name} do not match")
             assertEquals(expected.name, actual.name, "Names of tensors do not match")
-            ArrayAssertions.assertArrayEquals(expected.data.info.shape.toTypedArray(), actual.data.info.shape.toTypedArray(), "Shapes do not match")
-            if (expected.data.info.type == OnnxJavaType.BOOL) {
-                ArrayAssertions.assertArrayEquals(
-                    expected.data.byteBuffer.array(),
-                    actual.data.byteBuffer.array(),
-                    { l, r -> (l - r).toDouble() },
-                    delta = 0.0,
-                    message = ""
-                )
-            } else {
-                ArrayAssertions.assertArrayEquals(
-                    expected.data.intBuffer.array(),
-                    actual.data.intBuffer.array(),
-                    { l, r -> abs(l - r).toDouble() },
-                    delta = 0.0,
-                    ""
-                )
-            }
+            ArrayAssertions.assertArrayEquals(expected.data.info.shape.toTypedArray(), actual.data.info.shape.toTypedArray()) { "Shapes of tensors ${expected.name} do not match" }
 
+            when (expected.data.info.type) {
+                OnnxJavaType.INT32 -> ArrayAssertions.assertArrayEquals(expected.toIntArray(), actual.toIntArray(), delta = 0.0) { "Tensors ${expected.name} do not match" }
+                OnnxJavaType.BOOL -> ArrayAssertions.assertArrayEquals(expected.toBooleanArray(), actual.toBooleanArray(), delta = 0.0) { "Tensors ${expected.name} do not match"}
+                else -> error("Unsupported data type: ${expected.data.info.type}")
+            }
         }
     }
 }
diff --git a/...multik/adapter-multik-ort/src/jvmTest/kotlin/io/kinference/multik/ORTMultikAdapterTest.kt b/...multik/adapter-multik-ort/src/jvmTest/kotlin/io/kinference/multik/ORTMultikAdapterTest.kt
@@ -5,7 +5,6 @@ import io.kinference.ort.data.tensor.ORTTensor
 import io.kinference.utils.*
 import org.jetbrains.kotlinx.multik.ndarray.data.*
 import java.nio.IntBuffer
-import kotlin.math.abs
 import kotlin.test.*
 
 class ORTMultikAdapterTest {
@@ -55,25 +54,13 @@ class ORTMultikAdapterTest {
         fun assertTensorEquals(expected: ORTTensor, actual: ORTTensor) {
             assertEquals(expected.data.info.type, actual.data.info.type, "Types of tensors ${expected.name} do not match")
             assertEquals(expected.name, actual.name, "Names of tensors do not match")
-            ArrayAssertions.assertArrayEquals(expected.data.info.shape.toTypedArray(), actual.data.info.shape.toTypedArray(), "Shapes do not match")
-            if (expected.data.info.type == OnnxJavaType.BOOL) {
-                ArrayAssertions.assertArrayEquals(
-                    expected.data.byteBuffer.array(),
-                    actual.data.byteBuffer.array(),
-                    { l, r -> (l - r).toDouble() },
-                    delta = 0.0,
-                    message = ""
-                )
-            } else {
-                ArrayAssertions.assertArrayEquals(
-                    expected.data.intBuffer.array(),
-                    actual.data.intBuffer.array(),
-                    { l, r -> abs(l - r).toDouble() },
-                    delta = 0.0,
-                    ""
-                )
-            }
+            ArrayAssertions.assertArrayEquals(expected.data.info.shape.toTypedArray(), actual.data.info.shape.toTypedArray()) { "Shapes of tensors ${expected.name} do not match" }
 
+            when (expected.data.info.type) {
+                OnnxJavaType.INT32 -> ArrayAssertions.assertArrayEquals(expected.toIntArray(), actual.toIntArray(), delta = 0.0) { "Tensors ${expected.name} do not match" }
+                OnnxJavaType.BOOL -> ArrayAssertions.assertArrayEquals(expected.toBooleanArray(), actual.toBooleanArray(), delta = 0.0) { "Tensors ${expected.name} do not match"}
+                else -> error("Unsupported data type: ${expected.data.info.type}")
+            }
         }
     }
 }
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -11,6 +11,7 @@ plugins {
     kotlin("multiplatform") apply false
     idea apply true
     `maven-publish`
+    id("io.kinference.primitives") version "0.1.26" apply false
 }
 
 allprojects {

diff --git a/...rence-core/src/commonMain/kotlin/io/kinference.core/operators/ml/TreeEnsembleRegressor.kt b/...rence-core/src/commonMain/kotlin/io/kinference.core/operators/ml/TreeEnsembleRegressor.kt
@@ -6,8 +6,8 @@ import io.kinference.core.data.tensor.asTensor
 import io.kinference.core.operators.ml.trees.*
 import io.kinference.data.ONNXData
 import io.kinference.graph.Contexts
-import io.kinference.ndarray.arrays.NumberNDArray
-import io.kinference.ndarray.arrays.NumberNDArrayCore
+import io.kinference.ndarray.arrays.*
+import io.kinference.ndarray.blockSizeByStrides
 import io.kinference.operator.*
 import io.kinference.protobuf.message.AttributeProto.AttributeType
 import io.kinference.protobuf.message.TensorProto
@@ -100,6 +100,18 @@ class TreeEnsembleRegressorVer1(
 
     override suspend fun <D : ONNXData<*, *>> apply(contexts: Contexts<D>, inputs: List<KITensor?>): List<KITensor?> {
         val inputData = inputs[0]!!.data as NumberNDArrayCore
-        return listOf(treeEnsemble.execute(inputData).asTensor("Y"))
+
+        val output = treeEnsemble.execute(inputData)
+
+        val outputBlockSize = blockSizeByStrides(output.strides)
+        val actualOutput = if (outputBlockSize != output.array.blockSize) {
+            val newOutput = FloatNDArray(output.strides)
+            output.array.copyInto(newOutput.array)
+            newOutput
+        } else {
+            output
+        }
+
+        return listOf(actualOutput.asTensor("Y"))
     }
 }
diff --git a/inference/inference-core/src/commonTest/kotlin/io/kinference/KITestEngine.kt b/inference/inference-core/src/commonTest/kotlin/io/kinference/KITestEngine.kt
@@ -7,13 +7,17 @@ import io.kinference.core.data.seq.KIONNXSequence
 import io.kinference.data.ONNXDataType
 import io.kinference.runners.AccuracyRunner
 import io.kinference.runners.PerformanceRunner
-import io.kinference.utils.KIAssertions
+import io.kinference.utils.*
 
-object KITestEngine : TestEngine<KIONNXData<*>>(KIEngine) {
+object KITestEngine : TestEngine<KIONNXData<*>>(KIEngine), Cacheable {
     override fun checkEquals(expected: KIONNXData<*>, actual: KIONNXData<*>, delta: Double) {
         KIAssertions.assertEquals(expected, actual, delta)
     }
 
+    override fun calculateErrors(expected: KIONNXData<*>, actual: KIONNXData<*>): List<Errors.ErrorsData> {
+        return KIErrors.calculateErrors(expected, actual)
+    }
+
     override fun getInMemorySize(data: KIONNXData<*>): Int {
         return when(data.type) {
             ONNXDataType.ONNX_TENSOR -> 1
@@ -22,6 +26,10 @@ object KITestEngine : TestEngine<KIONNXData<*>>(KIEngine) {
         }
     }
 
+    override fun clearCache() {
+        KIEngine.clearCache()
+    }
+
     val KIAccuracyRunner = AccuracyRunner(KITestEngine)
     val KIPerformanceRunner = PerformanceRunner(KITestEngine)
 }
diff --git a/...rence/inference-core/src/commonTest/kotlin/io/kinference/models/SimultaneousModelsTest.kt b/...rence/inference-core/src/commonTest/kotlin/io/kinference/models/SimultaneousModelsTest.kt
@@ -11,11 +11,11 @@ class SimultaneousModelsTest {
     @Test
     fun heavy_test_jvm_simultaneous_bert_electra_models() = TestRunner.runTest(Platform.JVM) {
         val bertDeferred = async {
-            KIAccuracyRunner.runFromS3("bert:standard:en:v1")
+            KIAccuracyRunner.runFromS3("bert:standard:en:v1", errorsVerbose = false)
         }
 
         val electraDeferred = async {
-            KIAccuracyRunner.runFromS3("bert:electra")
+            KIAccuracyRunner.runFromS3("bert:electra", errorsVerbose = false)
         }
 
         bertDeferred.await()

diff --git a/inference/inference-core/src/commonTest/kotlin/io/kinference/models/bert/ElectraTest.kt b/inference/inference-core/src/commonTest/kotlin/io/kinference/models/bert/ElectraTest.kt
@@ -9,7 +9,7 @@ import kotlin.test.Test
 class ElectraTest {
     @Test
     fun heavy_test_jvm_electra() = TestRunner.runTest(Platform.JVM) {
-        KIAccuracyRunner.runFromS3("bert:electra")
+        KIAccuracyRunner.runFromS3("bert:electra", errorsVerbose = false)
     }
 
     @Test

diff --git a/...ence/inference-core/src/commonTest/kotlin/io/kinference/operators/ml/SvmClassifierTest.kt b/...ence/inference-core/src/commonTest/kotlin/io/kinference/operators/ml/SvmClassifierTest.kt
@@ -74,7 +74,7 @@ class SvmClassifierTest {
 
     @Test
     fun test_labels_string() = TestRunner.runTest {
-        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_labels_string"))
+        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_labels_string"), errorsVerbose = false)
     }
 
     @Test

diff --git a/...ence-core/src/commonTest/kotlin/io/kinference/operators/operations/ReverseSequenceTest.kt b/...ence-core/src/commonTest/kotlin/io/kinference/operators/operations/ReverseSequenceTest.kt
@@ -14,7 +14,7 @@ class ReverseSequenceTest {
 
     @Test
     fun test_reverse_sequence_string_batch() = TestRunner.runTest {
-        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_reverse_sequence_string_batch"))
+        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_reverse_sequence_string_batch"), errorsVerbose = false)
     }
 
     @Test
@@ -24,7 +24,7 @@ class ReverseSequenceTest {
 
     @Test
     fun test_reverse_sequence_string_batch_3d() = TestRunner.runTest {
-        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_reverse_sequence_string_batch_3d"))
+        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_reverse_sequence_string_batch_3d"), errorsVerbose = false)
     }
 
     @Test
@@ -39,7 +39,7 @@ class ReverseSequenceTest {
 
     @Test
     fun test_reverse_sequence_string_time() = TestRunner.runTest {
-        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_reverse_sequence_string_time"))
+        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_reverse_sequence_string_time"), errorsVerbose = false)
     }
 
     @Test
@@ -49,7 +49,7 @@ class ReverseSequenceTest {
 
     @Test
     fun test_reverse_sequence_string_time_3d() = TestRunner.runTest {
-        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_reverse_sequence_string_time_3d"))
+        KITestEngine.KIAccuracyRunner.runFromResources(getTargetPath("test_reverse_sequence_string_time_3d"), errorsVerbose = false)
     }
 
     @Test