Add serializable metrics (#156)

Co-authored-by: Till Schallau <till.schallau@tu-dortmund.de>

.github/workflows/analyze-build-deploy.yml

@@ -2,7 +2,7 @@ name: Analyze, Build and Deploy
 
 on:
   pull_request:
-    types: [opened, synchronize]
+    types: [opened, synchronize, ready_for_review]
     branches:
       - main
   push:
@@ -68,7 +68,7 @@ jobs:
         uses: gradle/actions/setup-gradle@v4
 
       - name: Discover Projects and save into matrix
-        run: | 
+        run: |
           ./gradlew projects --quiet | grep "Project ':" | awk -F"'" '{print $2}' | sed 's/^://' > projects.txt
           projects=$(cat projects.txt | sed '/^$/d' | awk '{print "\""$0"\""}' | paste -sd "," - | sed 's/^/[/' | sed 's/$/]/')
           echo "matrix=${projects}" >> $GITHUB_OUTPUT
@@ -106,7 +106,7 @@ jobs:
         mkdir sarif &&
         find * -name detekt.sarif -print0 |
         xargs -n1 -0 bash -c 'cp "$1" "sarif/${1//\//-}"' '{}'
-      
+
     - name: Upload SARIF file
       uses: github/codeql-action/upload-sarif@v3
       with:
@@ -142,10 +142,30 @@ jobs:
 
     - name: Upload test results
       uses: mikepenz/action-junit-report@v4
-      if: always() # Must execute after failed tests
+      if: ${{ !cancelled() }} # Must execute after failed tests
       with:
         report_paths: '**/build/test-results/test/TEST-*.xml'
 
+  reproduction:
+    name: Trigger reproduction check
+    needs: compile
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger GitLab
+        run: |
+          curl -X POST \
+           --fail \
+           -F token=$REPRODUCTION_RUNNER_TRIGGER_TOKEN \
+           -F ref=main \
+           -F variables[TARGET_BRANCH]=${{ github.head_ref || github.ref_name }} \
+           -F variables[IS_DRAFT]=${{ github.event.pull_request.draft }} \
+           -F variables[TARGET_SHA]=${{ github.event.pull_request.head.sha }} \
+           -s -o /dev/null \
+           $REPRODUCTION_RUNNER_URL
+        env:
+          REPRODUCTION_RUNNER_TRIGGER_TOKEN: ${{ secrets.REPRODUCTION_RUNNER_TRIGGER_TOKEN }}
+          REPRODUCTION_RUNNER_URL: ${{ secrets.REPRODUCTION_RUNNER_URL }}
+
   spotless:
     name: Run Spotless (${{ matrix.project }})
     needs: discover-projects
@@ -173,11 +193,11 @@ jobs:
         run: ./gradlew :${{ matrix.project }}:spotlessCheck
 
       - name: Apply Spotless fixes
-        if: failure()
+        if: ${{ failure() }}
         run: ./gradlew :${{ matrix.project }}:spotlessApply
 
       - name: Generate diff
-        if: failure()
+        if: ${{ failure() }}
         shell: sh
         run: |
           echo "# Spotless violations" >> $GITHUB_STEP_SUMMARY

CHANGELOG.md

@@ -21,17 +21,23 @@ All notable changes to this project will be documented in this file.
 - Add `identifier` field to `TSC`.
 - Add `euclideanDistance` function to `Location`.
 - Add `vehicleType` field to `Vehicle`.
+- Add `loggerIdentifier` field to `Loggable` interface.
+- Add `Serializable` interface.
+  - This adds the functionality to compare your current analysis results with the previous run or a specified baseline. 
 
 ### Changed
 - `TSCEvaluation` now accepts multiple `TSCs` instead of `TSCProjections`.
 - `registerMetricProviders` now throws an `IllegalArgumentException` when multiple instances of the same `MetricProvider` class is registered.
+- Root nodes in a `TSC` now **must not** have a condition.
+- Move `label` from `TSCEdge` to `TSCNode`.
+- All default metrics now implement the new `Serializable` interface.
 - Rename `ProjectionMetricProvider` to `TSCMetricProvider`.
 - Rename `ProjectionAndTSCInstanceNodeMetricProvider` to `TSCAndTSCInstanceNodeMetricProvider`.
 - Rename `InvalidTSCInstancesPerProjectionMetric` to `InvalidTSCInstancesPerTSCMetric`.
 - Rename `ValidTSCInstancesPerProjectionMetric` to `ValidTSCInstancesPerTSCMetric`.
 - Rename `MissedTSCInstancesPerProjectionMetric` to `MissedTSCInstancesPerTSCMetric`.
 - Rename `MissingPredicateCombinationsPerProjectionMetric` to `MissingPredicateCombinationsPerTSCMetric`.
-- Root nodes in a `TSC` now must not have a condition.
+- Rename `DataSaver` to `PlotDataSaver`.
 
 ### Fixed
 - Fix `toString()` function of `TSCNode` to include the root node's label.

contrib/detekt-rules.yml

@@ -380,7 +380,7 @@ style:
   SafeCast:
     active: true
   SerialVersionUIDInSerializableClass:
-    active: true
+    active: false
   SpacingBetweenPackageAndImports:
     # Rationale: handled by spotless
     active: false

gradle/libs.versions.toml

@@ -21,7 +21,7 @@ letsplot-imageexport = { group = "org.jetbrains.lets-plot", name = "lets-plot-im
 kotlin-test = { group = "org.jetbrains.kotlin", name="kotlin-test" }
 
 # Gradle Plugins
-gradle-kotlin = { group = "org.jetbrains.kotlin", name = "kotlin-gradle-plugin", version = "2.0.0" }
+gradle-kotlin = { group = "org.jetbrains.kotlin", name = "kotlin-gradle-plugin", version = "2.0.21" }
 gradle-dokka = { group = "org.jetbrains.dokka", name = "dokka-gradle-plugin", version = "1.9.20" }
 gradle-detekt = { group = "io.gitlab.arturbosch.detekt", name = "detekt-gradle-plugin", version = "1.23.6" }
 gradle-bmVersions = { group = "com.github.ben-manes", name = "gradle-versions-plugin", version = "0.51.0" }

stars-core/build.gradle.kts

@@ -15,7 +15,10 @@
  * limitations under the License.
  */
 
-plugins { id("tools.aqua.stars.library-conventions") }
+plugins {
+  id("tools.aqua.stars.library-conventions")
+  kotlin("plugin.serialization") version "2.0.10"
+}
 
 mavenMetadata {
   name.set("STARS Core Library")
@@ -27,4 +30,5 @@ dependencies {
   implementation(libs.letsplot.imageexport)
   implementation(libs.slf4j.api)
   implementation(libs.slf4j.simple)
+  implementation(libs.kotlinx.serialization.json)
 }

stars-core/src/main/kotlin/tools/aqua/stars/core/evaluation/TSCEvaluation.kt

@@ -27,6 +27,11 @@ import tools.aqua.stars.core.hooks.PreSegmentEvaluationHook.Companion.evaluate
 import tools.aqua.stars.core.hooks.PreTSCEvaluationHook.Companion.evaluate
 import tools.aqua.stars.core.hooks.defaulthooks.MinTicksPerSegmentHook
 import tools.aqua.stars.core.metric.providers.*
+import tools.aqua.stars.core.metric.serialization.SerializableResultComparison.Companion.noMismatch
+import tools.aqua.stars.core.metric.serialization.extensions.compareToBaselineResults
+import tools.aqua.stars.core.metric.serialization.extensions.compareToPreviousResults
+import tools.aqua.stars.core.metric.serialization.extensions.writeSerializedResults
+import tools.aqua.stars.core.metric.utils.saveAsJsonFiles
 import tools.aqua.stars.core.tsc.TSC
 import tools.aqua.stars.core.tsc.instance.TSCInstanceNode
 import tools.aqua.stars.core.types.*
@@ -43,6 +48,13 @@ import tools.aqua.stars.core.types.*
  * @property tscList The list of [TSC]s to evaluate.
  * @property writePlots (Default: ``true``) Whether to write plots after the analysis.
  * @property writePlotDataCSV (Default: ``false``) Whether to write CSV files after the analysis.
+ * @property writeSerializedResults (Default: ``true``) Whether to write result files and compare
+ *   them to previous runs after the analysis.
+ * @property compareToBaselineResults (Default: ``false``) Whether to compare the results to the
+ *   baseline results.
+ * @property compareToPreviousRun (Default: ``false``) Whether to compare the results to the
+ *   previous run.
+ * @property loggerIdentifier identifier (name) for the logger.
  * @property logger [Logger] instance.
  */
 class TSCEvaluation<
@@ -54,9 +66,46 @@ class TSCEvaluation<
     val tscList: List<TSC<E, T, S, U, D>>,
     val writePlots: Boolean = true,
     val writePlotDataCSV: Boolean = false,
-    override val logger: Logger = Loggable.getLogger("evaluation-time")
+    val writeSerializedResults: Boolean = true,
+    val compareToBaselineResults: Boolean = false,
+    val compareToPreviousRun: Boolean = false,
+    override val loggerIdentifier: String = "evaluation-time",
+    override val logger: Logger = Loggable.getLogger(loggerIdentifier),
 ) : Loggable {
 
+  /** Test. */
+  private val mutex: Any = Any()
+
+  /**
+   * Holds the aggregated [Boolean] verdict of all compared results with the baseline data. Setting
+   * a new value will be conjugated with the old value such that a verdict 'false' may not be
+   * changed to 'true' again.
+   */
+  var resultsReproducedFromBaseline: Boolean? = null
+    set(value) {
+      synchronized(mutex) {
+        when {
+          field == null -> field = value
+          field != null && value != null -> field = field!! && value
+        }
+      }
+    }
+
+  /**
+   * Holds the aggregated [Boolean] verdict of all compared results with the previous evaluation
+   * results. Setting a new value will be conjugated with the old value such that a verdict 'false'
+   * may not be changed to 'true' again.
+   */
+  var resultsReproducedFromPreviousRun: Boolean? = null
+    set(value) {
+      synchronized(mutex) {
+        when {
+          field == null -> field = value
+          field != null && value != null -> field = field!! && value
+        }
+      }
+    }
+
   /** Holds a [List] of all [MetricProvider]s registered by [registerMetricProviders]. */
   private val metricProviders: MutableList<MetricProvider<E, T, S, U, D>> = mutableListOf()
 
@@ -71,7 +120,12 @@ class TSCEvaluation<
   private val preTSCEvaluationHooks: MutableList<PreTSCEvaluationHook<E, T, S, U, D>> =
       mutableListOf()
 
-  /** Holds the results of the [PreSegmentEvaluationHook]s after calling [runEvaluation]. */
+  /**
+   * Holds the results (Map of [PreSegmentEvaluationHook.identifier] to [EvaluationHookResult]) of
+   * the [PreSegmentEvaluationHook]s after calling [runEvaluation] that did not return
+   * [EvaluationHookResult.OK] for each segment identifier obtained by
+   * [SegmentType.getSegmentIdentifier].
+   */
   val preSegmentEvaluationHookResults: MutableMap<String, Map<String, EvaluationHookResult>> =
       mutableMapOf()
 
@@ -275,5 +329,34 @@ class TSCEvaluation<
       println("Writing CSVs")
       metricProviders.filterIsInstance<Plottable>().forEach { it.writePlotDataCSV() }
     }
+
+    val serializableMetrics = metricProviders.filterIsInstance<Serializable>()
+    if (serializableMetrics.any()) {
+      // Write JSON files of all Serializable metrics
+      if (writeSerializedResults) {
+        println("Writing serialized results")
+        serializableMetrics.forEach { t -> t.writeSerializedResults() }
+      }
+
+      // Compare the results to the baseline
+      if (compareToBaselineResults) {
+        println("Comparing to baseline")
+        serializableMetrics.compareToBaselineResults().let {
+          resultsReproducedFromBaseline = it.noMismatch()
+
+          if (writeSerializedResults) it.saveAsJsonFiles(comparedToBaseline = true)
+        }
+      }
+
+      // Compare the results to the latest run
+      if (compareToPreviousRun) {
+        println("Comparing to previous run")
+        serializableMetrics.compareToPreviousResults().let {
+          resultsReproducedFromPreviousRun = it.noMismatch()
+
+          if (writeSerializedResults) it.saveAsJsonFiles(comparedToBaseline = false)
+        }
+      }
+    }
   }
 }

stars-core/src/main/kotlin/tools/aqua/stars/core/metric/metrics/evaluation/InvalidTSCInstancesPerTSCMetric.kt

@@ -19,8 +19,12 @@ package tools.aqua.stars.core.metric.metrics.evaluation
 
 import java.util.logging.Logger
 import tools.aqua.stars.core.metric.providers.Loggable
+import tools.aqua.stars.core.metric.providers.Serializable
 import tools.aqua.stars.core.metric.providers.Stateful
 import tools.aqua.stars.core.metric.providers.TSCAndTSCInstanceNodeMetricProvider
+import tools.aqua.stars.core.metric.serialization.SerializableTSCOccurrenceResult
+import tools.aqua.stars.core.metric.serialization.tsc.SerializableTSCNode
+import tools.aqua.stars.core.metric.serialization.tsc.SerializableTSCOccurrence
 import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.CONSOLE_INDENT
 import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.CONSOLE_SEPARATOR
 import tools.aqua.stars.core.tsc.TSC
@@ -35,13 +39,17 @@ import tools.aqua.stars.core.types.*
  * This class implements the [Stateful] interface. Its state contains the [Map] of [TSC]s to a
  * [List] of invalid [TSCInstance]s.
  *
+ * This class implements the [Serializable] interface. It serializes all invalid [TSCInstance] for
+ * their respective [TSC].
+ *
  * This class implements [Loggable] and logs the final [Map] of invalid [TSCInstance]s for [TSC]s.
  *
  * @param E [EntityType].
  * @param T [TickDataType].
  * @param S [SegmentType].
  * @param U [TickUnit].
  * @param D [TickDifference].
+ * @property loggerIdentifier identifier (name) for the logger.
  * @property logger [Logger] instance.
  */
 @Suppress("unused")
@@ -51,8 +59,9 @@ class InvalidTSCInstancesPerTSCMetric<
     S : SegmentType<E, T, S, U, D>,
     U : TickUnit<U, D>,
     D : TickDifference<D>>(
-    override val logger: Logger = Loggable.getLogger("invalid-tsc-instances-per-tsc")
-) : TSCAndTSCInstanceNodeMetricProvider<E, T, S, U, D>, Stateful, Loggable {
+    override val loggerIdentifier: String = "invalid-tsc-instances-per-tsc",
+    override val logger: Logger = Loggable.getLogger(loggerIdentifier)
+) : TSCAndTSCInstanceNodeMetricProvider<E, T, S, U, D>, Stateful, Serializable, Loggable {
   /** Map the [TSC] to a map in which the occurrences of invalid [TSCInstance]s are stored. */
   private val invalidInstancesMap:
       MutableMap<
@@ -124,4 +133,19 @@ class InvalidTSCInstancesPerTSCMetric<
       }
     }
   }
+
+  override fun getSerializableResults(): List<SerializableTSCOccurrenceResult> =
+      invalidInstancesMap.map { (tsc, invalidInstances) ->
+        val resultList =
+            invalidInstances.map { (tscInstanceNode, tscInstances) ->
+              SerializableTSCOccurrence(
+                  tscInstance = SerializableTSCNode(tscInstanceNode),
+                  segmentIdentifiers = tscInstances.map { it.sourceSegmentIdentifier })
+            }
+        SerializableTSCOccurrenceResult(
+            identifier = tsc.identifier,
+            source = loggerIdentifier,
+            count = resultList.size,
+            value = resultList)
+      }
 }

stars-core/src/main/kotlin/tools/aqua/stars/core/metric/metrics/evaluation/MissedTSCInstancesPerTSCMetric.kt

@@ -18,9 +18,14 @@
 package tools.aqua.stars.core.metric.metrics.evaluation
 
 import java.util.logging.Logger
+import kotlin.collections.component1
+import kotlin.collections.component2
 import tools.aqua.stars.core.metric.providers.Loggable
+import tools.aqua.stars.core.metric.providers.Serializable
 import tools.aqua.stars.core.metric.providers.Stateful
 import tools.aqua.stars.core.metric.providers.TSCAndTSCInstanceNodeMetricProvider
+import tools.aqua.stars.core.metric.serialization.SerializableTSCResult
+import tools.aqua.stars.core.metric.serialization.tsc.SerializableTSCNode
 import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.CONSOLE_INDENT
 import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.CONSOLE_SEPARATOR
 import tools.aqua.stars.core.tsc.TSC
@@ -35,13 +40,17 @@ import tools.aqua.stars.core.types.*
  * This class implements the [Stateful] interface. Its state contains the [Map] of [TSC]s to a
  * [List] of missed [TSCInstance]s.
  *
+ * This class implements the [Serializable] interface. It serializes all missed [TSCInstance] for
+ * their respective [TSC].
+ *
  * This class implements [Loggable] and logs the final [Map] of missed [TSCInstance]s for [TSC]s.
  *
  * @param E [EntityType].
  * @param T [TickDataType].
  * @param S [SegmentType].
  * @param U [TickUnit].
  * @param D [TickDifference].
+ * @property loggerIdentifier identifier (name) for the logger.
  * @property logger [Logger] instance.
  */
 @Suppress("unused")
@@ -51,8 +60,9 @@ class MissedTSCInstancesPerTSCMetric<
     S : SegmentType<E, T, S, U, D>,
     U : TickUnit<U, D>,
     D : TickDifference<D>>(
-    override val logger: Logger = Loggable.getLogger("missed-tsc-instances-per-tsc")
-) : TSCAndTSCInstanceNodeMetricProvider<E, T, S, U, D>, Stateful, Loggable {
+    override val loggerIdentifier: String = "missed-tsc-instances-per-tsc",
+    override val logger: Logger = Loggable.getLogger(loggerIdentifier)
+) : TSCAndTSCInstanceNodeMetricProvider<E, T, S, U, D>, Stateful, Serializable, Loggable {
   /**
    * Map a [TSC] to a map in which the missed valid [TSCInstanceNode]s are stored:
    * Map<tsc,Map<referenceInstance,missed>>.
@@ -110,4 +120,17 @@ class MissedTSCInstancesPerTSCMetric<
       missedInstances.forEach { logFine(it) }
     }
   }
+
+  override fun getSerializableResults(): List<SerializableTSCResult> =
+      missedInstancesMap.map { (tsc, missedInstances) ->
+        val resultList =
+            missedInstances
+                .filter { it.value }
+                .map { (tscInstanceNode, _) -> SerializableTSCNode(tscInstanceNode) }
+        SerializableTSCResult(
+            identifier = tsc.identifier,
+            source = loggerIdentifier,
+            count = resultList.size,
+            value = resultList)
+      }
 }