Skip to content

Commit

Permalink
Merge branch 'main' into 141-serializable-metrics
Browse files Browse the repository at this point in the history 
# Conflicts:
#	stars-core/src/main/kotlin/tools/aqua/stars/core/evaluation/TSCEvaluation.kt
#	stars-core/src/main/kotlin/tools/aqua/stars/core/metric/utils/ApplicationConstantsHolder.kt
  • Loading branch information
@dominikmaeckel
dominikmaeckel committed Aug 20, 2024
2 parents 91e6a9c + 4757691 commit 71feb19
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 128 deletions.
220 changes: 103 additions & 117 deletions stars-core/src/main/kotlin/tools/aqua/stars/core/evaluation/TSCEvaluation.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,138 +95,124 @@ class TSCEvaluation<
writePlotDataCSV: Boolean = false,
writeSerializedResults: Boolean = true
) {
try {
require(metricProviders.any()) {
"There needs to be at least one registered MetricProviders."
}
require(metricProviders.any()) { "There needs to be at least one registered MetricProviders." }

val totalEvaluationTime = measureTime {
/** Holds the [List] of [TSCProjection] based on the base [tsc]. */
var tscProjections: List<TSCProjection<E, T, S, U, D>>
val totalEvaluationTime = measureTime {
/** Holds the [List] of [TSCProjection] based on the base [tsc]. */
var tscProjections: List<TSCProjection<E, T, S, U, D>>

// Build all projections of the base TSC
val tscProjectionCalculationTime = measureTime {
tscProjections = tsc.buildProjections(projectionIgnoreList)
}
// Build all projections of the base TSC
val tscProjectionCalculationTime = measureTime {
tscProjections = tsc.buildProjections(projectionIgnoreList)
}

logFine(
"The calculation of the projections for the given tsc took: $tscProjectionCalculationTime")

val segmentsEvaluationTime = measureTime {
segments
.forEachIndexed { index, segment ->
print("\rCurrently evaluating segment $index")
val segmentEvaluationTime = measureTime {
// Run the "evaluate" function for all SegmentMetricProviders on the current
// segment
metricProviders.filterIsInstance<SegmentMetricProvider<E, T, S, U, D>>().forEach {
it.evaluate(segment)
}
val projectionsEvaluationTime = measureTime {
tscProjections.forEach { projection ->
val projectionEvaluationTime = measureTime {
// Run the "evaluate" function for all ProjectionMetricProviders on the
// current
// segment
metricProviders
.filterIsInstance<ProjectionMetricProvider<E, T, S, U, D>>()
.forEach { it.evaluate(projection) }
// Holds the PredicateContext for the current segment
val context = PredicateContext(segment)

// Holds the [TSCInstanceNode] of the current [projection] using the
// [PredicateContext], representing a whole TSC.
val segmentProjectionTSCInstance = projection.tsc.evaluate(context)

// Run the "evaluate" function for all TSCInstanceMetricProviders on the
// current segment
metricProviders
.filterIsInstance<TSCInstanceMetricProvider<E, T, S, U, D>>()
.forEach { it.evaluate(segmentProjectionTSCInstance) }

// Run the "evaluate" function for all
// ProjectionAndTSCInstanceNodeMetricProviders on the current projection and
// instance
metricProviders
.filterIsInstance<
ProjectionAndTSCInstanceNodeMetricProvider<E, T, S, U, D>>()
.forEach { it.evaluate(projection, segmentProjectionTSCInstance) }
}
logFine(
"The evaluation of projection '${projection.id}' for segment '$segment' took: $projectionEvaluationTime")
logFine(
"The calculation of the projections for the given tsc took: $tscProjectionCalculationTime")

val segmentsEvaluationTime = measureTime {
segments
.forEachIndexed { index, segment ->
print("\rCurrently evaluating segment $index")
val segmentEvaluationTime = measureTime {
// Run the "evaluate" function for all SegmentMetricProviders on the current
// segment
metricProviders.filterIsInstance<SegmentMetricProvider<E, T, S, U, D>>().forEach {
it.evaluate(segment)
}
val projectionsEvaluationTime = measureTime {
tscProjections.forEach { projection ->
val projectionEvaluationTime = measureTime {
// Run the "evaluate" function for all ProjectionMetricProviders on the
// current
// segment
metricProviders
.filterIsInstance<ProjectionMetricProvider<E, T, S, U, D>>()
.forEach { it.evaluate(projection) }
// Holds the PredicateContext for the current segment
val context = PredicateContext(segment)

// Holds the [TSCInstanceNode] of the current [projection] using the
// [PredicateContext], representing a whole TSC.
val segmentProjectionTSCInstance = projection.tsc.evaluate(context)

// Run the "evaluate" function for all TSCInstanceMetricProviders on the
// current segment
metricProviders
.filterIsInstance<TSCInstanceMetricProvider<E, T, S, U, D>>()
.forEach { it.evaluate(segmentProjectionTSCInstance) }

// Run the "evaluate" function for all
// ProjectionAndTSCInstanceNodeMetricProviders on the current projection and
// instance
metricProviders
.filterIsInstance<
ProjectionAndTSCInstanceNodeMetricProvider<E, T, S, U, D>>()
.forEach { it.evaluate(projection, segmentProjectionTSCInstance) }
}
logFine(
"The evaluation of projection '${projection.id}' for segment '$segment' took: $projectionEvaluationTime")
}
logFine(
"The evaluation of all projections for segment '$segment' took: $projectionsEvaluationTime")
}
logFine("The evaluation of segment '$segment' took: $segmentEvaluationTime")
logFine(
"The evaluation of all projections for segment '$segment' took: $projectionsEvaluationTime")
}
.also { println() }
}
logInfo("The evaluation of all segments took: $segmentsEvaluationTime")
logFine("The evaluation of segment '$segment' took: $segmentEvaluationTime")
}
.also { println() }
}
logInfo("The whole evaluation took: $totalEvaluationTime")
logInfo("The evaluation of all segments took: $segmentsEvaluationTime")
}
logInfo("The whole evaluation took: $totalEvaluationTime")

// Print the results of all Stateful metrics
metricProviders.filterIsInstance<Stateful>().forEach { it.printState() }
// Print the results of all Stateful metrics
metricProviders.filterIsInstance<Stateful>().forEach { it.printState() }

// Call the 'evaluate' and then the 'print' function for all PostEvaluationMetricProviders
println("Running post evaluation metrics")
metricProviders.filterIsInstance<PostEvaluationMetricProvider<E, T, S, U, D>>().forEach {
it.postEvaluate()
it.printPostEvaluationResult()
}
// Call the 'evaluate' and then the 'print' function for all PostEvaluationMetricProviders
println("Running post evaluation metrics")
metricProviders.filterIsInstance<PostEvaluationMetricProvider<E, T, S, U, D>>().forEach {
it.postEvaluate()
it.printPostEvaluationResult()
}

// Plot the results of all Plottable metrics
if (writePlots) {
println("Creating Plots")
metricProviders.filterIsInstance<Plottable>().forEach { it.writePlots() }
}
// Plot the results of all Plottable metrics
if (writePlots) {
println("Creating Plots")
metricProviders.filterIsInstance<Plottable>().forEach { it.writePlots() }
}

// Write CSV of the results of all Plottable metrics
if (writePlotDataCSV) {
println("Writing CSVs")
metricProviders.filterIsInstance<Plottable>().forEach { it.writePlotDataCSV() }
}
// Write CSV of the results of all Plottable metrics
if (writePlotDataCSV) {
println("Writing CSVs")
metricProviders.filterIsInstance<Plottable>().forEach { it.writePlotDataCSV() }
}

// Write JSON files of all Serializable metrics
if (writeSerializedResults) {
val serializableMetrics = metricProviders.filterIsInstance<Serializable>()
if (serializableMetrics.any()) {
println("Writing serialized results")
metricProviders.filterIsInstance<Serializable>().let {
it.forEach { t -> t.writeSerializedResults() }

it.compareToGroundTruthResults()
.also { comparisonResults ->
resultsReproducedFromGroundTruth =
comparisonResults.all { t ->
t.verdict in listOf(EQUAL_RESULTS, NEW_METRIC_SOURCE, NEW_IDENTIFIER)
}
}
.forEach { resultComparison -> resultComparison.saveAsJsonFile(true) }

it.compareToLatestResults()
.also { comparisonResults ->
resultsReproducedFromLatestRun =
comparisonResults.all { t ->
t.verdict in listOf(EQUAL_RESULTS, NEW_METRIC_SOURCE, NEW_IDENTIFIER)
}
}
.forEach { resultComparison -> resultComparison.saveAsJsonFile(false) }
}
// Write JSON files of all Serializable metrics
if (writeSerializedResults) {
val serializableMetrics = metricProviders.filterIsInstance<Serializable>()
if (serializableMetrics.any()) {
println("Writing serialized results")
metricProviders.filterIsInstance<Serializable>().let {
it.forEach { t -> t.writeSerializedResults() }

it.compareToGroundTruthResults()
.also { comparisonResults ->
resultsReproducedFromGroundTruth =
comparisonResults.all { t ->
t.verdict in listOf(EQUAL_RESULTS, NEW_METRIC_SOURCE, NEW_IDENTIFIER)
}
}
.forEach { resultComparison -> resultComparison.saveAsJsonFile(true) }

it.compareToLatestResults()
.also { comparisonResults ->
resultsReproducedFromLatestRun =
comparisonResults.all { t ->
t.verdict in listOf(EQUAL_RESULTS, NEW_METRIC_SOURCE, NEW_IDENTIFIER)
}
}
.forEach { resultComparison -> resultComparison.saveAsJsonFile(false) }
}
}
} finally {
// Close all logging handlers to prevent .lck files to remain
println("Closing Loggers")
metricProviders.filterIsInstance<Loggable>().forEach { it.closeLogger() }
closeLogger()

if (writeSerializedResults) {
println("Results reproduced from ground truth: $resultsReproducedFromGroundTruth")
println("Results reproduced from latest run: $resultsReproducedFromLatestRun")
}
}
}
}
25 changes: 14 additions & 11 deletions stars-core/src/main/kotlin/tools/aqua/stars/core/metric/providers/Loggable.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package tools.aqua.stars.core.metric.providers
import java.io.File
import java.util.logging.*
import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder
import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.activeLoggers

/** This interface can be implemented to be able to log data into the stdout and log files. */
@Suppress("unused")
Expand Down Expand Up @@ -116,17 +117,19 @@ interface Loggable {
}
val file = "$logFolderFile/$name-${currentTimeAndDate}"

return@run Logger.getAnonymousLogger().apply {
useParentHandlers = false
level = Level.FINEST

addHandler(getLoggerHandler(file, Level.SEVERE))
addHandler(getLoggerHandler(file, Level.WARNING))
addHandler(getLoggerHandler(file, Level.INFO))
addHandler(getLoggerHandler(file, Level.FINE))
addHandler(getLoggerHandler(file, Level.FINER))
addHandler(getLoggerHandler(file, Level.FINEST))
}
return@run Logger.getAnonymousLogger()
.apply {
useParentHandlers = false
level = Level.FINEST

addHandler(getLoggerHandler(file, Level.SEVERE))
addHandler(getLoggerHandler(file, Level.WARNING))
addHandler(getLoggerHandler(file, Level.INFO))
addHandler(getLoggerHandler(file, Level.FINE))
addHandler(getLoggerHandler(file, Level.FINER))
addHandler(getLoggerHandler(file, Level.FINEST))
}
.also { activeLoggers.add(it) }
}

private fun getLoggerHandler(file: String, level: Level) =
Expand Down
5 changes: 5 additions & 0 deletions stars-core/src/main/kotlin/tools/aqua/stars/core/metric/utils/ApplicationConstantsHolder.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import java.io.File
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.util.logging.LogManager
import java.util.logging.Logger
import kotlinx.serialization.json.Json

/**
Expand Down Expand Up @@ -57,6 +58,9 @@ object ApplicationConstantsHolder {
val logFolder: String
get() = if (isTestRun()) TEST_LOG_FOLDER else ANALYSIS_LOG_FOLDER

/** Holds the [MutableList] of all currently registered [Logger]s. */
val activeLoggers: MutableList<Logger> = mutableListOf()

/** Holds the folder name for the logs. */
val serializedResultsFolder: String
get() = if (isTestRun()) "test-$SERIALIZED_RESULTS_FOLDER" else SERIALIZED_RESULTS_FOLDER
Expand Down Expand Up @@ -84,6 +88,7 @@ object ApplicationConstantsHolder {
.addShutdownHook(
Thread {
LogManager.getLogManager().reset()
activeLoggers.forEach { it.handlers.forEach { handler -> handler.close() } }
File(TEST_LOG_FOLDER).deleteRecursively()
File("test-$SERIALIZED_RESULTS_FOLDER").deleteRecursively()
File("test-$COMPARED_RESULTS_FOLDER").deleteRecursively()
Expand Down

0 comments on commit 71feb19

Please sign in to comment.