M3SA - Multi-Meta-Model Simulation Analyzer

.gitignore

@@ -115,3 +115,6 @@ gradle-app.setting
 
 # Cache of project
 .gradletasknamecache
+
+# Demo files
+demo/*

opendc-experiments/opendc-experiments-base/build.gradle.kts

@@ -37,6 +37,7 @@ dependencies {
     api(projects.opendcCompute.opendcComputeSimulator)
 
     implementation(libs.clikt)
+    implementation(project(":opendc-experiments:opendc-m3sa"))
 
     implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.0")
     implementation(libs.progressbar)

opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ExperimentCli.kt

@@ -27,10 +27,12 @@ package org.opendc.experiments.base.runner
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.parameters.options.default
 import com.github.ajalt.clikt.parameters.options.defaultLazy
+import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.types.file
 import com.github.ajalt.clikt.parameters.types.int
-import org.opendc.experiments.base.scenario.getExperiment
+import m3saRun
+import org.opendc.experiments.base.scenario.getExperiments
 import java.io.File
 
 /**
@@ -56,8 +58,18 @@ internal class ExperimentCommand : CliktCommand(name = "experiment") {
         .int()
         .default(Runtime.getRuntime().availableProcessors() - 1)
 
+    private val analyzeResults by option("-a", "--analyze-results", help = "analyze the results")
+        .flag(default = false)
+
     override fun run() {
-        val experiment = getExperiment(scenarioPath)
-        runExperiment(experiment, parallelism)
+        val experiments = getExperiments(scenarioPath)
+        runExperiment(experiments, parallelism)
+
+        if (analyzeResults) {
+            m3saRun(
+                outputFolderPath = experiments[0].outputFolder,
+                m3saSetupPath = experiments[0].m3saSetup,
+            )
+        }
     }
 }

opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ExperimentFactories.kt

@@ -35,8 +35,8 @@ private val experimentWriter = ExperimentWriter()
  * @param filePath The path to the file containing the scenario specifications.
  * @return A list of Scenarios.
  */
-public fun getExperiment(filePath: String): List<Scenario> {
-    return getExperiment(File(filePath))
+public fun getExperiments(filePath: String): List<Scenario> {
+    return getExperiments(File(filePath))
 }
 
 /**
@@ -45,8 +45,8 @@ public fun getExperiment(filePath: String): List<Scenario> {
  * @param file The file containing the scenario specifications.
  * @return A list of Scenarios.
  */
-public fun getExperiment(file: File): List<Scenario> {
-    return getExperiment(experimentReader.read(file))
+public fun getExperiments(file: File): List<Scenario> {
+    return getExperiments(experimentReader.read(file))
 }
 
 /**
@@ -56,7 +56,7 @@ public fun getExperiment(file: File): List<Scenario> {
  * @param experimentSpec The ScenarioSpec containing the scenario specifications.
  * @return A list of Scenarios.
  */
-public fun getExperiment(experimentSpec: ExperimentSpec): List<Scenario> {
+public fun getExperiments(experimentSpec: ExperimentSpec): List<Scenario> {
     val outputFolder = experimentSpec.outputFolder + "/" + experimentSpec.name
     File(outputFolder).mkdirs()
 
@@ -79,6 +79,7 @@ public fun getExperiment(experimentSpec: ExperimentSpec): List<Scenario> {
                 carbonTracePath = scenarioSpec.carbonTracePath,
                 exportModelSpec = scenarioSpec.exportModel,
                 outputFolder = outputFolder,
+                m3saSetup = experimentSpec.m3saSetup,
                 name = scenarioID.toString(),
                 runs = experimentSpec.runs,
                 initialSeed = experimentSpec.initialSeed,

opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt

@@ -52,6 +52,7 @@ public data class Scenario(
     val failureModelSpec: FailureModelSpec?,
     val checkpointModelSpec: CheckpointModelSpec?,
     val carbonTracePath: String? = null,
+    val m3saSetup: String = "",
     val exportModelSpec: ExportModelSpec = ExportModelSpec(),
     val outputFolder: String = "output",
     val computeExportConfig: ComputeExportConfig,

opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ExperimentSpec.kt

@@ -42,13 +42,15 @@ import java.util.UUID
  * @property computeExportConfig configures which parquet columns are to
  * be included in the output files.
  */
+
 @Serializable
 public data class ExperimentSpec(
     var id: Int = -1,
     var name: String = "",
     val outputFolder: String = "output",
     val initialSeed: Int = 0,
     val runs: Int = 1,
+    val m3saSetup: String = "",
     val topologies: Set<ScenarioTopologySpec>,
     val workloads: Set<WorkloadSpec>,
     val allocationPolicies: Set<AllocationPolicySpec> = setOf(AllocationPolicySpec()),

opendc-experiments/opendc-m3sa/build.gradle.kts

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2020 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+description = "Multi-Meta-Model Simulation Analysis (M3SA) used across OpenDC modules"
+
+// Build configuration
+plugins {
+    `kotlin-library-conventions`
+}
+
+dependencies {
+    api(libs.kotlinx.coroutines)
+
+    testImplementation(projects.opendcSimulator.opendcSimulatorCore)
+}
+
+sourceSets {
+    main {
+        kotlin.srcDirs("src/main/kotlin", "src/main/python")
+        resources.srcDir("src/main/resources")
+    }
+}

opendc-experiments/opendc-m3sa/src/main/kotlin/M3saRunner.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2024 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+import kotlin.io.path.Path
+
+/**
+ * This constant variable should be changed depending on the root folder that is being run.
+ * PATH_TO_PYTHON_MAIN should point to the main python file, ran when the analysis starts.
+ */
+
+public val ANALYSIS_SCRIPTS_DIRECTORY: String = "./opendc-experiments/opendc-m3sa/src/main/python"
+public val ABSOLUTE_SCRIPT_PATH: String =
+    Path("$ANALYSIS_SCRIPTS_DIRECTORY/main.py").toAbsolutePath().normalize().toString()
+public val SCRIPT_LANGUAGE: String = "python3"
+
+public fun m3saRun(
+    outputFolderPath: String,
+    m3saSetupPath: String,
+) {
+    val process =
+        ProcessBuilder(
+            SCRIPT_LANGUAGE,
+            ABSOLUTE_SCRIPT_PATH,
+            outputFolderPath,
+            m3saSetupPath,
+        ).directory(Path(ANALYSIS_SCRIPTS_DIRECTORY).toFile())
+            .start()
+
+    val exitCode = process.waitFor()
+    if (exitCode == 0) {
+        println("[M3SA says] M3SA operation(s) completed successfully.")
+    } else {
+        val errors = process.errorStream.bufferedReader().readText()
+        println("[M3SA says] Exit code $exitCode; Error(s): $errors")
+    }
+}

opendc-experiments/opendc-m3sa/src/main/python/accuracy_evaluator.py

@@ -0,0 +1,114 @@
+import numpy as np
+
+from models.MetaModel import MetaModel
+
+
+def accuracy_evaluator(
+    real_data,
+    multi_model,
+    compute_mape=True,
+    compute_nad=True,
+    compute_rmsle=True,
+    rmsle_hyperparameter=0.5,
+    only_metamodel=False
+):
+    """
+    :param real_data: the real-world data of the simulation
+    :param multi_model: the Multi-Model, containing individual models (possibly also a Meta-Model, with id=101)
+    :param MAPE: whether to calculate Mean Absolute Percentage Error (MAPE)
+    :param NAD: whether to calculate Normalized Absolute Differences (NAD)
+    :param RMSLE: whether to calculate Root Mean Square Logarithmic Error (RMSLE)
+    :param rmsle_hyperparameter: the hyperparameter that balances the ration underestimations:overestimations
+        - default is 0.5 (balanced penalty)
+        - < 0.5: more penalty for overestimations
+        - > 0.5: more penalty for underestimations
+        e.g., RMSLE_hyperparameter=0.3 -> 30% penalty for overestimations, 70% penalty for underestimations (3:7 ratio)
+    :return: None, but prints the accuracy metrics
+    """
+
+    meta_model = MetaModel(multimodel=multi_model)
+    multi_model.models.append(meta_model.meta_model)  # metamodel
+    # multi_model.models.append(Model(raw_host_data=real_data, id=-1, path=None))  # real-world data
+
+    with open(multi_model.output_folder_path + "/accuracy_report.txt", "a") as f:
+        f.write("====================================\n")
+        f.write("Accuracy Report, against ground truth\n")
+
+        for model in multi_model.models:
+            if only_metamodel and model.id != 101:
+                continue
+
+            if model.id == -1:
+                f.write("Real-World data")
+            elif model.id == 101:
+                f.write(
+                    f"Meta-Model, meta-function: {multi_model.user_input['meta_function']}, window_size: {meta_model.multi_model.window_size}")
+            else:
+                f.write(f"Model {model.id}")
+
+            simulation_data = model.raw_sim_data
+            min_len = min(len(real_data), len(simulation_data))
+            real_data = real_data[:min_len]
+            simulation_data = simulation_data[:min_len]
+            if compute_mape:
+                accuracy_mape = mape(
+                    real_data=real_data,
+                    simulation_data=simulation_data
+                )
+                f.write(f"\nMean Absolute Percentage Error (MAPE): {accuracy_mape}%")
+
+            if compute_nad:
+                accuracy_nad = nad(
+                    real_data=real_data,
+                    simulation_data=simulation_data
+                )
+                f.write(f"\nNormalized Absolute Differences (NAD): {accuracy_nad}%")
+
+            if compute_rmsle:
+                accuracy_rmsle = rmsle(
+                    real_data=real_data,
+                    simulation_data=simulation_data,
+                    alpha=rmsle_hyperparameter
+                )
+                f.write(
+                    f"\nRoot Mean Square Logarithmic Error (RMSLE), alpha={rmsle_hyperparameter}:{accuracy_rmsle}\n\n")
+
+        f.write("====================================\n")
+
+
+def mape(real_data, simulation_data):
+    """
+    Calculate Mean Absolute Percentage Error (MAPE)
+    :param real_data: Array of real values
+    :param simulation_data: Array of simulated values
+    :return: MAPE value
+    """
+    real_data = np.array(real_data)
+    simulation_data = np.array(simulation_data)
+    return round(np.mean(np.abs((real_data - simulation_data) / real_data)) * 100, 3)
+
+
+def nad(real_data, simulation_data):
+    """
+    Calculate Normalized Absolute Differences (NAD)
+    :param real_data: Array of real values
+    :param simulation_data: Array of simulated values
+    :return: NAD value
+    """
+    real_data = np.array(real_data)
+    simulation_data = np.array(simulation_data)
+    return round(np.sum(np.abs(real_data - simulation_data)) / np.sum(real_data) * 100, 3)
+
+
+def rmsle(real_data, simulation_data, alpha=0.5):
+    """
+    Calculate Root Mean Square Logarithmic Error (RMSLE) with an adjustable alpha parameter
+    :param real_data: Array of real values
+    :param simulation_data: Array of simulated values
+    :param alpha: Hyperparameter that balances the penalty between underestimations and overestimations
+    :return: RMSLE value
+    """
+    real_data = np.array(real_data)
+    simulation_data = np.array(simulation_data)
+    log_diff = alpha * np.log(real_data) - (1 - alpha) * np.log(simulation_data)
+    return round(np.sqrt(np.mean(log_diff ** 2)) * 100, 3)