feat(autotuner): create statistics graph output (#643)

* feat(autotuner): create statistics graph output

discopop_library/EmpiricalAutotuning/Autotuner.py

@@ -14,6 +14,7 @@
 from discopop_library.EmpiricalAutotuning.ArgumentClasses import AutotunerArguments
 from discopop_library.EmpiricalAutotuning.Classes.CodeConfiguration import CodeConfiguration
 from discopop_library.EmpiricalAutotuning.Classes.ExecutionResult import ExecutionResult
+from discopop_library.EmpiricalAutotuning.Statistics.StatisticsGraph import NodeShape, StatisticsGraph
 from discopop_library.EmpiricalAutotuning.Types import SUGGESTION_ID
 from discopop_library.EmpiricalAutotuning.utils import get_applicable_suggestion_ids
 from discopop_library.HostpotLoader.HotspotLoaderArguments import HotspotLoaderArguments
@@ -36,10 +37,17 @@ def get_unique_configuration_id() -> int:
 def run(arguments: AutotunerArguments) -> None:
     logger.info("Starting discopop autotuner.")
     debug_stats: List[Tuple[List[SUGGESTION_ID], float]] = []
+    statistics_graph = StatisticsGraph()
+    statistics_step_num = 0
 
     # get untuned reference result
     reference_configuration = CodeConfiguration(arguments.project_path, arguments.dot_dp_path)
-    reference_configuration.execute(timeout=None)
+    reference_configuration.execute(timeout=None, is_initial=True)
+    statistics_graph.set_root(
+        reference_configuration.get_statistics_graph_label(),
+        color=reference_configuration.get_statistics_graph_color(),
+        shape=NodeShape.BOX,
+    )
     timeout_after = cast(ExecutionResult, reference_configuration.execution_result).runtime * 2
     debug_stats.append(([], cast(ExecutionResult, reference_configuration.execution_result).runtime))
 
@@ -66,6 +74,21 @@ def run(arguments: AutotunerArguments) -> None:
         loop_tuples = hotspot_information[hotspot_type]
         sorted_loop_tuples = sorted(loop_tuples, key=lambda x: x[4], reverse=True)
         for loop_tuple in sorted_loop_tuples:
+            loop_str = (
+                ""
+                + str(loop_tuple[0])
+                + "@"
+                + str(loop_tuple[1])
+                + " - "
+                + str(loop_tuple[2])
+                + " "
+                + loop_tuple[3]
+                + " "
+                + str(round(loop_tuple[4], 3))
+                + "s"
+            )
+            statistics_graph.add_child(loop_str)
+            statistics_graph.update_current_node(loop_str)
             # identify all applicable suggestions for this loop
             logger.debug(str(hotspot_type) + " loop: " + str(loop_tuple))
             # create code and execute for all applicable suggestions
@@ -80,6 +103,16 @@ def run(arguments: AutotunerArguments) -> None:
                 tmp_config = reference_configuration.create_copy(get_unique_configuration_id)
                 tmp_config.apply_suggestions(arguments, current_config)
                 tmp_config.execute(timeout=timeout_after)
+                statistics_graph.add_child(
+                    "step "
+                    + str(statistics_step_num)
+                    + "\n"
+                    + str(current_config)
+                    + "\n"
+                    + tmp_config.get_statistics_graph_label(),
+                    shape=NodeShape.BOX,
+                    color=tmp_config.get_statistics_graph_color(),
+                )
                 # only consider valid code
                 if (
                     cast(ExecutionResult, tmp_config.execution_result).result_valid
@@ -92,6 +125,16 @@ def run(arguments: AutotunerArguments) -> None:
                     tmp_config.deleteFolder()
             # add current best configuration for reference / to detect "no suggestions is beneficial"
             suggestion_effects.append(best_suggestion_configuration)
+            statistics_graph.add_child(
+                "step "
+                + str(statistics_step_num)
+                + "\n"
+                + str(best_suggestion_configuration[0])
+                + "\n"
+                + best_suggestion_configuration[1].get_statistics_graph_label(),
+                shape=NodeShape.BOX,
+                color=best_suggestion_configuration[1].get_statistics_graph_color(),
+            )
 
             logger.debug(
                 "Suggestion effects:\n" + str([(str(t[0]), str(t[1].execution_result)) for t in suggestion_effects])
@@ -110,6 +153,26 @@ def run(arguments: AutotunerArguments) -> None:
                 + " stored at "
                 + best_suggestion_configuration[1].root_path
             )
+            statistics_graph.add_child(
+                "step "
+                + str(statistics_step_num)
+                + "\n"
+                + str(best_suggestion_configuration[0])
+                + "\n"
+                + best_suggestion_configuration[1].get_statistics_graph_label(),
+                shape=NodeShape.BOX,
+                color=best_suggestion_configuration[1].get_statistics_graph_color(),
+            )
+            statistics_graph.update_current_node(
+                "step "
+                + str(statistics_step_num)
+                + "\n"
+                + str(best_suggestion_configuration[0])
+                + "\n"
+                + best_suggestion_configuration[1].get_statistics_graph_label()
+            )
+            statistics_graph.output()
+            statistics_step_num += 1
             # cleanup other configurations (excluding original version)
             logger.debug("Cleanup:")
             for _, config in sorted_suggestion_effects:
@@ -143,3 +206,6 @@ def run(arguments: AutotunerArguments) -> None:
         print("Speedup: ", round(speedup, 3))
         print("Parallel efficiency: ", round(parallel_efficiency, 3))
         print("##############################")
+
+    # output statistics graph
+    statistics_graph.output()

discopop_library/EmpiricalAutotuning/Classes/CodeConfiguration.py

@@ -15,6 +15,7 @@
 from typing import Callable, List, Optional
 from discopop_library.EmpiricalAutotuning.ArgumentClasses import AutotunerArguments
 from discopop_library.EmpiricalAutotuning.Classes.ExecutionResult import ExecutionResult
+from discopop_library.EmpiricalAutotuning.Statistics.StatisticsGraph import NodeColor
 from discopop_library.EmpiricalAutotuning.Types import SUGGESTION_ID
 from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments
 from discopop_library.PatchApplicator.patch_applicator import run as apply_patches
@@ -40,9 +41,11 @@ def __init__(self, root_path: str, config_dot_dp_path: str):
     def __str__(self) -> str:
         return self.root_path
 
-    def execute(self, timeout: Optional[float]) -> None:
+    def execute(self, timeout: Optional[float], is_initial: bool = False) -> None:
         # create timeout string
         timeout_string = "" if timeout is None else "timeout " + str(timeout) + " "
+        if is_initial:
+            timeout_string += "source "
         # compile code
         logger.info("Compiling configuration: " + str(self))
         compile_result = subprocess.run(
@@ -134,3 +137,21 @@ def apply_suggestions(self, arguments: AutotunerArguments, suggestion_ids: List[
             sub_logger.debug("Got Exception during call to patch applicator.")
             os.chdir(save_dir)
             raise ex
+
+    def get_statistics_graph_label(self) -> str:
+        res_str = "" + self.root_path + "\n"
+        if self.execution_result is None:
+            res_str += "Not executed."
+        else:
+            res_str += str(round(self.execution_result.runtime, 3)) + "s"
+
+        return res_str
+
+    def get_statistics_graph_color(self) -> NodeColor:
+        if self.execution_result is None:
+            return NodeColor.ORANGE
+        if self.execution_result.result_valid and self.execution_result.return_code == 0:
+            return NodeColor.GREEN
+        if self.execution_result.return_code == 0 and not self.execution_result.result_valid:
+            return NodeColor.ORANGE
+        return NodeColor.RED

discopop_library/EmpiricalAutotuning/Statistics/StatisticsGraph.py

@@ -0,0 +1,79 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+from enum import Enum
+import logging
+import os
+import subprocess
+from typing import Any
+import networkx as nx  # type: ignore
+
+logger = logging.getLogger("StatisticsGraph")
+
+
+class NodeShape(Enum):
+    SQUARE = "square"
+    CIRCLE = "circle"
+    NONE = "none"
+    BOX = "box"
+
+
+class NodeColor(Enum):
+    RED = "red"
+    WHITE = "white"
+    GREEN = "green"
+    ORANGE = "orange"
+
+
+class StatisticsGraph(object):
+    G: nx.DiGraph
+    current_node: str = ""
+
+    def __init__(self) -> None:
+        self.G = nx.DiGraph()
+
+    def dump_to_dot(self) -> None:
+        filename = "dp_autotuner_statistics.dot"
+        if os.path.exists(filename):
+            os.remove(filename)
+        nx.drawing.nx_pydot.write_dot(self.G, filename)
+
+    def output(self) -> None:
+        self.dump_to_dot()
+        self.create_svg_from_dot()
+
+    def set_root(self, label: str, color: NodeColor = NodeColor.WHITE, shape: NodeShape = NodeShape.NONE) -> None:
+        if shape == NodeShape.NONE:
+            self.G.add_node(label, color="black", fillcolor=color.value, style="filled")
+        else:
+            self.G.add_node(label, color="black", fillcolor=color.value, style="filled", shape=shape.value)
+        self.current_node = label
+
+    def update_current_node(self, label: str) -> None:
+        self.current_node = label
+
+    def add_child(self, child: str, color: NodeColor = NodeColor.WHITE, shape: NodeShape = NodeShape.NONE) -> None:
+        if shape == NodeShape.NONE:
+            self.G.add_node(child, color="black", fillcolor=color.value, style="filled")
+        else:
+            self.G.add_node(child, color="black", fillcolor=color.value, shape=shape.value, style="filled")
+        self.G.add_edge(self.current_node, child)
+
+    def create_svg_from_dot(self) -> None:
+
+        cmd = "dot -Tsvg dp_autotuner_statistics.dot -o dp_autotuner_statistics.svg"
+        res = subprocess.run(
+            cmd,
+            cwd=os.getcwd(),
+            executable="/bin/bash",
+            shell=True,
+        )
+        if res.returncode != 0:
+            logger.warning("Failed: dot -Tsvg dp_autotuner_statistics.dot -o dp_autotuner_statistics.svg")
+        else:
+            logger.info("Updated dp_autotuner_statistics.svg")

setup.py

@@ -48,6 +48,7 @@
         "sympy_plot_backends",
         "alive_progress",
         "filelock",
+        "pydot",
     ],
     extras_require={
         "dev": ["mypy", "black", "data-science-types", "pre-commit"],