[#53551] scenarios: render_report: Create summarization report

Signed-off-by: Mateusz Leonowicz <mleonowicz@antmicro.com>

kenning/scenarios/render_report.py

@@ -40,6 +40,7 @@
     compute_detection_metrics,
     compute_performance_metrics,
     compute_renode_metrics,
+    compute_text_summarization_metrics,
 )
 from kenning.resources import reports
 from kenning.utils.class_loader import get_command
@@ -50,8 +51,15 @@
 PERFORMANCE = "performance"
 CLASSIFICATION = "classification"
 DETECTION = "detection"
+TEXT_SUMMARIZATION = "text_summarization"
 RENODE = "renode_stats"
-REPORT_TYPES = [PERFORMANCE, CLASSIFICATION, DETECTION, RENODE]
+REPORT_TYPES = [
+    PERFORMANCE,
+    CLASSIFICATION,
+    DETECTION,
+    RENODE,
+    TEXT_SUMMARIZATION,
+]
 
 
 def get_model_name(filepath: Path) -> str:
@@ -74,8 +82,8 @@ def get_model_name(filepath: Path) -> str:
 
 def get_plot_wildcard_path(plot_path: Path, root_dir: Path) -> str:
     """
-    Generate wildcard plot path relative to given directory wshich can be used
-    int report.
+    Generate wildcard plot path relative to given directory which can be used
+    in report.
 
     Parameters
     ----------
@@ -1699,6 +1707,190 @@ def retrieve_non_zero_profiler_data(
         )
 
 
+def text_summarization_report(
+    measurementsdata: Dict[str, Any],
+    imgdir: Path,
+    imgprefix: str,
+    root_dir: Path,
+    image_formats: Set[str],
+    colors: Optional[List] = None,
+    color_offset: int = 0,
+    draw_titles: bool = True,
+    **kwargs: Any,
+) -> str:
+    """
+    Creates text summarization section of the report.
+
+    Parameters
+    ----------
+    measurementsdata : Dict[str, Any]
+        Statistics from the Measurements class.
+    imgdir : Path
+        Path to the directory for images.
+    imgprefix : str
+        Prefix to the image file name.
+    root_dir : Path
+        Path to the root of the documentation project involving this report.
+    image_formats : Set[str]
+        Collection with formats which should be used to generate plots.
+    colors : Optional[List]
+        Colors to be used in the plots.
+    color_offset : int
+        How many colors from default color list should be skipped.
+    draw_titles : bool
+        Should titles be drawn on the plot.
+    **kwargs : Any
+        Additional keyword arguments.
+
+    Returns
+    -------
+    str
+        Content of the report in MyST format.
+    """
+    from kenning.core.drawing import Barplot
+    from kenning.core.report import create_report_from_measurements
+
+    KLogger.info(
+        "Running text_summarization_report for "
+        + measurementsdata["model_name"]
+    )
+    report_variables = {
+        "model_name": measurementsdata["model_name"],
+        "report_name": measurementsdata["report_name"],
+        "report_name_simple": measurementsdata["report_name_simple"],
+    }
+
+    metrics = compute_text_summarization_metrics(measurementsdata)
+    if not metrics:
+        KLogger.error(
+            "No metrics were computed for the text summarization task "
+            + f'for {measurementsdata["model_name"]}'
+        )
+        return ""
+
+    rouge_keys = [key for key in metrics.keys() if key.startswith("rouge")]
+    rouge_scores = [metrics[key] for key in rouge_keys]
+
+    if "predictions" in measurementsdata:
+        from random import sample
+
+        NUM_OF_EXAMPLES = 10
+
+        KLogger.info(
+            f"Including {NUM_OF_EXAMPLES} example predictions to "
+            "the text summarization report"
+        )
+        report_variables["example_predictions"] = sample(
+            measurementsdata["predictions"], NUM_OF_EXAMPLES
+        )
+
+    rouge_path = imgdir / f"{imgprefix}rouge"
+    Barplot(
+        title="Rouge scores" if draw_titles else None,
+        x_label="Rouge",
+        y_label="Score",
+        y_unit="%",
+        x_data=rouge_keys,
+        y_data={"scores": rouge_scores},
+        colors=colors,
+        color_offset=color_offset,
+    ).plot(rouge_path, image_formats)
+    report_variables["barplot_rouge_path"] = get_plot_wildcard_path(
+        rouge_path, root_dir
+    )
+
+    with path(reports, "text_summarization.md") as reporttemplate:
+        return create_report_from_measurements(
+            reporttemplate, report_variables
+        )
+
+
+def comparison_text_summarization_report(
+    measurementsdata: List[Dict],
+    imgdir: Path,
+    root_dir: Path,
+    image_formats: Set[str],
+    colors: Optional[List] = None,
+    draw_titles: bool = True,
+    **kwargs: Any,
+) -> str:
+    """
+    Creates text summarization comparison section of report.
+
+    Parameters
+    ----------
+    measurementsdata : List[Dict]
+        Statistics of every model from the Measurements class.
+    imgdir : Path
+        Path to the directory for images.
+    root_dir : Path
+        Path to the root of the documentation project involving this report.
+    image_formats : Set[str]
+        Collection with formats which should be used to generate plots.
+    colors : Optional[List]
+        Colors to be used in the plots.
+    draw_titles : bool
+        Should titles be drawn on the plot.
+    **kwargs : Any
+        Additional keyword arguments.
+
+    Returns
+    -------
+    str
+        Content of the report in MyST format.
+    """
+    from kenning.core.drawing import Barplot
+    from kenning.core.report import create_report_from_measurements
+
+    KLogger.info("Running comparison_text_summarization_report")
+
+    rouge_data = {}
+    report_variables = {
+        "report_name": measurementsdata[0]["report_name"],
+        "report_name_simple": measurementsdata[0]["report_name_simple"],
+    }
+
+    for data in measurementsdata:
+        metrics = compute_text_summarization_metrics(data)
+        if not metrics:
+            KLogger.warning(
+                "No metrics were computed for the text summarization task "
+                + f'for {data["model_name"]} in a comparison report'
+            )
+            continue
+
+        rouge_keys = [key for key in metrics.keys() if key.startswith("rouge")]
+        rouge_scores = [metrics[key] for key in rouge_keys]
+        rouge_data[data["model_name"]] = rouge_scores
+
+    rouge_path = imgdir / "rouge"
+
+    if len(rouge_data.keys()) < 2:
+        KLogger.error(
+            f"Not enough metrics ({len(rouge_data.keys())}) were computed "
+            + "for the comparison text summarization task"
+        )
+        return ""
+
+    Barplot(
+        title="Rouge scores" if draw_titles else None,
+        x_label="Rouge",
+        y_label="Score",
+        y_unit="%",
+        x_data=rouge_keys,
+        y_data=rouge_data,
+        colors=colors,
+    ).plot(rouge_path, image_formats)
+    report_variables["barplot_rouge_path_comparison"] = get_plot_wildcard_path(
+        rouge_path, root_dir
+    )
+
+    with path(reports, "text_summarization_comparison.md") as reporttemplate:
+        return create_report_from_measurements(
+            reporttemplate, report_variables
+        )
+
+
 def generate_report(
     report_name: str,
     data: List[Dict],
@@ -1756,12 +1948,14 @@ def generate_report(
         CLASSIFICATION: classification_report,
         DETECTION: detection_report,
         RENODE: renode_stats_report,
+        TEXT_SUMMARIZATION: text_summarization_report,
     }
     comparereptypes = {
         PERFORMANCE: comparison_performance_report,
         CLASSIFICATION: comparison_classification_report,
         DETECTION: comparison_detection_report,
         RENODE: comparison_renode_stats_report,
+        TEXT_SUMMARIZATION: comparison_text_summarization_report,
     }
 
     header_data = {
@@ -1852,6 +2046,11 @@ def _append_type_if(_type: str, func: Callable):
     _append_type_if(DETECTION, lambda data: "eval_gtcount" in data)
     _append_type_if(RENODE, lambda data: "opcode_counters" in data)
 
+    _append_type_if(
+        TEXT_SUMMARIZATION,
+        lambda data: any([key.startswith("rouge") for key in data.keys()]),
+    )
+
     if len(report_types) == 0:
         KLogger.error(
             "There is no report type which is suitable for all measurements"