Anomalib CLI Improvements - Update metrics and create post_processing section in the config file (#607)

* Rename image/pixel_metrics_names to image/pixel_metrics

* Rename image/pixel_metrics_names to image/pixel_metrics

* Modified config files.

* Modify get_callbacks function for the old cli

* Create pre-processing-configuration callback

* Add new CLI configuration for the post-processing configuration

* Add options to normalization_method

* Address mypy issues

* Fix docstring

* Address codacy issues

Author: samet-akcay

anomalib/utils/callbacks/__init__.py

@@ -19,19 +19,22 @@
 from .metrics_configuration import MetricsConfigurationCallback
 from .min_max_normalization import MinMaxNormalizationCallback
 from .model_loader import LoadModelCallback
+from .post_processing_configuration import PostProcessingConfigurationCallback
 from .tiler_configuration import TilerConfigurationCallback
 from .timer import TimerCallback
 from .visualizer import ImageVisualizerCallback, MetricVisualizerCallback
 
 __all__ = [
     "CdfNormalizationCallback",
+    "GraphLogger",
+    "ImageVisualizerCallback",
     "LoadModelCallback",
     "MetricsConfigurationCallback",
+    "MetricVisualizerCallback",
     "MinMaxNormalizationCallback",
+    "PostProcessingConfigurationCallback",
     "TilerConfigurationCallback",
     "TimerCallback",
-    "ImageVisualizerCallback",
-    "MetricVisualizerCallback",
 ]
 
 
@@ -64,20 +67,25 @@ def get_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]:
 
     callbacks.extend([checkpoint, TimerCallback()])
 
-    # Add metric configuration to the model via MetricsConfigurationCallback
-    image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None
-    pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None
+    # Add post-processing configurations to AnomalyModule.
     image_threshold = (
         config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None
     )
     pixel_threshold = (
         config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None
     )
+    post_processing_callback = PostProcessingConfigurationCallback(
+        adaptive_threshold=config.metrics.threshold.adaptive,
+        default_image_threshold=image_threshold,
+        default_pixel_threshold=pixel_threshold,
+    )
+    callbacks.append(post_processing_callback)
+
+    # Add metric configuration to the model via MetricsConfigurationCallback
+    image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None
+    pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None
     metrics_callback = MetricsConfigurationCallback(
-        config.metrics.threshold.adaptive,
         config.dataset.task,
-        image_threshold,
-        pixel_threshold,
         image_metric_names,
         pixel_metric_names,
     )
@@ -172,7 +180,8 @@ def add_visualizer_callback(callbacks: List[Callback], config: Union[DictConfig,
         config.visualization.inputs_are_normalized = not config.model.normalization_method == "none"
     else:
         config.visualization.task = config.data.init_args.task
-        config.visualization.inputs_are_normalized = not config.metrics.normalization_method == "none"
+        config.visualization.inputs_are_normalized = not config.post_processing.normalization_method == "none"
+
     if config.visualization.log_images or config.visualization.save_images or config.visualization.show_images:
         image_save_path = (
             config.visualization.image_save_path

anomalib/utils/callbacks/metrics_configuration.py

@@ -8,7 +8,6 @@
 from typing import List, Optional
 
 import pytorch_lightning as pl
-import torch
 from pytorch_lightning.callbacks import Callback
 from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY
 
@@ -26,13 +25,9 @@ class MetricsConfigurationCallback(Callback):
 
     def __init__(
         self,
-        adaptive_threshold: bool,
         task: str = "segmentation",
-        default_image_threshold: Optional[float] = None,
-        default_pixel_threshold: Optional[float] = None,
-        image_metric_names: Optional[List[str]] = None,
-        pixel_metric_names: Optional[List[str]] = None,
-        normalization_method: str = "min_max",
+        image_metrics: Optional[List[str]] = None,
+        pixel_metrics: Optional[List[str]] = None,
     ):
         """Create image and pixel-level AnomalibMetricsCollection.
 
@@ -43,30 +38,12 @@ def __init__(
 
         Args:
             task (str): Task type of the current run.
-            adaptive_threshold (bool): Flag indicating whether threshold should be adaptive.
-            default_image_threshold (Optional[float]): Default image threshold value.
-            default_pixel_threshold (Optional[float]): Default pixel threshold value.
-            image_metric_names (Optional[List[str]]): List of image-level metrics.
-            pixel_metric_names (Optional[List[str]]): List of pixel-level metrics.
-            normalization_method(Optional[str]): Normalization method. <None, min_max, cdf>
+            image_metrics (Optional[List[str]]): List of image-level metrics.
+            pixel_metrics (Optional[List[str]]): List of pixel-level metrics.
         """
-        # TODO: https://github.com/openvinotoolkit/anomalib/issues/384
         self.task = task
-        self.image_metric_names = image_metric_names
-        self.pixel_metric_names = pixel_metric_names
-
-        # TODO: https://github.com/openvinotoolkit/anomalib/issues/384
-        # TODO: This is a workaround. normalization-method is actually not used in metrics.
-        #   It's only accessed from `before_instantiate` method in `AnomalibCLI` to configure
-        #   its callback.
-        self.normalization_method = normalization_method
-
-        assert (
-            adaptive_threshold or default_image_threshold is not None and default_pixel_threshold is not None
-        ), "Default thresholds must be specified when adaptive threshold is disabled."
-        self.adaptive_threshold = adaptive_threshold
-        self.default_image_threshold = default_image_threshold
-        self.default_pixel_threshold = default_pixel_threshold
+        self.image_metric_names = image_metrics
+        self.pixel_metric_names = pixel_metrics
 
     def setup(
         self,
@@ -97,12 +74,6 @@ def setup(
             pixel_metric_names = self.pixel_metric_names
 
         if isinstance(pl_module, AnomalyModule):
-            pl_module.adaptive_threshold = self.adaptive_threshold
-            if not self.adaptive_threshold:
-                # pylint: disable=not-callable
-                pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu()
-                pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu()
-
             pl_module.image_metrics = metric_collection_from_names(image_metric_names, "image_")
             pl_module.pixel_metrics = metric_collection_from_names(pixel_metric_names, "pixel_")
 

anomalib/utils/callbacks/post_processing_configuration.py

@@ -0,0 +1,63 @@
+"""Post-Processing Configuration Callback."""
+
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+from typing import Optional
+
+import torch
+from pytorch_lightning import Callback, LightningModule, Trainer
+from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY
+
+from anomalib.models.components.base.anomaly_module import AnomalyModule
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["PostProcessingConfigurationCallback"]
+
+
+@CALLBACK_REGISTRY
+class PostProcessingConfigurationCallback(Callback):
+    """Post-Processing Configuration Callback.
+
+    Args:
+        normalization_method(Optional[str]): Normalization method. <None, min_max, cdf>
+        adaptive_threshold (bool): Flag indicating whether threshold should be adaptive.
+        default_image_threshold (Optional[float]): Default image threshold value.
+        default_pixel_threshold (Optional[float]): Default pixel threshold value.
+    """
+
+    def __init__(
+        self,
+        normalization_method: str = "min_max",
+        adaptive_threshold: bool = True,
+        default_image_threshold: Optional[float] = None,
+        default_pixel_threshold: Optional[float] = None,
+    ) -> None:
+        super().__init__()
+        self.normalization_method = normalization_method
+
+        assert (
+            adaptive_threshold or default_image_threshold is not None and default_pixel_threshold is not None
+        ), "Default thresholds must be specified when adaptive threshold is disabled."
+
+        self.adaptive_threshold = adaptive_threshold
+        self.default_image_threshold = default_image_threshold
+        self.default_pixel_threshold = default_pixel_threshold
+
+    # pylint: disable=unused-argument
+    def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[str] = None) -> None:
+        """Setup post-processing configuration within Anomalib Model.
+
+        Args:
+            trainer (Trainer): PyTorch Lightning Trainer
+            pl_module (LightningModule): Anomalib Model that inherits pl LightningModule.
+            stage (Optional[str], optional): fit, validate, test or predict. Defaults to None.
+        """
+        if isinstance(pl_module, AnomalyModule):
+            pl_module.adaptive_threshold = self.adaptive_threshold
+            if pl_module.adaptive_threshold is False:
+                pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu()
+                pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu()

anomalib/utils/cli/cli.py

@@ -26,6 +26,7 @@
     MetricsConfigurationCallback,
     MinMaxNormalizationCallback,
     ModelCheckpoint,
+    PostProcessingConfigurationCallback,
     TilerConfigurationCallback,
     TimerCallback,
     add_visualizer_callback,
@@ -90,7 +91,6 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None:
         Args:
             parser (LightningArgumentParser): Lightning Argument Parser.
         """
-        # TODO: https://github.com/openvinotoolkit/anomalib/issues/19
         # TODO: https://github.com/openvinotoolkit/anomalib/issues/20
         parser.add_argument(
             "--export_mode", type=str, default="", help="Select export mode to ONNX or OpenVINO IR format."
@@ -105,18 +105,24 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None:
         parser.add_lightning_class_args(TilerConfigurationCallback, "tiling")  # type: ignore
         parser.set_defaults({"tiling.enable": False})
 
+        parser.add_lightning_class_args(PostProcessingConfigurationCallback, "post_processing")  # type: ignore
+        parser.set_defaults(
+            {
+                "post_processing.normalization_method": "min_max",
+                "post_processing.adaptive_threshold": True,
+                "post_processing.default_image_threshold": None,
+                "post_processing.default_pixel_threshold": None,
+            }
+        )
+
         # TODO: Assign these default values within the MetricsConfigurationCallback
         #   - https://github.com/openvinotoolkit/anomalib/issues/384
         parser.add_lightning_class_args(MetricsConfigurationCallback, "metrics")  # type: ignore
         parser.set_defaults(
             {
-                "metrics.adaptive_threshold": True,
                 "metrics.task": "segmentation",
-                "metrics.default_image_threshold": None,
-                "metrics.default_pixel_threshold": None,
-                "metrics.image_metric_names": ["F1Score", "AUROC"],
-                "metrics.pixel_metric_names": ["F1Score", "AUROC"],
-                "metrics.normalization_method": "min_max",
+                "metrics.image_metrics": ["F1Score", "AUROC"],
+                "metrics.pixel_metrics": ["F1Score", "AUROC"],
             }
         )
 
@@ -203,7 +209,7 @@ def __set_callbacks(self) -> None:
         #  TODO: This could be set in PostProcessingConfiguration callback
         #   - https://github.com/openvinotoolkit/anomalib/issues/384
         # Normalization.
-        normalization = config.metrics.normalization_method
+        normalization = config.post_processing.normalization_method
         if normalization:
             if normalization == "min_max":
                 callbacks.append(MinMaxNormalizationCallback())

anomalib/utils/sweep/helpers/callbacks.py

@@ -9,7 +9,10 @@
 from omegaconf import DictConfig, ListConfig
 from pytorch_lightning import Callback
 
-from anomalib.utils.callbacks import MetricsConfigurationCallback
+from anomalib.utils.callbacks import (
+    MetricsConfigurationCallback,
+    PostProcessingConfigurationCallback,
+)
 from anomalib.utils.callbacks.timer import TimerCallback
 
 
@@ -24,23 +27,40 @@ def get_sweep_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]
     """
     callbacks: List[Callback] = [TimerCallback()]
     # Add metric configuration to the model via MetricsConfigurationCallback
-    image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None
-    pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None
-    image_threshold = (
-        config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None
-    )
-    pixel_threshold = (
-        config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None
-    )
-    metrics_callback = MetricsConfigurationCallback(
-        adaptive_threshold=config.metrics.threshold.adaptive,
-        task=config.dataset.task,
+
+    # TODO: Remove this once the old CLI is deprecated.
+    if isinstance(config, DictConfig):
+        image_metrics = config.metrics.image if "image" in config.metrics.keys() else None
+        pixel_metrics = config.metrics.pixel if "pixel" in config.metrics.keys() else None
+        image_threshold = (
+            config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None
+        )
+        pixel_threshold = (
+            config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None
+        )
+        normalization_method = config.model.normalization_method
+    # NOTE: This is for the new anomalib CLI.
+    else:
+        image_metrics = config.metrics.image_metrics if "image_metrics" in config.metrics else None
+        pixel_metrics = config.metrics.pixel_metrics if "pixel_metrics" in config.metrics else None
+        image_threshold = (
+            config.post_processing.default_image_threshold if "image_default" in config.post_processing.keys() else None
+        )
+        pixel_threshold = (
+            config.post_processing.default_pixel_threshold if "pixel_default" in config.post_processing.keys() else None
+        )
+        normalization_method = config.post_processing.normalization_method
+
+    post_processing_configuration_callback = PostProcessingConfigurationCallback(
+        normalization_method=normalization_method,
         default_image_threshold=image_threshold,
         default_pixel_threshold=pixel_threshold,
-        image_metric_names=image_metric_names,
-        pixel_metric_names=pixel_metric_names,
-        normalization_method=config.model.normalization_method,
     )
-    callbacks.append(metrics_callback)
+    callbacks.append(post_processing_configuration_callback)
+
+    metrics_configuration_callback = MetricsConfigurationCallback(
+        task=config.dataset.task, image_metrics=image_metrics, pixel_metrics=pixel_metrics
+    )
+    callbacks.append(metrics_configuration_callback)
 
     return callbacks

configs/model/cflow.yaml

@@ -37,21 +37,23 @@ model:
     permute_soft: false
     lr: 0.0001
 
-metrics:
+post_processing:
+  normalization_method: min_max # <null, min_max, cdf>
   adaptive_threshold: true
   default_image_threshold: null
   default_pixel_threshold: null
-  image_metric_names:
+
+metrics:
+  image_metrics:
     - F1Score
     - AUROC
-  pixel_metric_names:
+  pixel_metrics:
     - F1Score
     - AUROC
-  normalization_method: min_max
 
 visualization:
   show_images: False # show images on the screen
-  save_images: False # save images to the file system
+  save_images: True # save images to the file system
   log_images: False # log images to the available loggers (if any)
   mode: full # options: ["full", "simple"]
 

configs/model/dfkde.yaml

@@ -32,16 +32,20 @@ model:
     threshold_steepness: 0.05
     threshold_offset: 12
 
-metrics:
+post_processing:
+  normalization_method: min_max # <null, min_max, cdf>
   adaptive_threshold: true
   default_image_threshold: null
-  image_metric_names:
+  default_pixel_threshold: null
+
+metrics:
+  image_metrics:
     - F1Score
     - AUROC
 
 visualization:
   show_images: False # show images on the screen
-  save_images: False # save images to the file system
+  save_images: True # save images to the file system
   log_images: False # log images to the available loggers (if any)
   mode: full # options: ["full", "simple"]