Fixing metric names in plots (#163)

* Changed metric names from acronym to actual name * Metric name cleanup in flow * Reverted changes in aequitas plots * Update previous plots to support new naming scheme * Remove dictionary from bubble concatenation chart * Change reference to aequitas flow * Remove duplicated argument --------- Co-authored-by: Sérgio Jesus <sergio.jesus@feedzai.com>
dssg · Feb 12, 2024 · 8da3f6b · 8da3f6b
1 parent 59c1a02
commit 8da3f6b
Show file tree

Hide file tree

Showing 11 changed files with 239 additions and 161 deletions.
diff --git a/src/aequitas/flow/plots/bootstrap/plot.py b/src/aequitas/flow/plots/bootstrap/plot.py
@@ -1,8 +1,9 @@
-from typing import Literal, Optional
+from typing import Optional
 
 import numpy as np
 
 from ...utils.evaluation import bootstrap_hyperparameters
+from ...utils.metrics import METRIC_NAMES, FAIRNESS_METRIC, PERFORMANCE_METRIC
 from ...evaluation import Result
 
 
@@ -14,27 +15,13 @@
 }
 
 
-metrics = {
-    "Predictive Equality": "fpr_ratio",
-    "Equal Opportunity": "tpr_ratio",
-    "Demographic Parity": "pprev_ratio",
-    "TPR": "tpr",
-    "FPR": "fpr",
-    "FNR": "fnr",
-    "Accuracy": "accuracy",
-    "Precision": "precision",
-}
-
-
 class Plot:
     def __init__(
         self,
         results: dict[str, dict[str, Result]],
         dataset: str,
-        fairness_metric: Literal[
-            "Predictive Equality", "Equal Opportunity", "Demographic Parity"
-        ],
-        performance_metric: Literal["TPR", "FPR", "FNR", "Accuracy", "Precision"],
+        fairness_metric: FAIRNESS_METRIC,
+        performance_metric: PERFORMANCE_METRIC,
         method: Optional[str] = None,
         confidence_intervals: float = 0.95,
         **kwargs,
@@ -64,8 +51,8 @@ def __init__(
         for key, value in DEFAULT_KWARGS.items():
             if key not in self.kwargs:
                 self.kwargs[key] = value
-        self.kwargs["fairness_metric"] = metrics[fairness_metric]
-        self.kwargs["performance_metric"] = metrics[performance_metric]
+        self.kwargs["fairness_metric"] = METRIC_NAMES[fairness_metric]
+        self.kwargs["performance_metric"] = METRIC_NAMES[performance_metric]
         self.bootstrap_results = {}
         if isinstance(self.kwargs["alpha_points"], np.ndarray):
             self.x = self.kwargs["alpha_points"]

diff --git a/src/aequitas/flow/plots/bootstrap/visualize.py b/src/aequitas/flow/plots/bootstrap/visualize.py
@@ -4,6 +4,7 @@
 
 
 from .plot import Plot
+from ...utils.metrics import METRIC_NAMES
 
 sns.set()
 sns.set_style("whitegrid", {"grid.linestyle": "--"})
@@ -53,22 +54,11 @@
     "grid_search_folktables",
 ]
 
-metrics_names = {
-    "Predictive Equality": "Pred. Eq.",
-    "Equal Opportunity": "Eq. Opp.",
-    "Demographic Parity": "Dem. Par.",
-    "TPR": "TPR",
-    "FPR": "FPR",
-    "FNR": "FNR",
-    "Accuracy": "Acc.",
-    "Precision": "Prec.",
-}
-
 
 def visualize(plot: Plot):
     # define the name of the metrics for plot
-    perf_metric_plot = metrics_names[plot.performance_metric]
-    fair_metric_plot = metrics_names[plot.fairness_metric]
+    perf_metric_plot = METRIC_NAMES[plot.performance_metric]
+    fair_metric_plot = METRIC_NAMES[plot.fairness_metric]
 
     x = plot.x
 

diff --git a/src/aequitas/flow/plots/pareto/plot.py b/src/aequitas/flow/plots/pareto/plot.py
@@ -8,6 +8,7 @@
 from ....bias import Bias
 from ....group import Group
 from ....plot import summary, disparity
+from ...utils.metrics import FAIRNESS_METRIC, PERFORMANCE_METRIC
 
 
 _names = {
@@ -54,10 +55,12 @@ class Plot:
         Name of the dataset to be used in the Pareto plot.
     method : union[str, list], optional
         Name of the method to plot. If none, all methods will be plotted.
-    fairness_metric : {"Predictive Equality", "Equal Opportunity", "Demographic Parity"}
-        The default fairness metric to use in the Pareto plot.
-    performance_metric : {"TPR", "FPR", "FNR", "Accuracy", "Precision"}
-        The default performance metric to use in the Pareto plot.
+    fairness_metric : str
+        The default fairness metric to use in the Pareto plot. Possible values
+        are defined in aequitas.flow.utils.metrics
+    performance_metric : str
+        The default performance metric to use in the Pareto plot. Possible values
+        are defined in aequitas.flow.utils.metrics
     alpha : float, optional
         The alpha value to use in the Pareto plot.
     direction : {"minimize", "maximize"}, optional
@@ -68,10 +71,8 @@ def __init__(
         self,
         results: dict[str, dict[str, Result]],
         dataset: str,
-        fairness_metric: Literal[
-            "Predictive Equality", "Equal Opportunity", "Demographic Parity"
-        ],
-        performance_metric: Literal["TPR", "FPR", "FNR", "Accuracy", "Precision"],
+        fairness_metric: FAIRNESS_METRIC,
+        performance_metric: PERFORMANCE_METRIC,
         method: Optional[Union[str, list]] = None,
         alpha: float = 0.5,
         direction: Literal["minimize", "maximize"] = "maximize",
@@ -109,18 +110,6 @@ def __init__(
         self.performance_metric = performance_metric
         self.alpha = alpha
         self.direction = direction
-        self.available_fairness_metrics = {
-            "Predictive Equality",
-            "Equal Opportunity",
-            "Demographic Parity",
-        }  # Hardcoded for now
-        self.available_performance_metrics = [
-            "TPR",
-            "FPR",
-            "FNR",
-            "Accuracy",
-            "Precision",
-        ]
         self._best_model_idx: int = 0
 
     @property
@@ -302,7 +291,7 @@ def disparities(
         model_id: int,
         dataset: Any,
         sensitive_attribute: Union[str, list[str]],
-        metrics: list[str] = ["tpr", "fpr"],
+        metrics: list[str] = ["TPR", "FPR"],
         fairness_threshold: float = 1.2,
         results_path: Union[Path, str] = "examples/experiment_results",
         reference_groups: Optional[dict[str, str]] = None,

diff --git a/src/aequitas/flow/plots/pareto/visualize.py b/src/aequitas/flow/plots/pareto/visualize.py
@@ -8,6 +8,7 @@
 import pkg_resources
 
 from .plot import Plot
+from ...utils.metrics import FAIRNESS_METRICS, PERFORMANCE_METRICS
 
 
 # NumPy data types are not JSON serializable. This custom JSON encoder will
@@ -90,12 +91,9 @@ def visualize(wrapper: Plot, mode="display", save_path=None, pareto_only=False):
     if pareto_only:
         wrapper_results_flat = wrapper_results_flat[wrapper_results_flat["is_pareto"]]
 
-    fairness_metrics = list(wrapper.available_fairness_metrics)
-    performance_metrics = list(wrapper.available_performance_metrics)
-
     filtered_results = wrapper_results_flat[
-        fairness_metrics
-        + performance_metrics
+        FAIRNESS_METRICS
+        + PERFORMANCE_METRICS
         + ["model_id", "hyperparams", "is_pareto"]
     ]
 
@@ -108,8 +106,8 @@ def visualize(wrapper: Plot, mode="display", save_path=None, pareto_only=False):
         "recommended_model": wrapper.best_model_details,
         "optimized_fairness_metric": wrapper.fairness_metric,
         "optimized_performance_metric": wrapper.performance_metric,
-        "fairness_metrics": fairness_metrics,
-        "performance_metrics": performance_metrics,
+        "fairness_metrics": FAIRNESS_METRICS,
+        "performance_metrics": PERFORMANCE_METRICS,
         "tuner_type": "Random Search",  # Hardcoded for now
         "alpha": wrapper.alpha,
     }

diff --git a/src/aequitas/flow/utils/colab.py b/src/aequitas/flow/utils/colab.py
@@ -14,19 +14,19 @@ def get_examples(
         "methods/data_repair",
     ]
 ) -> None:
-    """Downloads the examples from the fairflow repository.
+    """Downloads the examples from the aequitas flow repository.
 
     Note that this should not be used outside Google Colab, as it clutters the directory
     with with the git files from Aequitas repository.
 
     Parameters
     ----------
     directory : Literal["configs", "examples/data_repair", "experiment_results"]
-        The directory to download from the fairflow repository.
+        The directory to download from the aequitas flow repository.
     """
     directory = "examples/" + directory
     logger = create_logger("utils.colab")
-    logger.info("Downloading examples from fairflow repository.")
+    logger.info("Downloading examples from aequitas flow repository.")
     # Create directory if it doesn't exist
     Path(directory).mkdir(parents=True, exist_ok=True)
     # Check if git repository already exists in folder

diff --git a/src/aequitas/flow/utils/metrics.py b/src/aequitas/flow/utils/metrics.py
@@ -0,0 +1,20 @@
+from typing import Literal
+
+FAIRNESS_METRICS = ["Predictive Equality", "Equal Opportunity", "Demographic Parity"]
+PERFORMANCE_METRICS = ["TPR", "FPR", "FNR", "Accuracy", "Precision"]
+
+METRIC_NAMES = {
+    "Predictive Equality": "fpr_ratio",
+    "Equal Opportunity": "tpr_ratio",
+    "Demographic Parity": "pprev_ratio",
+    "TPR": "tpr",
+    "FPR": "fpr",
+    "FNR": "fnr",
+    "Accuracy": "accuracy",
+    "Precision": "precision",
+}
+
+FAIRNESS_METRIC = Literal[
+    "Predictive Equality", "Equal Opportunity", "Demographic Parity"
+]
+PERFORMANCE_METRIC = Literal["TPR", "FPR", "FNR", "Accuracy", "Precision"]
diff --git a/src/aequitas/plot/bubble_concatenation_chart.py b/src/aequitas/plot/bubble_concatenation_chart.py
@@ -12,14 +12,17 @@
 from aequitas.plot.commons.style.sizes import Concat_Chart
 from aequitas.plot.commons import initializers as Initializer
 
-# Altair 2.4.1 requires that all chart receive a dataframe, for charts that don't need it
-# (like most annotations), we pass the following dummy dataframe to reduce the complexity of the resulting vega spec.
+# Altair 2.4.1 requires that all chart receive a dataframe, for charts that don't need
+# it (like most annotations), we pass the following dummy dataframe to reduce the
+# complexity of the resulting vega spec.
 DUMMY_DF = pd.DataFrame({"a": [1, 1], "b": [0, 0]})
 
 
 def __get_chart_sizes(chart_width):
-    """Calculates the widths of the disparity and metric charts that make-up the concatenated chart.
-    The individual widths are calculated based on the provided desired overall chart width."""
+    """Calculates the widths of the disparity and metric charts that make-up the
+    concatenated chart. The individual widths are calculated based on the provided
+    desired overall chart width.
+    """
 
     chart_sizes = dict(
         disparity_chart_width=0.5 * chart_width, metric_chart_width=0.5 * chart_width
@@ -59,7 +62,8 @@ def plot_concatenated_bubble_charts(
     chart_width=Concat_Chart.full_width,
     accessibility_mode=False,
 ):
-    """Draws a concatenation of the disparity bubble chart and the metric values bubble chart,
+    """Draws a concatenation of the disparity bubble chart and the metric values bubble
+    chart,
     of the selected metrics for a given attribute.
 
     :param disparity_df: a dataframe generated by the Aequitas Bias class
@@ -68,19 +72,20 @@ def plot_concatenated_bubble_charts(
     :type metrics_list: list
     :param attribute: an attribute to plot
     :type attribute: str
-    :param fairness_threshold: a value for the maximum allowed disparity, defaults to 1.25
+    :param fairness_threshold: a value for the maximum allowed disparity, defaults to
+    1.25
     :type fairness_threshold: float, optional
     :param chart_height: a value (in pixels) for the height of the chart
     :type chart_height: int, optional
     :param chart_width: a value (in pixels) for the width of the chart
     :type chart_width: int, optional
-    :param accessibility_mode: a switch for the display of more accessible visual elements, defaults to False
+    :param accessibility_mode: a switch for the display of more accessible visual
+    elements, defaults to False
     :type accessibility_mode: bool, optional
 
     :return: the full disparities chart
     :rtype: Altair chart object
     """
-
     (
         plot_table,
         metrics,
@@ -170,10 +175,11 @@ def plot_concatenated_bubble_charts(
             offset=Chart_Title.offset,
         )
         .properties(
-            title=attribute.title(), 
+            title=attribute.title(),
             padding={
                 "top": Concat_Chart.full_chart_padding,
-                "bottom": -FONT_SIZE_SMALL * 0.75/3 * len(metrics_list) + Concat_Chart.full_chart_padding,
+                "bottom": -FONT_SIZE_SMALL * 0.75 / 3 * len(metrics_list)
+                + Concat_Chart.full_chart_padding,
                 "left": Concat_Chart.full_chart_padding,
                 "right": Concat_Chart.full_chart_padding,
             },