Add some unit tests for the ModelCardReport methods (#529)

Author: amrit110

.pre-commit-config.yaml

@@ -56,7 +56,7 @@ repos:
     - id: doctest
       name: doctest
       entry: python3 -m doctest -o NORMALIZE_WHITESPACE
-      files: "^cyclops/evaluate/"
+      files: "^cyclops/"
       language: system
 
   - repo: local

cyclops/data/slicer.py

@@ -110,7 +110,7 @@ class SliceSpec:
     ...         {
     ...             "feature_1": {"value": "value_1"},
     ...             "feature_2": {
-    ...                 "min_value": "2020-01-01", keep_nulls: False,
+    ...                 "min_value": "2020-01-01", "keep_nulls": False,
     ...             },
     ...             "feature_3": {"year": ["2000", "2010", "2020"]},
     ...         },
@@ -119,8 +119,22 @@ class SliceSpec:
     >>> for slice_name, slice_func in slice_spec.slices():
     ...     print(slice_name)
     ...     # do something with slice_func here (e.g. dataset.filter(slice_func))
-
-    """
+    feature_1:non_null
+    feature_2:non_null&feature_3:non_null
+    feature_1:value_1
+    feature_1:value_1, value_2
+    !(feature_1:value_1)
+    feature_1:[2020-01-01 - 2020-12-31]
+    feature_1:(5 - 60)
+    feature_1:year=[2020, 2021, 2022]
+    feature_1:month=[6, 7, 8]
+    feature_1:month=6, day=1
+    feature_1:contains value_1
+    feature_1:contains ['value_1', 'value_2']
+    feature_1:value_1&feature_2:[2020-01-01 - inf]&feature_3:year=['2000', '2010', '2020']
+    overall
+
+    """  # noqa: W505
 
     spec_list: List[Dict[str, Dict[str, Any]]] = field(
         default_factory=lambda: [{}],

cyclops/monitor/clinical_applicator.py

@@ -17,16 +17,16 @@ class ClinicalShiftApplicator:
     The source and target datasets are then generated by splitting
     the original dataset along the categorical feature.
 
-    Examples
-    --------
-    >>> from cyclops.monitor.clinical_applicator import ClinicalShiftApplicator
-    >>> from cyclops.data.utils import load_nih
-    >>> ds = load_nih(path="/mnt/data/nihcxr")
-    >>> applicator = ClinicalShiftApplicator("hospital_type",
-                    source = ["hospital_type_1", "hospital_type_2"]
-                    target = ["hospital_type_3", "hospital_type_4", "hospital_type_5"]
-                    )
-    >>> ds_source, ds_target = applicator.apply_shift(ds)
+    # Examples
+    # --------
+    # >>> from cyclops.monitor.clinical_applicator import ClinicalShiftApplicator
+    # >>> from cyclops.data.loader import load_nihcxr
+    # >>> ds = load_nihcxr(path="/mnt/data/nihcxr")
+    # >>> applicator = ClinicalShiftApplicator("hospital_type",
+    #                 source = ["hospital_type_1", "hospital_type_2"]
+    #                 target = ["hospital_type_3", "hospital_type_4", "hospital_type_5"]
+    #                 )
+    # >>> ds_source, ds_target = applicator.apply_shift(ds)
 
 
     Parameters

cyclops/monitor/reductor.py

@@ -43,7 +43,7 @@ class Reductor:
     Examples
     --------
     >>> # (Data is loaded from memory)
-    >>> from drift_detection.reductor import Reductor
+    >>> from cyclops.monitor.reductor import Reductor
     >>> from sklearn.datasets import load_diabetes
     >>> X, y = load_diabetes(return_X_y=True)
     >>> reductor = Reductor("pca")

cyclops/monitor/synthetic_applicator.py

@@ -14,14 +14,14 @@
 class SyntheticShiftApplicator:
     """The SyntheticShiftApplicator class is used induce synthetic dataset shift.
 
-    Examples
-    --------
-    >>> from drift_detection.experimenter import Experimenter
-    >>> from sklearn.datasets import load_diabetes
-    >>> X, y = load_diabetes(return_X_y=True)
-    >>> X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.5, random_state=42)
-    >>> applicator = SyntheticShiftApplicator(shift_type="gn_shift")
-    >>> X_shift = applicator.apply_shift(X_train, noise_amt=0.1, delta=0.1)
+    # Examples
+    # --------
+    # >>> from sklearn.datasets import load_diabetes
+    # >>> X, y = load_diabetes(return_X_y=True)
+    # >>> dataset = Dataset.from_dict({"X": X, "y": y})
+    # >>> dataset = dataset.train_test_split(test_size=0.5, seed=42)
+    # >>> applicator = SyntheticShiftApplicator(shift_type="gn_shift")
+    # >>> X_shift = applicator.apply_shift(dataset["test"])
 
     Parameters
     ----------

cyclops/monitor/tester.py

@@ -147,7 +147,7 @@ class TSTester:
     >>> tester.fit(X_s)
     >>> p_val, dist = tester.test_shift(X_t)
     >>> print(p_val, dist)
-    0.0 0.3
+    1.3805797e-12 0.51
     """
 
     def __init__(
@@ -348,26 +348,27 @@ class DCTester:
     rate and entropy are shown to be powerful discriminative statistics
     for harmful covariate shift (HCS).
 
-    Examples
-    --------
-    >>> from cyclops.monitor.tester import DCTester
-
-    >>> nih_ds = load_nihcxr(DATA_DIR)
-    >>> base_model = DenseNet(weights="densenet121-res224-nih")
-    >>> detectron = DCTester("detectron", model=base_model)
-    >>> detectron = DCTester("detectron",
-                        base_model=base_model,
-                        model=base_model,
-                        feature_columns="image",
-                        transforms=transforms,
-                        task="multilabel",
-                        max_epochs_per_model=5,
-                        ensemble_size=5,
-                        lr=0.01,
-                        num_runs=5)
-
-    >>> detectron.fit(source_ds)
-    >>> p_val, distance = detectron.predict(target_ds)
+    # Examples
+    # --------
+    # >>> from cyclops.monitor.tester import DCTester
+    # >>> from cyclops.data.loader import load_nihcxr
+    # >>> from cyclops.models.catalog import DenseNet
+
+    # >>> nih_ds = load_nihcxr(DATA_DIR)
+    # >>> base_model = DenseNet(weights="densenet121-res224-nih")
+    # >>> detectron = DCTester("detectron",
+    #                          base_model=base_model,
+    #                          model=base_model,
+    #                          feature_columns="image",
+    #                          transforms=None,
+    #                          task="multilabel",
+    #                          max_epochs_per_model=5,
+    #                          ensemble_size=5,
+    #                          lr=0.01,
+    #                          num_runs=5
+    #                 )
+    # >>> detectron.fit(source_ds)
+    # >>> p_val, distance = detectron.predict(target_ds)
 
     Parameters
     ----------

cyclops/report/report.py

@@ -202,7 +202,7 @@ def log_descriptor(
 
         Examples
         --------
-        >>> from cylops.report import ModelCardReport
+        >>> from cyclops.report import ModelCardReport
         >>> report = ModelCardReport()
         >>> report.log_descriptor(
         ...     name="tradeoff",

cyclops/report/utils.py

@@ -43,7 +43,7 @@ def str_to_snake_case(string: str) -> str:
     >>> str_to_snake_case("Hello-World")
     'hello_world'
     >>> str_to_snake_case("Hello_World")
-    'hello_world'
+    'hello__world'
     >>> str_to_snake_case("Hello World")
     'hello_world'
     >>> str_to_snake_case("hello_world")

tests/cyclops/report/test_report.py

@@ -0,0 +1,168 @@
+"""Test cyclops report module model report."""
+
+from unittest import TestCase
+
+from cyclops.report import ModelCardReport
+
+
+class TestModelCardReport(TestCase):
+    """Test ModelCardReport."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.model_card_report = ModelCardReport("reports")
+
+    def test_instantiation_with_optional_output_dir(self):
+        """Test instantiation with optional output_dir."""
+        assert self.model_card_report.output_dir == "reports"
+
+    def test_log_owner_with_name(self):
+        """Test log_owner with name."""
+        self.model_card_report.log_owner(name="John Doe")
+        assert (
+            self.model_card_report._model_card.model_details.owners[0].name
+            == "John Doe"
+        )
+
+    def test_log_owner_with_name_and_contact(self):
+        """Test log_owner with name and contact."""
+        self.model_card_report.log_owner(
+            name="John Doe",
+            contact="john.doe@example.com",
+        )
+        assert (
+            self.model_card_report._model_card.model_details.owners[0].name
+            == "John Doe"
+        )
+        assert (
+            self.model_card_report._model_card.model_details.owners[0].contact
+            == "john.doe@example.com"
+        )
+
+    def test_log_owner_with_name_and_role(self):
+        """Test log_owner with name and role."""
+        self.model_card_report.log_owner(name="John Doe", role="Developer")
+        assert (
+            self.model_card_report._model_card.model_details.owners[0].name
+            == "John Doe"
+        )
+        assert (
+            self.model_card_report._model_card.model_details.owners[0].role
+            == "Developer"
+        )
+
+    def test_valid_name_and_description(self):
+        """Test valid name and description."""
+        self.model_card_report.log_descriptor(
+            name="ethical_considerations",
+            description="This model was trained on data collected from a potentially biased source.",
+            section_name="considerations",
+        )
+
+        section = self.model_card_report._model_card.get_section("considerations")
+        descriptor = section.ethical_considerations
+
+        assert (
+            descriptor[0].description
+            == "This model was trained on data collected from a potentially biased source."
+        )
+
+    def test_log_user_with_description_to_considerations_section(self):
+        """Test log_user with description to considerations section."""
+        self.model_card_report.log_user(description="This is a user description")
+        assert len(self.model_card_report._model_card.considerations.users) == 1
+        assert (
+            self.model_card_report._model_card.considerations.users[0].description
+            == "This is a user description"
+        )
+
+    def test_log_performance_metric(self):
+        """Test log_performance_metric."""
+        self.model_card_report.log_quantitative_analysis(
+            analysis_type="performance",
+            name="accuracy",
+            value=0.85,
+            metric_slice="test",
+            decision_threshold=0.8,
+            description="Accuracy of the model on the test set",
+            pass_fail_thresholds=[0.9, 0.85, 0.8],
+            pass_fail_threshold_fns=[lambda x, t: x >= t for _ in range(3)],
+        )
+        assert (
+            self.model_card_report._model_card.quantitative_analysis.performance_metrics[
+                0
+            ].type
+            == "accuracy"
+        )
+        assert (
+            self.model_card_report._model_card.quantitative_analysis.performance_metrics[
+                0
+            ].value
+            == 0.85
+        )
+        assert (
+            self.model_card_report._model_card.quantitative_analysis.performance_metrics[
+                0
+            ].slice
+            == "test"
+        )
+        assert (
+            self.model_card_report._model_card.quantitative_analysis.performance_metrics[
+                0
+            ].decision_threshold
+            == 0.8
+        )
+        assert (
+            self.model_card_report._model_card.quantitative_analysis.performance_metrics[
+                0
+            ].description
+            == "Accuracy of the model on the test set"
+        )
+        assert (
+            len(
+                self.model_card_report._model_card.quantitative_analysis.performance_metrics[
+                    0
+                ].tests,
+            )
+            == 3
+        )
+
+    def test_log_quantitative_analysis_performance(self):
+        """Test log_quantitative_analysis (performance)."""
+        self.model_card_report.log_quantitative_analysis(
+            analysis_type="performance",
+            name="accuracy",
+            value=0.85,
+        )
+        assert (
+            self.model_card_report._model_card.quantitative_analysis.performance_metrics[
+                0
+            ].type
+            == "accuracy"
+        )
+        assert (
+            self.model_card_report._model_card.quantitative_analysis.performance_metrics[
+                0
+            ].value
+            == 0.85
+        )
+
+    def test_log_quantitative_analysis_fairness(self):
+        """Test log_quantitative_analysis (fairness)."""
+        self.model_card_report.log_quantitative_analysis(
+            analysis_type="fairness",
+            name="disparate_impact",
+            value=0.9,
+        )
+        assert (
+            self.model_card_report._model_card.fairness_analysis.fairness_reports[
+                0
+            ].type
+            == "disparate_impact"
+        )
+        assert (
+            self.model_card_report._model_card.fairness_analysis.fairness_reports[
+                0
+            ].value
+            == 0.9
+        )