Merge pull request #496 from VectorInstitute/report_baseline_comparison

Report Template Update

Author: a-kore

cyclops/data/loader.py

@@ -8,6 +8,7 @@
 from datasets.arrow_dataset import Dataset
 from datasets.features import Image, Value
 from datasets.utils.logging import disable_progress_bar, enable_progress_bar
+from sklearn.model_selection import GroupShuffleSplit
 
 from cyclops.data.preprocess import nihcxr_preprocess
 from cyclops.data.utils import generate_timestamps
@@ -20,6 +21,7 @@ def load_nihcxr(
     train_time_range: Tuple[str, str] = ("1/1/2019", "10/19/2019"),
     test_time_range: Tuple[str, str] = ("10/20/2019", "12/25/2019"),
     progress: bool = False,
+    seed: int = 0,
 ) -> Dataset:
     """Load NIH Chest X-Ray dataset as a Huggingface dataset."""
     if not progress:
@@ -40,9 +42,15 @@ def load_nihcxr(
     train_df = df[df["Image Index"].isin(train_id)]
     test_df = df[df["Image Index"].isin(test_id)]
 
+    gss = GroupShuffleSplit(train_size=0.8, test_size=0.2, random_state=seed)
+    train_inds, val_inds = next(
+        gss.split(X=range(len(train_df)), groups=train_df["Patient ID"]),
+    )
+
     nih_ds = DatasetDict(
         {
-            "train": Dataset.from_pandas(train_df),
+            "train": Dataset.from_pandas(train_df.iloc[train_inds]),
+            "val": Dataset.from_pandas(train_df.iloc[val_inds]),
             "test": Dataset.from_pandas(test_df),
         },
     )

cyclops/monitor/detector.py

@@ -316,8 +316,7 @@ def balanced_sensitivity_test(
                     np.random.choice(ds_target.shape[0], sample, replace=False),
                 )
                 ds_target_balanced = concatenate_datasets(
-                    ds_target_sample1,
-                    ds_target_sample2,
+                    [ds_target_sample1, ds_target_sample2],
                 )
 
                 drift_results = self._detect_shift_sample(ds_target_balanced)

cyclops/report/model_card/fields.py

@@ -545,3 +545,90 @@ class FairnessReport(BaseModelCardField, composable_with=["FairnessAnalysis"]):
         description="Tests related to fairness considerations.",
         default_factory=list,
     )
+
+
+class MetricCard(
+    BaseModelCardField,
+    list_factory=True,
+    composable_with=["MetricCardCollection"],
+):
+    """A metric card."""
+
+    name: Optional[StrictStr] = Field(
+        None,
+        description="The name of the metric.",
+    )
+
+    type: Optional[StrictStr] = Field(
+        None,
+        description="The type of metric.",
+    )
+
+    slice: Optional[StrictStr] = Field(
+        None,
+        description="The name of the slice the metric was computed on.",
+    )
+
+    tooltip: Optional[StrictStr] = Field(
+        None,
+        description="A tooltip for the metric.",
+    )
+
+    value: Optional[StrictFloat] = Field(
+        None,
+        description="The value of the metric.",
+    )
+
+    threshold: Optional[StrictFloat] = Field(
+        None,
+        description="Threshold required to pass the test.",
+    )
+
+    passed: Optional[StrictBool] = Field(
+        None,
+        description="Whether the model result satisfies the given threshold.",
+    )
+
+    history: List[StrictFloat] = Field(
+        None,
+        description="History of the metric over time.",
+    )
+
+    trend: Optional[StrictStr] = Field(
+        None,
+        description="The trend of the metric over time.",
+    )
+
+    plot: Optional[GraphicsCollection] = Field(
+        None,
+        description="A plot of the performance over time.",
+    )
+
+
+class MetricCardCollection(BaseModelCardField, composable_with="Overview"):
+    """A collection of metric cards to be displayed in the model card."""
+
+    metrics: Optional[List[StrictStr]] = Field(
+        None,
+        description="A list of metric names in the Metric Card collection.",
+    )
+
+    tooltips: Optional[List[StrictStr]] = Field(
+        None,
+        description="A list of tooltips in the Metric Card collection.",
+    )
+
+    slices: Optional[List[StrictStr]] = Field(
+        None,
+        description="A list of slices in the Metric Card collection.",
+    )
+
+    values: Optional[List[List[StrictStr]]] = Field(
+        None,
+        description="A list of values for each slice in the Metric Card collection.",
+    )
+
+    collection: Optional[List[MetricCard]] = Field(
+        description="A collection of metric cards.",
+        default_factory=list,
+    )

cyclops/report/model_card/model_card.py

@@ -13,6 +13,7 @@
     FairnessAnalysis,
     ModelDetails,
     ModelParameters,
+    Overview,
     QuantitativeAnalysis,
 )
 
@@ -30,27 +31,15 @@ class Config(BaseModelCardConfig):
 
         extra: Extra = Extra.forbid
 
-    model_details: Optional[ModelDetails] = Field(
-        None,
-        description="Descriptive metadata for the model.",
-    )
-    model_parameters: Optional[ModelParameters] = Field(
+    overview: Optional[Overview] = Field(
         None,
-        description="Technical metadata for the model.",
+        description="A high-level overview of the model.",
     )
     datasets: Optional[Datasets] = Field(
         None,
         description="Information about the datasets used to train, validate \
         and/or test the model.",
     )
-    considerations: Optional[Considerations] = Field(
-        None,
-        description=inspect.cleandoc(
-            """
-            Any considerations related to model construction, training, and
-             application""",
-        ),
-    )
     quantitative_analysis: Optional[QuantitativeAnalysis] = Field(
         None,
         description="Quantitative analysis of model performance.",
@@ -63,6 +52,22 @@ class Config(BaseModelCardConfig):
         None,
         description="Fairness analysis being reported.",
     )
+    model_details: Optional[ModelDetails] = Field(
+        None,
+        description="Descriptive metadata for the model.",
+    )
+    model_parameters: Optional[ModelParameters] = Field(
+        None,
+        description="Technical metadata for the model.",
+    )
+    considerations: Optional[Considerations] = Field(
+        None,
+        description=inspect.cleandoc(
+            """
+            Any considerations related to model construction, training, and
+             application""",
+        ),
+    )
 
     def get_section(self, section_name: str) -> BaseModelCardSection:
         """Retrieve a section from the model card.

cyclops/report/model_card/sections.py

@@ -15,6 +15,7 @@
     GraphicsCollection,
     KeyVal,
     License,
+    MetricCardCollection,
     Owner,
     PerformanceMetric,
     Reference,
@@ -26,6 +27,15 @@
 )
 
 
+class Overview(BaseModelCardSection):
+    """Overview section with aggregate metrics."""
+
+    metric_cards: Optional[MetricCardCollection] = Field(
+        None,
+        description="Comparative metrics between baseline and periodic report.",
+    )
+
+
 class ModelDetails(BaseModelCardSection):
     """Details about the model."""