Fix SliceSpec and evaluator (#619)

* fixes for slicer and evaluators

* fix state reference creation in `MetricDict`

Author: fcogidi

cyclops/data/slicer.py

@@ -3,6 +3,7 @@
 import copy
 import datetime
 import itertools
+import json
 from dataclasses import dataclass, field
 from functools import partial
 from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
@@ -248,6 +249,19 @@ def _create_intersections(self) -> None:
             )
         self.spec_list.extend(intersect_list)
 
+        # remove duplicates
+        seen = set()
+        result = []
+
+        for spec in self.spec_list:
+            spec_str = json.dumps(spec, sort_keys=True)
+            if spec_str not in seen:
+                seen.add(spec_str)
+                result.append(spec)
+
+        seen.clear()
+        self.spec_list = result
+
     def _parse_and_register_slice_specs(
         self,
         slice_spec: Dict[str, Dict[str, Any]],

cyclops/evaluate/evaluator.py

@@ -152,7 +152,9 @@ def evaluate(
         fairness_config.batch_size = batch_size
         fairness_config.remove_columns = ignore_columns
 
-        fairness_results = evaluate_fairness(**asdict(fairness_config))
+        fairness_results = evaluate_fairness(
+            **asdict(fairness_config), array_lib=array_lib
+        )
         results["fairness"] = fairness_results
 
     return results
@@ -304,7 +306,7 @@ def _compute_metrics(
                         metrics.update(targets, predictions)
 
                     metric_output = metrics.compute()
-                    metrics.reset()
+                metrics.reset()
 
                 model_name: str = "model_for_%s" % prediction_column
                 results.setdefault(model_name, {})

cyclops/evaluate/fairness/evaluator.py

@@ -728,6 +728,9 @@ def _compute_metrics(  # noqa: C901, PLR0912
         The batch size to use for the computation.
     metric_name : Optional[str]
         The name of the metric to compute.
+    array_lib : {"torch", "numpy, "cupy"}, default="numpy"
+        The array library to use for the metric computation. The metric results
+        will be returned in the format of `array_lib`.
 
     Returns
     -------

cyclops/evaluate/metrics/experimental/metric.py

@@ -431,6 +431,7 @@ def reset(self) -> None:
                     "object or a list of array API objects. But got "
                     f"`{type(default_value)} instead.",
                 )
+        self._defaults = {}
 
         self._update_count = 0
         self._computed = None

cyclops/evaluate/metrics/experimental/metric_dict.py

@@ -361,7 +361,7 @@ def deepcopy_state(obj: Any) -> Any:
             for metric_names in self._metric_groups.values():
                 base_metric = self.data[metric_names[0]]
                 for metric_name in metric_names[1:]:
-                    for state in self.data[metric_name]._defaults:
+                    for state in base_metric._defaults:
                         base_metric_state = getattr(base_metric, state)
                         setattr(
                             self.data[metric_name],

docs/source/tutorials/synthea/los_prediction.ipynb

@@ -1068,16 +1068,6 @@
     ")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "172a1654",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "results"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "7d2d1d75-f7d8-44d3-a782-2aba9a4fbac0",

tests/cyclops/evaluate/metrics/experimental/test_metric.py

@@ -331,7 +331,7 @@ def test_reset_compute():
         anp.asarray(42, dtype=anp.float32),
     )
     metric.reset()
-    assert metric.state_vars == {"x": anp.asarray(0, dtype=anp.float32)}
+    assert metric.state_vars == {}
 
 
 def test_error_on_compute_before_update():
@@ -397,7 +397,7 @@ def test_call():
     assert metric._computed is None
 
     metric.reset()
-    assert metric.state_vars == {"x": anp.asarray(0, dtype=anp.float32)}
+    assert metric.state_vars == {}
     assert metric._computed is None