fixed measurement and codelist phenotype tests

Author: a-hartens

phenex/codelists/codelists.py

@@ -1,5 +1,6 @@
 import os
 from typing import Dict, List, Union, Optional
+import pandas as pd
 
 
 class Codelist:
@@ -149,8 +150,15 @@ def __repr__(self):
     codelist={self.codelist}
 )"""
 
+    def to_pandas(self) -> pd.DataFrame:
+        """
+        Convert the codelist to a pandas DataFrame.
+        """
+
+        _df = pd.DataFrame(self.to_tuples(), columns=["code_type", "code"])
+        _df['codelist'] = self.name
+        return _df
 
-import pandas as pd
 
 
 class LocalCSVCodelistFactory:

phenex/filters/aggregator.py

@@ -46,10 +46,13 @@ def aggregate(self, input_table: Table):
 
         # Apply the distinct reduction if required
         if self.reduce:
+            print("REDUCING")
             selected_columns = self.aggregation_index + [self.event_date_column]
             input_table = input_table.select(selected_columns).distinct()
+            print(selected_columns)
             input_table = input_table.mutate(VALUE=ibis.null())
 
+        print(input_table.to_pandas())
         return input_table
 
 class Nearest(VerticalDateAggregator):

phenex/phenotypes/codelist_phenotype.py

@@ -110,13 +110,13 @@ def _perform_time_filtering(self, code_table):
                 code_table = rtr.filter(code_table)
         return code_table
 
-    def _perform_date_selection(self, code_table):
+    def _perform_date_selection(self, code_table, reduce=True):
         if self.return_date is None or self.return_date == "all":
             return code_table
         if self.return_date == "first":
-            aggregator = First()
+            aggregator = First(reduce=reduce)
         elif self.return_date == "last":
-            aggregator = Last()
+            aggregator = Last(reduce=reduce)
         else:
             raise ValueError(f"Unknown return_date: {self.return_date}")
         return aggregator.aggregate(code_table)

phenex/phenotypes/measurement_phenotype.py

@@ -114,7 +114,7 @@ def _execute(self, tables) -> PhenotypeTable:
         code_table = tables[self.domain]
         code_table = self._perform_codelist_filtering(code_table)
         code_table = self._perform_time_filtering(code_table)
-        code_table = self._perform_date_selection(code_table)
+        code_table = self._perform_date_selection(code_table, reduce=False)
         code_table = self._perform_nonphysiological_value_filtering(code_table)
         code_table = self._perform_value_aggregation(code_table)
         code_table = self._perform_value_filtering(code_table)

phenex/test/phenotypes/test_codelist_phenotype.py

@@ -547,6 +547,6 @@ def test_codelist_phenotype():
 
 if __name__ == "__main__":
     test_codelist_phenotype()
-    test_relative_time_range_filter()
-    test_anchor_phenotype()
-    test_return_date()
+    # test_relative_time_range_filter()
+    # test_anchor_phenotype()
+    # test_return_date()