Merge branch 'main' into feat/census-data-maps

Author: joaquimds

hub/graphql/types/model_types.py

@@ -959,6 +959,11 @@ class BaseDataSource(Analytics):
     allow_updates: bool = attr_field()
     default_data_type: Optional[str] = attr_field()
     defaults: JSON = attr_field()
+    auto_update_enabled: auto
+    auto_import_enabled: auto
+    remote_name: Optional[str] = fn_field()
+    remote_url: Optional[str] = fn_field()
+    healthcheck: bool = fn_field()
 
     field_definitions: Optional[List[FieldDefinition]] = strawberry_django.field(
         resolver=lambda self: self.field_definitions()
@@ -1063,11 +1068,6 @@ class ExternalDataSource(BaseDataSource):
         strawberry_django_dataloaders.fields.auto_dataloader_field()
     )
     update_mapping: Optional[List["AutoUpdateConfig"]]
-    auto_update_enabled: auto
-    auto_import_enabled: auto
-    remote_name: Optional[str] = fn_field()
-    remote_url: Optional[str] = fn_field()
-    healthcheck: bool = fn_field()
     orgs_with_access: List[Organisation]
 
     @strawberry_django.field
@@ -1222,6 +1222,7 @@ def get_queryset(cls, queryset, info, **kwargs):
 @strawberry.enum
 class InspectorDisplayType(Enum):
     BigNumber = "BigNumber"
+    BigRecord = "BigRecord"
     ElectionResult = "ElectionResult"
     List = "List"
     Properties = "Properties"
@@ -1239,9 +1240,7 @@ class MapLayer:
     icon_image: Optional[str] = dict_key_field()
     mapbox_paint: Optional[JSON] = dict_key_field()
     mapbox_layout: Optional[JSON] = dict_key_field()
-    inspector_type: Optional[InspectorDisplayType] = dict_key_field(
-        default=InspectorDisplayType.Table
-    )
+    inspector_type: Optional[str] = dict_key_field(default=InspectorDisplayType.Table)
     inspector_config: Optional[JSON] = dict_key_field()
 
     @strawberry_django.field
@@ -1679,6 +1678,12 @@ class MapBounds:
     south: float
     west: float
 
+@strawberry.enum
+class ChoroplethMode(Enum):
+    Count = "Count"
+    Field = "Field"
+    Formula = "Formula"
+
 
 @strawberry_django.field()
 def choropleth_data_for_source(
@@ -1687,8 +1692,10 @@ def choropleth_data_for_source(
     analytical_area_key: AnalyticalAreaType,
     # Field could be a column name or a Pandas formulaic expression
     # or, if not provided, a count of records
+    mode: Optional[ChoroplethMode] = ChoroplethMode.Count,
     field: Optional[str] = None,
     map_bounds: Optional[MapBounds] = None,
+    formula: Optional[str] = None,
 ) -> List[GroupedDataCount]:
     # Check user can access the external data source
     user = get_current_user(info)
@@ -1728,7 +1735,7 @@ def choropleth_data_for_source(
         )
         combined_area = areas.aggregate(union=GisUnion("polygon"))["union"]
         # all geocoded GenericData should have `point` set
-        qs = qs.filter(point__within=combined_area)
+        qs = qs.filter(point__within=combined_area or bbox)
 
     qs = qs.values("json", "label", "gss")
 
@@ -1753,12 +1760,47 @@ def choropleth_data_for_source(
     # TODO: maybe make this explicit via an argument?
     # is_data_source_statistical = external_data_source.data_type == models.ExternalDataSource.DataSourceType.AREA_STATS
     # check that field is in DF
-    field_is_set = field and field is not None and len(field)
-    is_explicit_row_count = field_is_set and field == "__COUNT__"
-    is_valid_statistical_field = field_is_set and not is_explicit_row_count
-    is_valid_row_counter = is_explicit_row_count or not field_is_set
+    is_valid_field = field and field is not None and len(field) and field in df.columns
+    is_row_count = mode is ChoroplethMode.Count
+    is_valid_formula = formula and formula is not None and len(formula)
+
+    if mode is ChoroplethMode.Field and not is_valid_field:
+        raise ValueError("Field not found in data source")
+
+    if mode is ChoroplethMode.Formula and not is_valid_formula:
+        raise ValueError("Formula is invalid")
 
-    if is_valid_statistical_field:
+    if is_row_count:
+        # Simple count of data points per area
+
+        # Count the number of rows per GSS
+        df_count = (
+            df.drop(columns=["label"]).groupby("gss").size().reset_index(name="count")
+        )
+
+        # Calculate the mode for the 'label' column
+        def get_mode(series):
+            try:
+                return series.mode()[0]
+            except KeyError:
+                return None
+
+        df_mode = df.groupby("gss")["label"].agg(get_mode).reset_index()
+
+        # Merge the summed DataFrame with the mode DataFrame
+        df = pd.merge(df_count, df_mode, on="gss")
+
+        # Convert DF to GroupedDataCount(label=label, gss=gss, count=count) list
+        return [
+            GroupedDataCount(
+                label=row.label,
+                gss=row.gss,
+                count=row.count,
+                formatted_count=f"{row.count:,.0f}",
+            )
+            for row in df.itertuples()
+        ]
+    elif is_valid_field or is_valid_formula:
         # Convert any stringified JSON numbers to floats
         for column in df:
             if all(df[column].apply(check_numeric)):
@@ -1818,16 +1860,16 @@ def get_mode(series):
 
         # Now fetch the requested series from the dataframe
         # If the field is a column name, we can just return that column
-        if field in df.columns:
+        if is_valid_field:
             df["count"] = df[field]
         # If the field is a formula, we need to evaluate it
-        else:
+        elif is_valid_formula:
             try:
-                df["count"] = df.eval(field)
+                df["count"] = df.eval(formula)
             except ValueError:
                 # In case "where" is used, which pandas doesn't support
                 # https://github.com/pandas-dev/pandas/issues/34834
-                df["count"] = ne.evaluate(field, local_dict=df)
+                df["count"] = ne.evaluate(formula, local_dict=df)
 
         # Check if count is between 0 and 1: if so, it's a percentage
         is_percentage = df["count"].between(0, 2).all() or False
@@ -1852,35 +1894,6 @@ def get_mode(series):
             )
             for row in df.itertuples()
         ]
-    elif is_valid_row_counter:
-        # Simple count of data points per area
-
-        # Count the number of rows per GSS
-        df_count = (
-            df.drop(columns=["label"]).groupby("gss").size().reset_index(name="count")
-        )
-
-        # Calculate the mode for the 'label' column
-        def get_mode(series):
-            try:
-                return series.mode()[0]
-            except KeyError:
-                return None
-
-        df_mode = df.groupby("gss")["label"].agg(get_mode).reset_index()
-
-        # Merge the summed DataFrame with the mode DataFrame
-        df = pd.merge(df_count, df_mode, on="gss")
-
-        # Convert DF to GroupedDataCount(label=label, gss=gss, count=count) list
-        return [
-            GroupedDataCount(
-                label=row.label,
-                gss=row.gss,
-                count=row.count,
-                formatted_count=f"{row.count:,.0f}",
-            )
-            for row in df.itertuples()
-        ]
+    
     else:
         raise ValueError("Incorrect configuration for choropleth")

hub/models.py

@@ -800,16 +800,26 @@ def __str__(self):
 
     @property
     def name(self) -> Optional[str]:
-        if self.full_name:
-            return self.full_name
-        elif self.first_name and self.last_name:
-            return f"{self.first_name} {self.last_name}"
-        elif self.first_name:
-            return self.first_name
-        elif self.last_name:
-            return self.last_name
-        elif self.title:
-            return self.title
+        full_name = (
+            self.full_name if self.full_name and len(self.full_name) > 0 else None
+        )
+
+        merged_name = (
+            f"{self.first_name} {self.last_name}".strip()
+            if self.first_name
+            and self.last_name
+            and len(self.first_name) > 0
+            and len(self.last_name) > 0
+            else None
+        )
+
+        # pick whichever is longer
+        if full_name and merged_name:
+            full_name = sorted([full_name, merged_name], key=len, reverse=True)[0]
+
+        for option in [full_name, self.first_name, self.last_name, self.title]:
+            if option and len(option) > 0:
+                return option
 
         return None
 

hub/tests/test_external_data_source_parsers.py

@@ -349,7 +349,7 @@ def setUpTestData(cls):
                     "id": "3",
                     "venue_name": "Sainsbury's Local",
                     "address": "Gordon Street",
-                    "expected_postcode": "G1 3RS",
+                    "expected_postcode": "G1 3PE",
                 },
                 {
                     # Special case: "online"

hub/tests/test_permissions.py

@@ -107,7 +107,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss
@@ -251,7 +251,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss
@@ -393,7 +393,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss
@@ -532,7 +532,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss
@@ -659,7 +659,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss
@@ -751,7 +751,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss
@@ -852,7 +852,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss
@@ -944,7 +944,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss
@@ -1080,7 +1080,7 @@ def test_aggregate_data_count(self):
                 choroplethDataForSource(
                     sourceId: $sourceId
                     analyticalAreaKey: european_electoral_region
-                    field: ""
+                    mode: Count
                 )
                 {
                   gss