feat: add census data maps

* Command to import output areas and MSOA/LSOA
* GraphQL query for choropleth data can be filtered by map bounds
* Front-end automatically selects the tileset for the zoom level
  (census boundary type only)

Author: joaquimds

.DS_Store

@@ -43,7 +43,6 @@ echo '';
 echo 'Downloading LIH data...';
 echo '';
 echo '------------------------------------';
-python manage.py import_wards
 python manage.py import_areas
 python manage.py import_regions
 python manage.py import_mps

bin/ck_setup.sh

@@ -2,7 +2,7 @@
 
 if [ "$ENVIRONMENT" != "production" ]; then
     unzip -o data/areas.psql.zip -d data
-    PGPASSWORD=password psql -U postgres -h db test_local-intelligence < data/areas.psql
+    PGPASSWORD=password psql -U postgres -h db test_postgres < data/areas.psql
 else
     echo "This command cannot run in production environments."
 fi

bin/import_areas_seed.sh

@@ -1,14 +1,14 @@
 services:
   db:
-    image: kartoza/postgis:latest
+    image: kartoza/postgis:16-3
     restart: always
     volumes:
       - .:/app
-      - pgdata:/var/lib/postgresql/data
+      - pgdata:/var/lib/postgresql
     environment:
       POSTGRES_USER: 'postgres'
       POSTGRES_PASSWORD: 'password'
-      POSTGRES_DB: 'local-intelligence'
+      POSTGRES_DB: 'postgres'
       POSTGRES_PORT: 5432
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres -h 127.0.0.1"]
@@ -32,7 +32,7 @@ services:
     environment:
       SECRET_KEY: 'secret'
       DEBUG: 1
-      DATABASE_URL: 'postgis://postgres:password@db/local-intelligence'
+      DATABASE_URL: 'postgis://postgres:password@db/postgres'
       EMAIL_HOST: email.svc
       CACHE_FILE: 'data/cache'
       MAPIT_URL: 'https://mapit.mysociety.org/'

docker-compose.yml

@@ -336,75 +336,81 @@ async def import_area_data(
             geocoding_data["area_fields"][area.area_type.code] = area.gss
             update_data["geocode_data"].update({"data": geocoding_data})
     if area is not None:
-        sample_point = area.polygon.centroid
+        postcode_data = await get_postcode_data_for_area(area, loaders, steps)
+        update_data["postcode_data"] = postcode_data
+        update_data["area"] = area
+        update_data["point"] = area.point
+    else:
+        # Reset geocoding data
+        update_data["postcode_data"] = None
 
-        # get postcodeIO result for area.coordinates
-        try:
-            postcode_data: PostcodesIOResult = await loaders[
-                "postcodesIOFromPoint"
-            ].load(sample_point)
-        except Exception as e:
-            logger.error(f"Failed to get postcode data for {sample_point}: {e}")
-            postcode_data = None
+    # Update the geocode data regardless, for debugging purposes
+    update_data["geocode_data"].update({"steps": steps})
 
+    await GenericData.objects.aupdate_or_create(
+        data_type=data_type, data=source.get_record_id(record), defaults=update_data
+    )
+
+
+async def get_postcode_data_for_area(area, loaders, steps):
+    sample_point = area.polygon.centroid
+    # get postcodeIO result for area.coordinates
+    try:
+        postcode_data: PostcodesIOResult = await loaders["postcodesIOFromPoint"].load(
+            sample_point
+        )
+    except Exception as e:
+        logger.error(f"Failed to get postcode data for {sample_point}: {e}")
+        postcode_data = None
+
+    steps.append(
+        {
+            "task": "postcode_from_area_coordinates",
+            "service": Geocoder.POSTCODES_IO.value,
+            "result": "failed" if postcode_data is None else "success",
+        }
+    )
+
+    # Try a few other backup strategies (example postcode, another geocoder)
+    # to get postcodes.io data
+    if postcode_data is None:
+        postcode = await get_example_postcode_from_area_gss(area.gss)
         steps.append(
             {
-                "task": "postcode_from_area_coordinates",
-                "service": Geocoder.POSTCODES_IO.value,
-                "result": "failed" if postcode_data is None else "success",
+                "task": "postcode_from_area",
+                "service": Geocoder.FINDTHATPOSTCODE.value,
+                "result": "failed" if postcode is None else "success",
             }
         )
-
-        # Try a few other backup strategies (example postcode, another geocoder)
-        # to get postcodes.io data
-        if postcode_data is None:
-            postcode = await get_example_postcode_from_area_gss(area.gss)
+        if postcode is not None:
+            postcode_data = await loaders["postcodesIO"].load(postcode)
             steps.append(
                 {
-                    "task": "postcode_from_area",
-                    "service": Geocoder.FINDTHATPOSTCODE.value,
-                    "result": "failed" if postcode is None else "success",
+                    "task": "data_from_postcode",
+                    "service": Geocoder.POSTCODES_IO.value,
+                    "result": ("failed" if postcode_data is None else "success"),
                 }
             )
-            if postcode is not None:
-                postcode_data = await loaders["postcodesIO"].load(postcode)
-                steps.append(
-                    {
-                        "task": "data_from_postcode",
-                        "service": Geocoder.POSTCODES_IO.value,
-                        "result": ("failed" if postcode_data is None else "success"),
-                    }
-                )
-        if postcode_data is None:
-            postcode = await get_postcode_from_coords_ftp(sample_point)
+    if postcode_data is None:
+        postcode = await get_postcode_from_coords_ftp(sample_point)
+        steps.append(
+            {
+                "task": "postcode_from_area_coordinates",
+                "service": Geocoder.FINDTHATPOSTCODE.value,
+                "result": "failed" if postcode is None else "success",
+            }
+        )
+        if postcode is not None:
+            postcode_data = await loaders["postcodesIO"].load(postcode)
             steps.append(
                 {
-                    "task": "postcode_from_area_coordinates",
-                    "service": Geocoder.FINDTHATPOSTCODE.value,
-                    "result": "failed" if postcode is None else "success",
+                    "task": "data_from_postcode",
+                    "service": Geocoder.POSTCODES_IO.value,
+                    "result": ("failed" if postcode_data is None else "success"),
                 }
             )
-            if postcode is not None:
-                postcode_data = await loaders["postcodesIO"].load(postcode)
-                steps.append(
-                    {
-                        "task": "data_from_postcode",
-                        "service": Geocoder.POSTCODES_IO.value,
-                        "result": ("failed" if postcode_data is None else "success"),
-                    }
-                )
-
-        update_data["postcode_data"] = postcode_data
-    else:
-        # Reset geocoding data
-        update_data["postcode_data"] = None
-
-    # Update the geocode data regardless, for debugging purposes
-    update_data["geocode_data"].update({"steps": steps})
 
-    await GenericData.objects.aupdate_or_create(
-        data_type=data_type, data=source.get_record_id(record), defaults=update_data
-    )
+    return postcode_data
 
 
 async def import_address_data(

hub/data_imports/geocoding_config.py

@@ -4,7 +4,11 @@
 from enum import Enum
 from typing import List, Optional, Union
 
+from django.contrib.gis.db.models import Union as GisUnion
+from django.contrib.gis.geos import Polygon
 from django.db.models import F, Q
+from django.db.models.fields import FloatField
+from django.db.models.functions import Cast
 from django.http import HttpRequest
 
 import numexpr as ne
@@ -48,6 +52,7 @@
 pd.core.computation.ops.MATHOPS = (*pd.core.computation.ops.MATHOPS, "where")
 
 logger = logging.getLogger(__name__)
+pd.core.computation.ops.MATHOPS = (*pd.core.computation.ops.MATHOPS, "where")
 
 
 # Ideally we'd just import this from the library (procrastinate.jobs.Status) but
@@ -734,6 +739,9 @@ def query_filter(self) -> dict[str, str]:
     AnalyticalAreaType.admin_ward: AreaTypeFilter(lih_area_type="WD23"),
     AnalyticalAreaType.european_electoral_region: AreaTypeFilter(lih_area_type="EER"),
     AnalyticalAreaType.european_electoral_region: AreaTypeFilter(lih_area_type="CTRY"),
+    AnalyticalAreaType.msoa: AreaTypeFilter(lih_area_type="MSOA"),
+    AnalyticalAreaType.lsoa: AreaTypeFilter(lih_area_type="LSOA"),
+    AnalyticalAreaType.output_area: AreaTypeFilter(lih_area_type="OA21"),
 }
 
 
@@ -1664,6 +1672,14 @@ def check_numeric(x):
         return False
 
 
+@strawberry.input
+class MapBounds:
+    north: float
+    east: float
+    south: float
+    west: float
+
+
 @strawberry_django.field()
 def choropleth_data_for_source(
     info: Info,
@@ -1672,6 +1688,7 @@ def choropleth_data_for_source(
     # Field could be a column name or a Pandas formulaic expression
     # or, if not provided, a count of records
     field: Optional[str] = None,
+    map_bounds: Optional[MapBounds] = None,
 ) -> List[GroupedDataCount]:
     # Check user can access the external data source
     user = get_current_user(info)
@@ -1691,10 +1708,33 @@ def choropleth_data_for_source(
         .annotate(
             label=F(f"postcode_data__{analytical_area_key.value}"),
             gss=F(f"postcode_data__codes__{analytical_area_key.value}"),
+            latitude=Cast("postcode_data__latitude", output_field=FloatField()),
+            longitude=Cast("postcode_data__longitude", output_field=FloatField()),
         )
-        .values("json", "label", "gss")
     )
 
+    if map_bounds:
+        area_type_filter = postcodeIOKeyAreaTypeLookup[analytical_area_key]
+        bbox_coords = (
+            (map_bounds.west, map_bounds.north),  # Top left
+            (map_bounds.east, map_bounds.north),  # Top right
+            (map_bounds.east, map_bounds.south),  # Bottom right
+            (map_bounds.west, map_bounds.south),  # Bottom left
+            (map_bounds.west, map_bounds.north),  # Back to start to close polygon
+        )
+        bbox = Polygon(bbox_coords, srid=4326)
+        areas = models.Area.objects.filter(**area_type_filter.query_filter).filter(
+            point__within=bbox
+        )
+        combined_area = areas.aggregate(union=GisUnion("polygon"))["union"]
+        # all geocoded GenericData should have `point` set
+        qs = qs.filter(point__within=combined_area)
+
+    qs = qs.values("json", "label", "gss")
+
+    if not qs:
+        return []
+
     # ingest the .json data into a pandas dataframe so we can do analytics
     df = pd.DataFrame([record for record in qs])
 

hub/graphql/types/model_types.py

@@ -40,15 +40,7 @@ class Command(BaseCommand):
     without causing primary key conflicts.
     """
 
-    def add_arguments(self, parser):
-        parser.add_argument(
-            "-a",
-            "--all-names",
-            action="store_true",
-            help="Fetch alternative names from MapIt",
-        )
-
-    def handle(self, all_names: bool = False, *args, **options):
+    def handle(self, *args, **options):
         print("Exporting areas and area types from current database to data/areas.psql")
         count = 0
         output_file: Path = settings.BASE_DIR / "data" / "areas.psql"
@@ -101,6 +93,8 @@ def get_output_record(
             if column.name in table_config.output_column_templates:
                 template = table_config.output_column_templates[column.name]
                 value = self.template_output_value(template, row, columns)
+            elif row[i] is None:
+                value = "NULL"
             else:
                 # output the value as a string, cast to the correct type in postgres
                 value = f"'{self.escape_sql_string(row[i])}'::{column.type_display}"

hub/management/commands/export_areas_as_sql.py

@@ -57,14 +57,15 @@ def handle(self, quiet: bool = False, all_names: bool = False, *args, **options)
     def import_area(self, area, area_type, all_names):
         area_details = self.mapit_client.area_details(area["id"]) if all_names else {}
 
-        if "gss" not in area["codes"]:
+        gss = area["codes"].get("gss") or area["codes"].get("ons")
+        if not gss:
             # logger.debug(f"no gss code for {area['id']}")
             return
 
         geom = None
         try:
             geom_already_loaded = Area.objects.filter(
-                gss=area["codes"]["gss"], polygon__isnull=False
+                gss=gss, polygon__isnull=False
             ).exists()
             if geom_already_loaded:
                 # Only fetch geometry data if required, to speed things up
@@ -77,19 +78,19 @@ def import_area(self, area, area_type, all_names):
                     "type": "Feature",
                     "geometry": geom,
                     "properties": {
-                        "PCON13CD": area["codes"]["gss"],
+                        "PCON13CD": gss,
                         "name": area["name"],
                         "type": area_type.code,
                         "mapit_type": area["type"],
                     },
                 }
                 geom_str = json.dumps(geom)
-        except mapit.NotFoundException:  # pragma: no cover
+        except (mapit.NotFoundException, mapit.BadRequestException):  # pragma: no cover
             print(f"could not find mapit area for {area['name']}")
             geom = None
 
         a, created = Area.objects.update_or_create(
-            gss=area["codes"]["gss"],
+            gss=gss,
             area_type=area_type,
             defaults={
                 "mapit_id": area["id"],