diff --git a/hub/graphql/schema.py b/hub/graphql/schema.py index 5300b5e6c..559c1233e 100644 --- a/hub/graphql/schema.py +++ b/hub/graphql/schema.py @@ -131,7 +131,7 @@ class Query(UserQueries): resolver=model_types.statistics, extensions=[IsAuthenticated()], ) - statistics_for_choropleth: Optional[List[model_types.GroupedDataCount]] = ( + statistics_for_choropleth: List[model_types.GroupedDataCount] = ( strawberry_django.field( resolver=model_types.statistics_for_choropleth, extensions=[IsAuthenticated()], diff --git a/hub/graphql/types/model_types.py b/hub/graphql/types/model_types.py index 5f1a12ee2..7d4664d51 100644 --- a/hub/graphql/types/model_types.py +++ b/hub/graphql/types/model_types.py @@ -47,8 +47,7 @@ from hub.management.commands.import_mps import party_shades from utils.geo_reference import ( AnalyticalAreaType, - area_to_postcode_io_filter, - lih_to_postcodes_io_key_map, + area_to_postcode_io_key, ) from utils.postcode import get_postcode_data_for_gss from utils.statistics import ( @@ -519,7 +518,7 @@ class Area: @strawberry_django.field def analytical_area_type(self) -> Optional[AnalyticalAreaType]: - return area_to_postcode_io_filter(self) + return area_to_postcode_io_key(self) @strawberry_django.field async def last_election(self, info: Info) -> Optional[ConstituencyElectionResult]: @@ -1488,9 +1487,7 @@ def __get_generic_data_for_area_and_external_data_source( elif mode is stats.AreaQueryMode.AREA_OR_CHILDREN: filters = Q(area__gss=area.gss) # Or find GenericData tagged with area that is fully contained by this area's polygon - postcode_io_key = area_to_postcode_io_filter(area) - if postcode_io_key is None: - postcode_io_key = lih_to_postcodes_io_key_map.get(area.area_type.code, None) + postcode_io_key = area_to_postcode_io_key(area) if postcode_io_key: subclause = Q() # See if there's a matched postcode data field for this area @@ -1504,9 +1501,7 @@ def __get_generic_data_for_area_and_external_data_source( elif mode is stats.AreaQueryMode.AREA_OR_PARENTS: filters = Q(area__gss=area.gss) # Or find GenericData tagged with area that fully contains this area's polygon - postcode_io_key = area_to_postcode_io_filter(area) - if postcode_io_key is None: - postcode_io_key = lih_to_postcodes_io_key_map.get(area.area_type.code, None) + postcode_io_key = area_to_postcode_io_key(area) if postcode_io_key: subclause = Q() # See if there's a matched postcode data field for this area @@ -1520,9 +1515,7 @@ def __get_generic_data_for_area_and_external_data_source( elif mode is stats.AreaQueryMode.OVERLAPPING: filters = Q(area__gss=area.gss) # Or find GenericData tagged with area that overlaps this area's polygon - postcode_io_key = area_to_postcode_io_filter(area) - if postcode_io_key is None: - postcode_io_key = lih_to_postcodes_io_key_map.get(area.area_type.code, None) + postcode_io_key = area_to_postcode_io_key(area) if postcode_io_key: filters |= Q(**{f"postcode_data__codes__{postcode_io_key.value}": area.gss}) @@ -1932,14 +1925,19 @@ def statistics_for_choropleth( stats_config: stats.StatisticsConfig, category_key: Optional[str] = None, count_key: Optional[str] = None, + map_bounds: Optional[stats.MapBounds] = None, ): user = get_current_user(info) for source in stats_config.source_ids: check_user_can_view_source(user, source) - return stats.statistics( - stats_config, - as_grouped_data=True, - category_key=category_key, - count_key=count_key, - ) + return ( + stats.statistics( + stats_config, + as_grouped_data=True, + category_key=category_key, + count_key=count_key, + map_bounds=map_bounds, + ) + or [] + ) # Convert None to empty list for better front-end integration diff --git a/hub/graphql/types/stats.py b/hub/graphql/types/stats.py index 041ec9eee..d8e79bd03 100644 --- a/hub/graphql/types/stats.py +++ b/hub/graphql/types/stats.py @@ -14,7 +14,7 @@ from hub import models from utils.geo_reference import ( AnalyticalAreaType, - area_to_postcode_io_filter, + area_to_postcode_io_key, lih_to_postcodes_io_key_map, ) from utils.statistics import ( @@ -141,7 +141,6 @@ class StatisticsConfig: # Querying gss_codes: Optional[List[str]] = None area_query_mode: Optional[AreaQueryMode] = None - map_bounds: Optional[MapBounds] = None # Grouping # Group absolutely: flatten all array items down to one, without any special transforms group_absolutely: Optional[bool] = False @@ -222,6 +221,7 @@ def statistics( category_key: Optional[str] = None, count_key: Optional[str] = None, return_numeric_keys_only: Optional[bool] = False, + map_bounds: Optional[MapBounds] = None, ): pre_calcs = ( [c for c in conf.pre_group_by_calculated_columns if not c.ignore] @@ -239,38 +239,40 @@ def statistics( data_type__data_set__external_data_source_id__in=conf.source_ids ) - area_type_filter = None # if group_by_area: # area_type_filter = postcodeIOKeyAreaTypeLookup[group_by_area] - if conf.map_bounds: - # area_type_filter = postcodeIOKeyAreaTypeLookup[group_by_area] + if map_bounds: bbox_coords = ( - (conf.map_bounds.west, conf.map_bounds.north), # Top left - (conf.map_bounds.east, conf.map_bounds.north), # Top right - (conf.map_bounds.east, conf.map_bounds.south), # Bottom right - (conf.map_bounds.west, conf.map_bounds.south), # Bottom left + (map_bounds.west, map_bounds.north), # Top left + (map_bounds.east, map_bounds.north), # Top right + (map_bounds.east, map_bounds.south), # Bottom right + (map_bounds.west, map_bounds.south), # Bottom left ( - conf.map_bounds.west, - conf.map_bounds.north, + map_bounds.west, + map_bounds.north, ), # Back to start to close polygon ) bbox = Polygon(bbox_coords, srid=4326) - # areas = models.Area.objects.filter(**area_type_filter.query_filter).filter( - # point__within=bbox - # ) - # combined_area = areas.aggregate(union=GisUnion("polygon"))["union"] - # all geocoded GenericData should have `point` set - qs = qs.filter(point__within=bbox) + if conf.group_by_area: + # If group_by_area is set, the analysis must get *all* the points within relevant Areas, + # not just the points within the bounding box. Otherwise, data for an Area + # might be incomplete. + area_type_filter = postcodeIOKeyAreaTypeLookup[conf.group_by_area] + areas = models.Area.objects.filter(**area_type_filter.query_filter).filter( + point__within=bbox + ) + combined_area = areas.aggregate(union=GisUnion("polygon"))["union"] + qs = qs.filter(point__within=combined_area or bbox) + else: + # all geocoded GenericData should have `point` set + qs = qs.filter(point__within=bbox) filters = Q() - if conf.gss_codes or area_type_filter: - if conf.gss_codes: - area_qs = models.Area.objects.filter(gss__in=conf.gss_codes) - example_area = area_qs.first() - combined_areas_polygon = area_qs.aggregate(union=GisUnion("polygon"))[ - "union" - ] + if conf.gss_codes: + area_qs = models.Area.objects.filter(gss__in=conf.gss_codes) + example_area = area_qs.first() + combined_areas_polygon = area_qs.aggregate(union=GisUnion("polygon"))["union"] if ( combined_areas_polygon @@ -292,11 +294,7 @@ def statistics( if combined_areas_polygon: filters |= Q(area__gss__in=conf.gss_codes) # Or find GenericData tagged with area that is fully contained by this area's polygon - postcode_io_key = area_to_postcode_io_filter(example_area) - if postcode_io_key is None: - postcode_io_key = lih_to_postcodes_io_key_map.get( - example_area.area_type.code, None - ) + postcode_io_key = area_to_postcode_io_key(example_area) if postcode_io_key: subclause = Q() # See if there's a matched postcode data field for this area @@ -315,11 +313,7 @@ def statistics( if combined_areas_polygon: filters |= Q(area__gss__in=conf.gss_codes) # Or find GenericData tagged with area that fully contains this area's polygon - postcode_io_key = area_to_postcode_io_filter(example_area) - if postcode_io_key is None: - postcode_io_key = lih_to_postcodes_io_key_map.get( - example_area.area_type.code, None - ) + postcode_io_key = area_to_postcode_io_key(example_area) if postcode_io_key: subclause = Q() # See if there's a matched postcode data field for this area @@ -338,11 +332,7 @@ def statistics( if conf.gss_codes: filters |= Q(area__gss__in=conf.gss_codes) # Or find GenericData tagged with area that overlaps this area's polygon - postcode_io_key = area_to_postcode_io_filter(example_area) - if postcode_io_key is None: - postcode_io_key = lih_to_postcodes_io_key_map.get( - example_area.area_type.code, None - ) + postcode_io_key = area_to_postcode_io_key(example_area) if postcode_io_key: filters |= Q( **{ @@ -443,6 +433,12 @@ def get_group_by_area_properties(row): return None, None, None # Find the key of `lih_to_postcodes_io_key_map` where the value is `group_by_area`: + # TODO: check this for admin_county and admin_district. For some admin_districts, + # we will return "DIS" when the correct value is "STC". + # + # admin_county => MapIt CTY => LIH STC + # admin_district => MapIt LBO/UTA/COI/LGD/MTD/NMD => LIH STC + # => MapIt DIS => LIH DIS area_type = next( ( k diff --git a/nextjs/src/__generated__/gql.ts b/nextjs/src/__generated__/gql.ts index 8bc5f58b8..e170e5fad 100644 --- a/nextjs/src/__generated__/gql.ts +++ b/nextjs/src/__generated__/gql.ts @@ -69,7 +69,7 @@ const documents = { "\n mutation DeleteMapReport($id: IDObject!) {\n deleteMapReport(data: $id) {\n id\n }\n }\n": types.DeleteMapReportDocument, "\n query GetMapReportName($id: ID!) {\n mapReport(pk: $id) {\n id\n name\n }\n }\n": types.GetMapReportNameDocument, "\n query SourceStatsByBoundary(\n $sourceId: String!\n $analyticalAreaType: AnalyticalAreaType!\n $mode: ChoroplethMode\n $field: String\n $formula: String\n $mapBounds: MapBounds\n ) {\n choroplethDataForSource(\n sourceId: $sourceId\n analyticalAreaKey: $analyticalAreaType\n mode: $mode\n field: $field\n formula: $formula\n mapBounds: $mapBounds\n ) {\n label\n gss\n count\n formattedCount\n category\n row\n columns\n gssArea {\n point {\n type\n geometry {\n type\n coordinates\n }\n }\n }\n }\n }\n": types.SourceStatsByBoundaryDocument, - "\n query Statistics(\n $config: StatisticsConfig!\n $categoryKey: String\n $countKey: String\n ) {\n statisticsForChoropleth(\n statsConfig: $config\n categoryKey: $categoryKey\n countKey: $countKey\n ) {\n label\n gss\n count\n formattedCount\n category\n row\n columns\n gssArea {\n point {\n type\n geometry {\n type\n coordinates\n }\n }\n }\n }\n }\n": types.StatisticsDocument, + "\n query Statistics(\n $config: StatisticsConfig!\n $categoryKey: String\n $countKey: String\n $mapBounds: MapBounds\n ) {\n statisticsForChoropleth(\n statsConfig: $config\n categoryKey: $categoryKey\n countKey: $countKey\n mapBounds: $mapBounds\n ) {\n label\n gss\n count\n formattedCount\n category\n row\n columns\n gssArea {\n point {\n type\n geometry {\n type\n coordinates\n }\n }\n }\n }\n }\n": types.StatisticsDocument, "\n query SourceMetadata($sourceId: ID!) {\n externalDataSource(pk: $sourceId) {\n fieldDefinitions {\n externalId\n value\n label\n }\n }\n }\n": types.SourceMetadataDocument, "\n mutation WebhookRefresh($ID: String!) {\n refreshWebhooks(externalDataSourceId: $ID) {\n id\n hasWebhooks\n automatedWebhooks\n webhookHealthcheck\n }\n }\n": types.WebhookRefreshDocument, "\n fragment DataSourceCard on ExternalDataSource {\n id\n name\n dataType\n crmType\n automatedWebhooks\n autoImportEnabled\n autoUpdateEnabled\n updateMapping {\n source\n sourcePath\n destinationColumn\n }\n jobs(pagination: { limit: 10 }) {\n lastEventAt\n status\n }\n sharingPermissions {\n id\n organisation {\n id\n name\n }\n }\n }\n": types.DataSourceCardFragmentDoc, @@ -335,7 +335,7 @@ export function gql(source: "\n query SourceStatsByBoundary(\n $sourceId: St /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ -export function gql(source: "\n query Statistics(\n $config: StatisticsConfig!\n $categoryKey: String\n $countKey: String\n ) {\n statisticsForChoropleth(\n statsConfig: $config\n categoryKey: $categoryKey\n countKey: $countKey\n ) {\n label\n gss\n count\n formattedCount\n category\n row\n columns\n gssArea {\n point {\n type\n geometry {\n type\n coordinates\n }\n }\n }\n }\n }\n"): (typeof documents)["\n query Statistics(\n $config: StatisticsConfig!\n $categoryKey: String\n $countKey: String\n ) {\n statisticsForChoropleth(\n statsConfig: $config\n categoryKey: $categoryKey\n countKey: $countKey\n ) {\n label\n gss\n count\n formattedCount\n category\n row\n columns\n gssArea {\n point {\n type\n geometry {\n type\n coordinates\n }\n }\n }\n }\n }\n"]; +export function gql(source: "\n query Statistics(\n $config: StatisticsConfig!\n $categoryKey: String\n $countKey: String\n $mapBounds: MapBounds\n ) {\n statisticsForChoropleth(\n statsConfig: $config\n categoryKey: $categoryKey\n countKey: $countKey\n mapBounds: $mapBounds\n ) {\n label\n gss\n count\n formattedCount\n category\n row\n columns\n gssArea {\n point {\n type\n geometry {\n type\n coordinates\n }\n }\n }\n }\n }\n"): (typeof documents)["\n query Statistics(\n $config: StatisticsConfig!\n $categoryKey: String\n $countKey: String\n $mapBounds: MapBounds\n ) {\n statisticsForChoropleth(\n statsConfig: $config\n categoryKey: $categoryKey\n countKey: $countKey\n mapBounds: $mapBounds\n ) {\n label\n gss\n count\n formattedCount\n category\n row\n columns\n gssArea {\n point {\n type\n geometry {\n type\n coordinates\n }\n }\n }\n }\n }\n"]; /** * The gql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ diff --git a/nextjs/src/__generated__/graphql.ts b/nextjs/src/__generated__/graphql.ts index 4440c5b21..e74e6ab50 100644 --- a/nextjs/src/__generated__/graphql.ts +++ b/nextjs/src/__generated__/graphql.ts @@ -2349,7 +2349,7 @@ export type Query = { sharedDataSource: SharedDataSource; sharedDataSources: Array; statistics?: Maybe>; - statisticsForChoropleth?: Maybe>; + statisticsForChoropleth: Array; testDataSource: ExternalDataSource; }; @@ -2528,6 +2528,7 @@ export type QueryStatisticsArgs = { export type QueryStatisticsForChoroplethArgs = { categoryKey?: InputMaybe; countKey?: InputMaybe; + mapBounds?: InputMaybe; statsConfig: StatisticsConfig; }; @@ -2776,7 +2777,6 @@ export type StatisticsConfig = { groupByArea?: InputMaybe; groupByColumns?: InputMaybe>; gssCodes?: InputMaybe>; - mapBounds?: InputMaybe; preGroupByCalculatedColumns?: InputMaybe>; returnColumns?: InputMaybe>; sourceIds: Array; @@ -3563,10 +3563,11 @@ export type StatisticsQueryVariables = Exact<{ config: StatisticsConfig; categoryKey?: InputMaybe; countKey?: InputMaybe; + mapBounds?: InputMaybe; }>; -export type StatisticsQuery = { __typename?: 'Query', statisticsForChoropleth?: Array<{ __typename?: 'GroupedDataCount', label?: string | null, gss?: string | null, count?: number | null, formattedCount?: string | null, category?: string | null, row?: any | null, columns?: Array | null, gssArea?: { __typename?: 'Area', point?: { __typename?: 'PointFeature', type: GeoJsonTypes, geometry: { __typename?: 'PointGeometry', type: GeoJsonTypes, coordinates: Array } } | null } | null }> | null }; +export type StatisticsQuery = { __typename?: 'Query', statisticsForChoropleth: Array<{ __typename?: 'GroupedDataCount', label?: string | null, gss?: string | null, count?: number | null, formattedCount?: string | null, category?: string | null, row?: any | null, columns?: Array | null, gssArea?: { __typename?: 'Area', point?: { __typename?: 'PointFeature', type: GeoJsonTypes, geometry: { __typename?: 'PointGeometry', type: GeoJsonTypes, coordinates: Array } } | null } | null }> }; export type SourceMetadataQueryVariables = Exact<{ sourceId: Scalars['ID']['input']; @@ -3799,7 +3800,7 @@ export const UpdateMapReportDocument = {"kind":"Document","definitions":[{"kind" export const DeleteMapReportDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteMapReport"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"IDObject"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteMapReport"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"data"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]} as unknown as DocumentNode; export const GetMapReportNameDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetMapReportName"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"mapReport"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]} as unknown as DocumentNode; export const SourceStatsByBoundaryDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"SourceStatsByBoundary"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"sourceId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"analyticalAreaType"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"AnalyticalAreaType"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"mode"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"ChoroplethMode"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"field"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"formula"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"mapBounds"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"MapBounds"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"choroplethDataForSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"sourceId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"sourceId"}}},{"kind":"Argument","name":{"kind":"Name","value":"analyticalAreaKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"analyticalAreaType"}}},{"kind":"Argument","name":{"kind":"Name","value":"mode"},"value":{"kind":"Variable","name":{"kind":"Name","value":"mode"}}},{"kind":"Argument","name":{"kind":"Name","value":"field"},"value":{"kind":"Variable","name":{"kind":"Name","value":"field"}}},{"kind":"Argument","name":{"kind":"Name","value":"formula"},"value":{"kind":"Variable","name":{"kind":"Name","value":"formula"}}},{"kind":"Argument","name":{"kind":"Name","value":"mapBounds"},"value":{"kind":"Variable","name":{"kind":"Name","value":"mapBounds"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"gss"}},{"kind":"Field","name":{"kind":"Name","value":"count"}},{"kind":"Field","name":{"kind":"Name","value":"formattedCount"}},{"kind":"Field","name":{"kind":"Name","value":"category"}},{"kind":"Field","name":{"kind":"Name","value":"row"}},{"kind":"Field","name":{"kind":"Name","value":"columns"}},{"kind":"Field","name":{"kind":"Name","value":"gssArea"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"point"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"geometry"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"coordinates"}}]}}]}}]}}]}}]}}]} as unknown as DocumentNode; -export const StatisticsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"Statistics"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"config"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"StatisticsConfig"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"categoryKey"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"countKey"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"statisticsForChoropleth"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"statsConfig"},"value":{"kind":"Variable","name":{"kind":"Name","value":"config"}}},{"kind":"Argument","name":{"kind":"Name","value":"categoryKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"categoryKey"}}},{"kind":"Argument","name":{"kind":"Name","value":"countKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"countKey"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"gss"}},{"kind":"Field","name":{"kind":"Name","value":"count"}},{"kind":"Field","name":{"kind":"Name","value":"formattedCount"}},{"kind":"Field","name":{"kind":"Name","value":"category"}},{"kind":"Field","name":{"kind":"Name","value":"row"}},{"kind":"Field","name":{"kind":"Name","value":"columns"}},{"kind":"Field","name":{"kind":"Name","value":"gssArea"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"point"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"geometry"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"coordinates"}}]}}]}}]}}]}}]}}]} as unknown as DocumentNode; +export const StatisticsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"Statistics"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"config"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"StatisticsConfig"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"categoryKey"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"countKey"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"mapBounds"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"MapBounds"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"statisticsForChoropleth"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"statsConfig"},"value":{"kind":"Variable","name":{"kind":"Name","value":"config"}}},{"kind":"Argument","name":{"kind":"Name","value":"categoryKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"categoryKey"}}},{"kind":"Argument","name":{"kind":"Name","value":"countKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"countKey"}}},{"kind":"Argument","name":{"kind":"Name","value":"mapBounds"},"value":{"kind":"Variable","name":{"kind":"Name","value":"mapBounds"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"label"}},{"kind":"Field","name":{"kind":"Name","value":"gss"}},{"kind":"Field","name":{"kind":"Name","value":"count"}},{"kind":"Field","name":{"kind":"Name","value":"formattedCount"}},{"kind":"Field","name":{"kind":"Name","value":"category"}},{"kind":"Field","name":{"kind":"Name","value":"row"}},{"kind":"Field","name":{"kind":"Name","value":"columns"}},{"kind":"Field","name":{"kind":"Name","value":"gssArea"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"point"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"geometry"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"type"}},{"kind":"Field","name":{"kind":"Name","value":"coordinates"}}]}}]}}]}}]}}]}}]} as unknown as DocumentNode; export const SourceMetadataDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"SourceMetadata"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"sourceId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"sourceId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"fieldDefinitions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalId"}},{"kind":"Field","name":{"kind":"Name","value":"value"}},{"kind":"Field","name":{"kind":"Name","value":"label"}}]}}]}}]}}]} as unknown as DocumentNode; export const WebhookRefreshDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"WebhookRefresh"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"refreshWebhooks"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"externalDataSourceId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"hasWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"webhookHealthcheck"}}]}}]}}]} as unknown as DocumentNode; export const ExternalDataSourceExternalDataSourceCardDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ExternalDataSourceExternalDataSourceCard"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"ID"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"externalDataSource"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pk"},"value":{"kind":"Variable","name":{"kind":"Name","value":"ID"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"DataSourceCard"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"DataSourceCard"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ExternalDataSource"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"dataType"}},{"kind":"Field","name":{"kind":"Name","value":"crmType"}},{"kind":"Field","name":{"kind":"Name","value":"automatedWebhooks"}},{"kind":"Field","name":{"kind":"Name","value":"autoImportEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"autoUpdateEnabled"}},{"kind":"Field","name":{"kind":"Name","value":"updateMapping"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"sourcePath"}},{"kind":"Field","name":{"kind":"Name","value":"destinationColumn"}}]}},{"kind":"Field","name":{"kind":"Name","value":"jobs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"pagination"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"limit"},"value":{"kind":"IntValue","value":"10"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lastEventAt"}},{"kind":"Field","name":{"kind":"Name","value":"status"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sharingPermissions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"organisation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode; diff --git a/nextjs/src/__generated__/zodSchema.ts b/nextjs/src/__generated__/zodSchema.ts index b039fd6de..29009140d 100644 --- a/nextjs/src/__generated__/zodSchema.ts +++ b/nextjs/src/__generated__/zodSchema.ts @@ -481,7 +481,6 @@ export function StatisticsConfigSchema(): z.ZodObject { if (activeTileset.useBoundsInDataQuery) { - fetchMore({ - variables: { mapBounds }, - }) + fetchMore({ variables: { mapBounds } }) } }, [mapBounds, mapView.mapOptions, activeTileset]) diff --git a/nextjs/src/app/reports/[id]/useDataByBoundary.ts b/nextjs/src/app/reports/[id]/useDataByBoundary.ts index 89f5bbc40..2879983a1 100644 --- a/nextjs/src/app/reports/[id]/useDataByBoundary.ts +++ b/nextjs/src/app/reports/[id]/useDataByBoundary.ts @@ -43,9 +43,6 @@ const useDataByBoundary = ({ getSourceFieldNames?: boolean }) => { const report = useReport() - const sourceId = report.report.layers.find( - (l) => l.id === view?.mapOptions.choropleth.layerId - )?.source // If mapBounds is required, send dummy empty bounds on the first request // This is required for fetchMore() to work, which is used to add data to @@ -60,6 +57,11 @@ const useDataByBoundary = ({ const { useAdvancedStatistics, advancedStatisticsConfig } = view?.mapOptions?.choropleth || {} + const sourceId = + report.report.layers.find( + (l) => l.id === view?.mapOptions.choropleth.layerId + )?.source || advancedStatisticsConfig?.sourceIds[0] + const query = useQuery< SourceStatsByBoundaryQuery, SourceStatsByBoundaryQueryVariables @@ -102,7 +104,6 @@ const useDataByBoundary = ({ config: { ...(advancedStatisticsConfig! || {}), groupByArea: analyticalAreaType!, - mapBounds, returnColumns: ['gss', 'label'] .concat( view?.mapOptions?.choropleth.dataType === @@ -125,11 +126,13 @@ const useDataByBoundary = ({ // [] // ), }, + mapBounds, }, skip: !useAdvancedStatistics || !advancedStatisticsConfig || !analyticalAreaType, + notifyOnNetworkStatusChange: true, // required to mark loading: true on fetchMore() } ) @@ -196,11 +199,13 @@ export const STATISTICS_QUERY = gql` $config: StatisticsConfig! $categoryKey: String $countKey: String + $mapBounds: MapBounds ) { statisticsForChoropleth( statsConfig: $config categoryKey: $categoryKey countKey: $countKey + mapBounds: $mapBounds ) { label gss diff --git a/nextjs/src/components/ApolloWrapper.tsx b/nextjs/src/components/ApolloWrapper.tsx index d25f3b18b..80c9836eb 100644 --- a/nextjs/src/components/ApolloWrapper.tsx +++ b/nextjs/src/components/ApolloWrapper.tsx @@ -8,7 +8,10 @@ import { NextSSRInMemoryCache, } from '@apollo/experimental-nextjs-app-support/ssr' -import { SourceStatsByBoundaryQuery } from '@/__generated__/graphql' +import { + SourceStatsByBoundaryQuery, + StatisticsQuery, +} from '@/__generated__/graphql' import { GRAPHQL_URL } from '@/env' import { authenticationHeaders } from '@/lib/auth' @@ -82,6 +85,32 @@ export function makeFrontEndClient() { return Object.values(dataByGss) }, }, + statisticsForChoropleth: { + // Use all argument values except mapBounds + // so results for different areas are merged + keyArgs: (_args) => { + const args = { ..._args } + delete args.mapBounds + let fullKey = '' + for (const key of Object.keys(args)) { + const value = args[key] + fullKey += `${key}:${JSON.stringify(value)};` + } + return fullKey + }, + merge(existing = [], incoming = []) { + // Deduplicate data + const dataByGss: Record< + string, + StatisticsQuery['statisticsForChoropleth'][0] + > = {} + const allData = [...existing, ...incoming] + for (const d of allData) { + dataByGss[d.gss || ''] = d + } + return Object.values(dataByGss) + }, + }, }, }, }, diff --git a/utils/geo_reference.py b/utils/geo_reference.py index 70cf4b991..0b4b7ca59 100644 --- a/utils/geo_reference.py +++ b/utils/geo_reference.py @@ -26,6 +26,9 @@ class AnalyticalAreaType(Enum): output_area = "output_area" +# Note: this should not be used directly, due to mismatch between LIH +# and postcodes.io area categorization. Instead, use area_to_postcode_io_key() +# and stats.postcodeIOKeyAreaTypeLookup. lih_to_postcodes_io_key_map = { "WMC": AnalyticalAreaType.parliamentary_constituency, "WMC23": AnalyticalAreaType.parliamentary_constituency_2024, @@ -39,7 +42,7 @@ class AnalyticalAreaType(Enum): } -def area_to_postcode_io_filter(area: "Area"): +def area_to_postcode_io_key(area: "Area"): if area.mapit_type in ["LBO", "UTA", "COI", "LGD", "MTD", "DIS", "NMD"]: return AnalyticalAreaType.admin_district elif area.mapit_type in ["CTY"]: