From 0da5bb8e96c81c846b25e95dd56258966be31182 Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Sun, 4 Aug 2024 13:53:37 -0700 Subject: [PATCH] make data sources more flexible, add tiles script and download supporting datasets --- README.md | 2 +- harvest_restrictions.py | 229 ++++++++------- source.schema.json | 56 ++-- sources.json | 535 ++++++++++++++--------------------- sql/tiles.sql | 75 +++++ test_harvest_restrictions.py | 61 ++-- 6 files changed, 475 insertions(+), 483 deletions(-) create mode 100644 sql/tiles.sql diff --git a/README.md b/README.md index 36631c6..5259610 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,4 @@ [![Lifecycle:Experimental](https://img.shields.io/badge/Lifecycle-Experimental-339999)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md) -Extract data from provincial inventories to quantify how much of British Columbia's provincially administered forested land is under some type of forest harvest management regime. \ No newline at end of file +Quantify how much of British Columbia's provincially administered forested land is under some type of forest harvest management regime. \ No newline at end of file diff --git a/harvest_restrictions.py b/harvest_restrictions.py index 6e85f6a..773a4ce 100644 --- a/harvest_restrictions.py +++ b/harvest_restrictions.py @@ -62,29 +62,33 @@ def parse_sources(sources): return parsed -def validate_file(layer): +def validate_file(source): """simple validation of file based sources - file exists - schema is as expected """ - source = layer["source"] - alias = layer["alias"] - query = layer["query"] - # load file df = geopandas.read_file( - os.path.expandvars(source), layer=layer["layer"], where=query + os.path.expandvars(source["source"]), + layer=source["layer"], + where=source["query"], ) # are expected columns present? - df.columns = [x.lower() for x in df.columns] - for col in ["primary_key", "name_column"]: - if layer[col]: - column = layer[col].lower() - if column not in df.columns: - raise ValueError(f"{alias} - {column} is not present in source") + columns = [x.lower() for x in df.columns] + if source["primary_key"]: + if source["primary_key"] not in columns: + raise ValueError( + f"Primary key - {source['primary_key']} is not present in source" + ) + for column in source["field_mapper"].values(): + if column.lower() not in columns: + raise ValueError( + f"Column {column} is not present in source, modify config 'field_mapper'" + ) # is there data? + alias = source["alias"] if len(df.index) == 0: raise ValueError(f"{alias} - no data returned for given source and query") @@ -92,30 +96,41 @@ def validate_file(layer): LOG.info(f"{alias} - validates successfully") -def validate_bcgw(layer): +def validate_bcgw(source): """validate bcdata sources against bcdc api and wfs""" # does source exist as written? - alias = layer["alias"] - table = layer["source"].upper() - query = layer["query"] + alias = source["alias"] + table = source["source"].upper() + query = source["query"] if table not in bcdata.list_tables(): raise ValueError( f"{alias} - {table} does not exist in BCGW or is not available via WFS" ) - # do specified name/pk columns exist in source? + # get columns present in source from data catalogue table_def = bcdata.get_table_definition(table) - for col in ["primary_key", "name_column"]: - if layer[col]: - column = layer[col].upper() - if column not in [c["column_name"] for c in table_def["schema"]]: + columns = [c["column_name"] for c in table_def["schema"]] + + # is primary key present? + if source["primary_key"]: + if source["primary_key"] not in columns: + raise ValueError( + f"Primary key - {source['primary_key']} is not present in source {table}" + ) + + # are other required columns in field mapping present? + for column in source["field_mapper"].values(): + if ( + column + ): # allow null source columns (adds the new column, but with no values from source) + if column.upper() not in columns: raise ValueError( - f"{alias} - {column} is not present in BCGW table {table}" + f"Column {column} is not present in source, modify config 'field_mapper'" ) # does query return values? - if layer["query"]: - if bcdata.get_count(table, query=layer["query"]) == 0: + if source["query"]: + if bcdata.get_count(table, query=source["query"]) == 0: raise ValueError( f"{alias} - provided query {query} returns no data for {table}" ) @@ -149,11 +164,11 @@ def validate_sources(sources, validate_data=True, alias=None): Validate json, whether data sources exist, and assign hierarchy index based on position in list """ - for layer in sources: - if layer["source_type"] == "BCGW": - validate_bcgw(layer) - elif layer["source_type"] == "FILE": - validate_file(layer) + for source in sources: + if source["source_type"] == "BCGW": + validate_bcgw(source) + elif source["source_type"] == "FILE": + validate_file(source) LOG.info("Validation successful - all layers appear valid") @@ -161,28 +176,31 @@ def validate_sources(sources, validate_data=True, alias=None): return sources -def download_source(layer, out_path="data", out_table="designations_cleaned"): - """download layer from source and save to parquet in out_path""" - - table, alias = (layer["source"], layer["alias"]) +def download(source): + """download layer from source to a standardized geodataframe""" # download WFS - if layer["source_type"] == "BCGW": + if source["source_type"] == "BCGW": df = bcdata.get_data( - table, + source["source"], crs="EPSG:3005", - query=layer["query"], + query=source["query"], as_gdf=True, lowercase=True, ) # download file - elif layer["source_type"] == "FILE": + elif source["source_type"] == "FILE": df = geopandas.read_file( - os.path.expandvars(layer["source"]), - layer=layer["layer"], - where=layer["query"], + os.path.expandvars(source["source"]), + layer=source["layer"], + where=source["query"], ) + if not df.crs: + raise ValueError( + "Source does not have a defined projection/coordinate reference system" + ) + # reproject to BC Albers if necessary if df.crs != CRS.from_user_input(3005): df = df.to_crs("EPSG:3005") # lowercasify column names @@ -192,74 +210,46 @@ def download_source(layer, out_path="data", out_table="designations_cleaned"): df = df.rename_geometry("geom") df = to_multipart(df) # sources can have mixed types, just make everything multi - # add new columns, prefixing and suffixing with "__" to avoid collisions - df["__index__"] = layer["index"] - df["__designation__"] = layer["designation"] - df["__alias__"] = layer["alias"].lower() - df["__harvest_restriction__"] = layer["harvest_restriction"] - df["__og_restriction__"] = layer["og_restriction"] - df["__mining_restriction__"] = layer["mining_restriction"] - # load pk/name if present, otherwise set to empty string - if layer["primary_key"]: - df["__primary_key__"] = layer["primary_key"] + # standardize columns, adding data as required + df["__index__"] = source["index"] + df["__description__"] = source["description"] + df["__alias__"] = source["alias"].lower() + if source["primary_key"]: + df["__primary_key__"] = source["primary_key"] else: df["__primary_key__"] = "" - if layer["name_column"]: - df["__name__"] = df[layer["name_column"].lower()] - else: - df["__name__"] = "" - - # retain only columns of interest - df = df[ - [ - "__index__", - "__designation__", - "__alias__", - "__harvest_restriction__", - "__og_restriction__", - "__mining_restriction__", - "__primary_key__", - "__name__", - "geom", - ] - ] - # rename remaining columns - df = df.rename( - columns={ - "__index__": "index", - "__designation__": "designation", - "__alias__": "alias", - "__harvest_restriction__": "harvest_restriction", - "__og_restriction__": "og_restriction", - "__mining_restriction__": "mining_restriction", - "__primary_key__": "source_primary_key", - "__name__": "source_name", - } + + # rename columns that we want to retain + for key, value in source["field_mapper"].items(): + if value: + df["__" + key + "__"] = df[ + value.lower() + ] # all incoming data is already lowercasified + else: + df["__" + key + "__"] = None + + # add additional constant data + if source["data"]: + for key, value in source["data"].items(): + df["__" + key + "__"] = value + + # retain only columns that have just been added + columns = ( + ["index", "description", "alias", "primary_key"] + + list(source["field_mapper"]) + + list(source["data"]) ) + df = df[["__" + c + "__" for c in columns] + ["geom"]] - # dump to file if out_path specified - if out_path: - out_file = os.path.join( - out_path, - ( - "rr_" - + str(layer["index"]).zfill(2) - + "_" - + layer["alias"].lower() - + ".parquet" - ), - ) - LOG.info(f"Writing {alias} to {out_file}") - df.to_parquet(out_file) + # strip the __ prefix/suffix + df = df.rename(columns={"__" + c + "__": c for c in columns}) - # load to postgres, writing everything to the same initial table - LOG.info(f"Writing {alias} to postgres") - df.to_postgis(out_table, DB, if_exists="append") + return df -def process(out_path): - """clean and overlay input data, dump to file if specified""" - DB.execute("create table designations_cleaned ") +# def process(out_path): +# """clean and overlay input data, dump to file if specified""" +# DB.execute("create table designations_cleaned ") @click.group() @@ -317,7 +307,7 @@ def validate(alias, sources_file, verbose, quiet): ) @verbose_opt @quiet_opt -def download(alias, sources_file, out_path, no_validate, verbose, quiet): +def setup(alias, sources_file, out_path, no_validate, verbose, quiet): configure_logging((verbose - quiet)) # load sources file @@ -334,10 +324,37 @@ def download(alias, sources_file, out_path, no_validate, verbose, quiet): sources = validate_sources(sources) # download each data source - for layer in sources: - download_source(layer, out_path) - - # download supporting datasets + for source in sources: + df = download(source) + + # load to postgres, writing everything to the same initial table + LOG.info(f"Writing {source['alias']} to postgres") + df.to_postgis("restrictions_source", DB, if_exists="append") + + # dump to file if out_path specified + if out_path: + out_file = os.path.join( + out_path, + ( + "rr_" + + str(source["index"]).zfill(2) + + "_" + + source["alias"].lower() + + ".parquet" + ), + ) + LOG.info(f"Writing {alias} to {out_file}") + df.to_parquet(out_file) + + # download additional supporting datasets + if not alias: + for table in [ + "WHSE_BASEMAPPING.BCGS_20K_GRID", + "WHSE_WILDLIFE_MANAGEMENT.CRIMS_MARINE_ECOSECTION", + "WHSE_LEGAL_ADMIN_BOUNDARIES.ABMS_PROVINCE_SP", + "WHSE_BASEMAPPING.NTS_250K_GRID", + ]: + bcdata.bc2pg(table, os.environ["DATABASE_URL"]) if __name__ == "__main__": diff --git a/source.schema.json b/source.schema.json index 3241b5f..555a7bb 100644 --- a/source.schema.json +++ b/source.schema.json @@ -8,28 +8,13 @@ "type": "object", "properties": { "alias": { - "description": "Unique (lowercase, _ separated) slug used to name/identify the data source when processing", + "description": "Key to uniquely identify the data source when processing (lower case, _ separated)", "type": "string" }, - "designation": { - "description": "Description of the land designation / restriction", + "description": { + "description": "Description of the data source as defined", "type": "string" }, - "harvest_restriction": { - "description": "Forest harvesting restriction level associated with the designation", - "type": "integer", - "enum": [1, 2, 3, 4, 5, 6] - }, - "og_restriction": { - "description": "Oil and gas restriction level associated with the designation", - "type": "integer", - "enum": [0, 1, 2, 3, 4, 5, 6] - }, - "mining_restriction": { - "description": "Mining restriction level associated with the designation", - "type": "integer", - "enum": [0, 1, 2, 3, 4, 5, 6] - }, "source_type": { "description": "Describes whether source is to be downloaded from BCGW (ie WFS using bcdata) or from FILE", "type": "string", @@ -43,19 +28,40 @@ "description": "Layer to use within a file based source", "type": "string" }, + "query": { + "description": "Query to subset data in source/layer - ECQL for BCGW sources, OGR SQL for files", + "type": ["string", "null"] + }, "primary_key": { "description": "The column holding primary key for source", "type": ["string", "null"] }, - "name_column": { - "description": "Text column in source/layer to use for restriction_name", - "type": ["string", "null"] + "field mapper": { + "description": "Mapping of source column names to new column names", + "type": "object" }, - "query": { - "description": "Query to subset data in source/layer - ECQL for BCGW sources, OGR SQL for files", - "type": ["string", "null"] + "data": { + "description": "Additional data to add to all records of the source, as key(column):value pair", + "type": "object", + "properties": { + "harvest_restriction": { + "description": "Forest harvesting restriction level associated with the designation", + "type": "integer", + "enum": [1, 2, 3, 4, 5, 6] + }, + "og_restriction": { + "description": "Oil and gas restriction level associated with the designation", + "type": "integer", + "enum": [0, 1, 2, 3, 4, 5, 6] + }, + "mining_restriction": { + "description": "Mining restriction level associated with the designation", + "type": "integer", + "enum": [0, 1, 2, 3, 4, 5, 6] + } + } } }, - "required": ["alias", "designation", "harvest_restriction", "og_restriction", "mining_restriction", "source_type", "source", "name_column", "query"] + "required": ["alias", "description", "source_type", "source", "query", "primary_key", "field_mapper", "data"] } } \ No newline at end of file diff --git a/sources.json b/sources.json index 201817f..bd03ab6 100644 --- a/sources.json +++ b/sources.json @@ -1,585 +1,488 @@ [ { "alias": "park_national", - "designation": "National Park", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_ADMIN_BOUNDARIES.CLAB_NATIONAL_PARKS", + "description": "National Park", "source_type": "BCGW", + "source": "WHSE_ADMIN_BOUNDARIES.CLAB_NATIONAL_PARKS", "primary_key": null, - "name_column": "ENGLISH_NAME", - "query": null + "query": null, + "field_mapper": {"name": "ENGLISH_NAME"}, + "data": {"harvest_restriction": 1, "og_restriction": 0, "mining_restriction": 0} }, - { - "alias": "park_er", - "designation": "Ecological Reserve", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_TANTALIS.TA_PARK_ECORES_PA_SVW", + { "alias": "park_er", "source_type": "BCGW", + "description": "Ecological Reserve", + "source": "WHSE_TANTALIS.TA_PARK_ECORES_PA_SVW", "primary_key": null, - "name_column": "PROTECTED_LANDS_NAME", - "query": "PROTECTED_LANDS_DESIGNATION = 'ECOLOGICAL RESERVE'" + "query": "PROTECTED_LANDS_DESIGNATION = 'ECOLOGICAL RESERVE'", + "field_mapper": {"name": "PROTECTED_LANDS_NAME"}, + "data": {"harvest_restriction": 1, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "park_provincial", - "designation": "Provincial Park", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_TANTALIS.TA_PARK_ECORES_PA_SVW", "source_type": "BCGW", + "description": "Provincial Park", + "source": "WHSE_TANTALIS.TA_PARK_ECORES_PA_SVW", "primary_key": null, - "name_column": "PROTECTED_LANDS_NAME", - "query": "PROTECTED_LANDS_DESIGNATION = 'PROVINCIAL PARK'" + "query": "PROTECTED_LANDS_DESIGNATION = 'PROVINCIAL PARK'", + "field_mapper": {"name": "PROTECTED_LANDS_NAME"}, + "data": {"harvest_restriction": 1, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "park_conservancy", - "designation": "Conservancy", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_TANTALIS.TA_CONSERVANCY_AREAS_SVW", "source_type": "BCGW", + "description": "Conservancy", + "source": "WHSE_TANTALIS.TA_CONSERVANCY_AREAS_SVW", "primary_key": null, - "name_column": "CONSERVANCY_AREA_NAME", - "query": null + "query": null, + "field_mapper": {"name": "CONSERVANCY_AREA_NAME"}, + "data": {"harvest_restriction": 1, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "park_protectedarea", - "designation": "Protected Area", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_TANTALIS.TA_PARK_ECORES_PA_SVW", "source_type": "BCGW", + "description": "Protected Area", + "source": "WHSE_TANTALIS.TA_PARK_ECORES_PA_SVW", "primary_key": null, - "name_column": "PROTECTED_LANDS_NAME", - "query": "PROTECTED_LANDS_DESIGNATION = 'PROTECTED AREA'" + "query": "PROTECTED_LANDS_DESIGNATION = 'PROTECTED AREA'", + "field_mapper": {"name": "PROTECTED_LANDS_NAME"}, + "data": {"harvest_restriction": 1, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "heritage_site", - "designation": "Heritage Site", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_HUMAN_CULTURAL_ECONOMIC.HIST_HISTORIC_ENVIRONMENTS_SP", "source_type": "BCGW", + "description": "Heritage Site", + "source": "WHSE_HUMAN_CULTURAL_ECONOMIC.HIST_HISTORIC_ENVIRONMENTS_SP", "primary_key": null, - "name_column": "COMMON_SITE_NAME", - "query": null + "query": null, + "field_mapper": {"name": "COMMON_SITE_NAME"}, + "data": {"harvest_restriction": 1, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "park_recreationarea", - "designation": "Recreation Area", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_TANTALIS.TA_PARK_ECORES_PA_SVW", "source_type": "BCGW", + "description": "Recreation Area", + "source": "WHSE_TANTALIS.TA_PARK_ECORES_PA_SVW", "primary_key": null, - "name_column": "PROTECTED_LANDS_NAME", - "query": "PROTECTED_LANDS_DESIGNATION = 'RECREATION AREA'" + "query": "PROTECTED_LANDS_DESIGNATION = 'RECREATION AREA'", + "field_mapper": {"name": "PROTECTED_LANDS_NAME"}, + "data": {"harvest_restriction": 1, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "park_regional", - "designation": "Regional Park", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_BASEMAPPING.GBA_LOCAL_REG_GREENSPACES_SP", "source_type": "BCGW", + "description": "Regional Park", + "source": "WHSE_BASEMAPPING.GBA_LOCAL_REG_GREENSPACES_SP", "primary_key": null, - "name_column": "PARK_NAME", - "query": "PARK_TYPE LIKE 'Regional%'" + "query": "PARK_TYPE LIKE 'Regional%'", + "field_mapper": {"name": "PARK_NAME"}, + "data": {"harvest_restriction": 1, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "uwr_no_harvest", - "designation": "Ungulate Winter Range: No harvest", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_WILDLIFE_MANAGEMENT.WCP_UNGULATE_WINTER_RANGE_SP", "source_type": "BCGW", + "description": "Ungulate Winter Range: No harvest", + "source": "WHSE_WILDLIFE_MANAGEMENT.WCP_UNGULATE_WINTER_RANGE_SP", "primary_key": null, - "name_column": "UWR_NUMBER", - "query": "TIMBER_HARVEST_CODE = 'NO HARVEST ZONE'" + "query": "TIMBER_HARVEST_CODE = 'NO HARVEST ZONE'", + "field_mapper": {"name": "UWR_NUMBER"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "wha_no_harvest", - "designation": "Wildlife Habitat Area: No harvest", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_WILDLIFE_MANAGEMENT.WCP_WILDLIFE_HABITAT_AREA_POLY", "source_type": "BCGW", + "description": "Wildlife Habitat Area: No harvest", + "source": "WHSE_WILDLIFE_MANAGEMENT.WCP_WILDLIFE_HABITAT_AREA_POLY", "primary_key": null, - "name_column": "COMMON_SPECIES_NAME", - "query": "(TIMBER_HARVEST_CODE = 'NO HARVEST ZONE' AND FEATURE_NOTES NOT LIKE '%not a legal boundary%') OR(TIMBER_HARVEST_CODE = 'NO HARVEST ZONE' AND FEATURE_NOTES IS NULL)" + "query": "(TIMBER_HARVEST_CODE = 'NO HARVEST ZONE' AND FEATURE_NOTES NOT LIKE '%not a legal boundary%') OR(TIMBER_HARVEST_CODE = 'NO HARVEST ZONE' AND FEATURE_NOTES IS NULL)", + "field_mapper": {"name": "COMMON_SPECIES_NAME"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "biodiv_mining_tourism_areas", - "designation": "Biodiversity Mining and Tourism Area", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "source_type": "BCGW", + "description": "Biodiversity Mining and Tourism Area", + "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "primary_key": null, - "name_column": "LEGAL_FEAT_ATRB_1_VALUE", - "query": "LEGAL_FEAT_OBJECTIVE = 'Biodiversity, Mining and Tourism Areas (BMTA)'" + "query": "LEGAL_FEAT_OBJECTIVE = 'Biodiversity, Mining and Tourism Areas (BMTA)'", + "field_mapper": {"name": "LEGAL_FEAT_ATRB_1_VALUE"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "sea_to_sky_wildland", - "designation": "Sea to Sky Wildland Zone", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "source_type": "BCGW", + "description": "Sea to Sky Wildland Zone", + "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "primary_key": null, - "name_column": "LEGAL_FEAT_ATRB_4_VALUE", - "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Sea to Sky Land and Resource Management Plan' AND LEGAL_FEAT_OBJECTIVE = 'Wildland Area'" + "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Sea to Sky Land and Resource Management Plan' AND LEGAL_FEAT_OBJECTIVE = 'Wildland Area'", + "field_mapper": {"name": "LEGAL_FEAT_ATRB_4_VALUE"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "mk_wildland", - "designation": "Special Wildland RMZ in Muskwa Kechia MA", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_NON_LEGAL_POLY_SVW", "source_type": "BCGW", + "description": "Special Wildland RMZ in Muskwa Kechia MA", + "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_NON_LEGAL_POLY_SVW", "primary_key": null, - "name_column": "NON_LEGAL_FEAT_ATRB_4_VALUE", - "query": "NON_LEGAL_FEAT_ID IN (65488,65490,65491,65492,65497,65501,65508)" + "query": "NON_LEGAL_FEAT_ID IN (65488,65490,65491,65492,65497,65501,65508)", + "field_mapper": {"name": "NON_LEGAL_FEAT_ATRB_4_VALUE"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "n_c_wildland", - "designation": "Nlhaxten/Cayoosh Wildland Area", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "source_type": "BCGW", + "description": "Nlhaxten/Cayoosh Wildland Area", + "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "primary_key": null, - "name_column": "LEGAL_FEAT_ATRB_4_VALUE", - "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Lillooet Land and Resource Management Plan' AND LEGAL_FEAT_OBJECTIVE = 'Wildland Area'" + "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Lillooet Land and Resource Management Plan' AND LEGAL_FEAT_OBJECTIVE = 'Wildland Area'", + "field_mapper": {"name": "LEGAL_FEAT_ATRB_4_VALUE"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "s_cayoosh_wildland", - "designation": "South Chilcotin Mountains Mining and Tourism Areas", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "source_type": "BCGW", + "description": "South Chilcotin Mountains Mining and Tourism Areas", + "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "primary_key": null, - "name_column": "LEGAL_FEAT_ATRB_4_VALUE", - "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Lillooet Land and Resource Management Plan' AND LEGAL_FEAT_OBJECTIVE = 'Mining and Tourism Area'" + "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Lillooet Land and Resource Management Plan' AND LEGAL_FEAT_OBJECTIVE = 'Mining and Tourism Area'", + "field_mapper": {"name": "LEGAL_FEAT_ATRB_4_VALUE"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "ngo_fee_simple", - "designation": "NGO Fee Simple Conservation Lands", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LEGAL_ADMIN_BOUNDARIES.WCL_CONSERVATION_AREAS_NGO_SP", "source_type": "BCGW", + "description": "NGO Fee Simple Conservation Lands", + "source": "WHSE_LEGAL_ADMIN_BOUNDARIES.WCL_CONSERVATION_AREAS_NGO_SP", "primary_key": null, - "name_column": "CONSERVATION_AREAS_NGO_ID", - "query": null + "query": null, + "field_mapper": {"name": "CONSERVATION_AREAS_NGO_ID"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "great_bear_sfma", - "designation": "Great Bear Rainforest (GBR) Special Forest Management Areas", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_ADMIN_BOUNDARIES.FADM_SPECIAL_PROTECTION_AREA", "source_type": "BCGW", + "description": "Great Bear Rainforest (GBR) Special Forest Management Areas", + "source": "WHSE_ADMIN_BOUNDARIES.FADM_SPECIAL_PROTECTION_AREA", "primary_key": null, - "name_column": "PROTECTED_AREA_TYPE", - "query": "PROTECTED_AREA_TYPE = 'SFMA'" + "query": "PROTECTED_AREA_TYPE = 'SFMA'", + "field_mapper": {"name": "PROTECTED_AREA_TYPE"}, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "great_bear_lrd", - "designation": "Great Bear Rainforest (GBR) Landscape Reserve Design (LRD)", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LAND_USE_PLANNING.RMP_LANDSCAPE_RSRV_DESIGN_SP", "source_type": "BCGW", + "description": "Great Bear Rainforest (GBR) Landscape Reserve Design (LRD)", + "source": "WHSE_LAND_USE_PLANNING.RMP_LANDSCAPE_RSRV_DESIGN_SP", "primary_key": null, - "name_column": "LU_NAME", - "query": "LRD_STATUS = 'current'" + "field_mapper": {"name": "LU_NAME"}, + "query": "LRD_STATUS = 'current'", + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "ogma_legal", - "designation": "Old Growth Management Area (OGMA): Legal", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LAND_USE_PLANNING.RMP_OGMA_LEGAL_CURRENT_SVW", "source_type": "BCGW", + "description": "Old Growth Management Area (OGMA): Legal", + "source": "WHSE_LAND_USE_PLANNING.RMP_OGMA_LEGAL_CURRENT_SVW", "primary_key": null, - "name_column": "LEGAL_OGMA_PROVID", - "query": null + "field_mapper": {"name": "LEGAL_OGMA_PROVID"}, + "query": null, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "great_bear_grizzly_class1", - "designation": "Class1 Coast Grizzly Bear Habitat (Coast LUP)", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Class1 Coast Grizzly Bear Habitat (Coast LUP)", "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "source_type": "BCGW", "primary_key": null, - "name_column": "LEGAL_FEAT_ATRB_1_NAME", - "query": "LEGAL_FEAT_ATRB_1_NAME = 'GRIZZLY_BEAR_HABITAT_CLASS' AND LEGAL_FEAT_ATRB_1_VALUE = '1'" + "field_mapper": {"name": "LEGAL_FEAT_ATRB_1_NAME"}, + "query": "LEGAL_FEAT_ATRB_1_NAME = 'GRIZZLY_BEAR_HABITAT_CLASS' AND LEGAL_FEAT_ATRB_1_VALUE = '1'", + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "ogma_nonlegal", - "designation": "Old Growth Management Area (OGMA): Non-Legal", - "harvest_restriction": 2, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LAND_USE_PLANNING.RMP_OGMA_NON_LEGAL_CURRENT_SVW", "source_type": "BCGW", + "description": "Old Growth Management Area (OGMA): Non-Legal", + "source": "WHSE_LAND_USE_PLANNING.RMP_OGMA_NON_LEGAL_CURRENT_SVW", "primary_key": null, - "name_column": null, - "query": null + "field_mapper": {"name": null}, + "query": null, + "data": {"harvest_restriction": 2, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "private_conservation_lands_admin", - "designation": "Private Conservation Land: Administered Land", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_LEGAL_ADMIN_BOUNDARIES.WCL_CONSERVATION_LANDS_SP", "source_type": "BCGW", + "description": "Private Conservation Land: Administered Land", + "source": "WHSE_LEGAL_ADMIN_BOUNDARIES.WCL_CONSERVATION_LANDS_SP", "primary_key": null, - "name_column": "SITE_NAME", - "query": "CONSERVATION_LAND_TYPE = 'Administered Lands'" + "field_mapper": {"name": "SITE_NAME"}, + "query": "CONSERVATION_LAND_TYPE = 'Administered Lands'", + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "crd_water_supply_area", - "designation": "CRD Water Supply Area", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, - "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/CRD.gdb.zip", "source_type": "FILE", + "description": "CRD Water Supply Area", + "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/CRD.gdb.zip", "layer": "WSA_Boundary", "primary_key": null, - "name_column": "Name", - "query": null + "field_mapper": {"name": "Name"}, + "query": null, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "gvrd_water_supply_area", - "designation": "Greater Vancouver Water District", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, - "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/GVRD_watershed.gdb.zip", "source_type": "FILE", + "description": "Greater Vancouver Water District", + "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/GVRD_watershed.gdb.zip", "layer": "GVRD_watershed", "primary_key": null, - "name_column": "Name", - "query": null + "query": null, + "field_mapper": {"name": "Name"}, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "wildlife_management_area", - "designation": "Wildlife Management Area (WMA)", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_TANTALIS.TA_WILDLIFE_MGMT_AREAS_SVW", "source_type": "BCGW", + "description": "Wildlife Management Area (WMA)", + "source": "WHSE_TANTALIS.TA_WILDLIFE_MGMT_AREAS_SVW", "primary_key": null, - "name_column": "WILDLIFE_MANAGEMENT_AREA_NAME", - "query": null + "query": null, + "field_mapper": {"name": "WILDLIFE_MANAGEMENT_AREA_NAME"}, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "creston_v_wma", - "designation": "Creston Valley Wildlife Management Area", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_TANTALIS.TA_WILDLIFE_MGMT_AREAS_SVW", "source_type": "BCGW", + "description": "Creston Valley Wildlife Management Area", + "source": "WHSE_TANTALIS.TA_WILDLIFE_MGMT_AREAS_SVW", "primary_key": null, - "name_column": "ADMIN_AREA_SID", - "query": "ADMIN_AREA_SID = 5364" + "query": "ADMIN_AREA_SID = 5364", + "field_mapper": {"name": "ADMIN_AREA_SID"}, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "vqo_preserve", - "designation": "Visual Quality Objective (VQO): Preservation", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_FOREST_VEGETATION.REC_VISUAL_LANDSCAPE_INVENTORY", "source_type": "BCGW", + "description": "Visual Quality Objective (VQO): Preservation", + "source": "WHSE_FOREST_VEGETATION.REC_VISUAL_LANDSCAPE_INVENTORY", "primary_key": null, - "name_column": "PROJECT_NAME", - "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_EVQO_CODE = 'P'" + "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_EVQO_CODE = 'P'", + "field_mapper": {"name": "PROJECT_NAME"}, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "vqo_retain", - "designation": "Visual Quality Objective (VQO): Retention", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Visual Quality Objective (VQO): Retention", "source": "WHSE_FOREST_VEGETATION.REC_VISUAL_LANDSCAPE_INVENTORY", "source_type": "BCGW", "primary_key": null, - "name_column": "PROJECT_NAME", - "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_EVQO_CODE = 'R'" + "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_EVQO_CODE = 'R'", + "field_mapper": {"name": "PROJECT_NAME"}, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "designated_area", - "designation": "FADM Designated Area", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, + "description": "FADM Designated Area", "source": "WHSE_ADMIN_BOUNDARIES.FADM_DESIGNATED_AREAS", "source_type": "BCGW", "primary_key": null, - "name_column": "DESIGNATED_AREA_NAME", - "query": "EXPIRED_OR_CANCELLED = 'N'" + "query": "EXPIRED_OR_CANCELLED = 'N'", + "field_mapper": {"name": "DESIGNATED_AREA_NAME"}, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "atlin_taku_fra", - "designation": "Atlin-Taku Forest Retention Areas", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Atlin-Taku Forest Retention Areas", "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "source_type": "BCGW", "primary_key": null, - "name_column": "LEGAL_FEAT_ID", - "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Atlin - Taku Strategic Land and Resource Plan' AND LEGAL_FEAT_OBJECTIVE = 'Forest Retention Area'" + "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Atlin - Taku Strategic Land and Resource Plan' AND LEGAL_FEAT_OBJECTIVE = 'Forest Retention Area'", + "field_mapper": {"name": "LEGAL_FEAT_ID"}, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "migratory_bird_sanctuaries", - "designation": "Migratory Bird Sanctuaries", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Migratory Bird Sanctuaries", "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/CPCAD-BDCAPC_Nov2023.gdb.zip", "source_type": "FILE", "layer": "CPCAD_Nov2023_Albers", "primary_key": null, - "name_column": "PARENT_ID", - "query": "LOC = 2 And BIOME = 'T' And TYPE_E = 'Migratory Bird Sanctuary'" + "query": "LOC = 2 And BIOME = 'T' And TYPE_E = 'Migratory Bird Sanctuary'", + "field_mapper": {"name": "PARENT_ID"}, + "data": {"harvest_restriction": 3, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "private_conservation_lands_reserve", - "designation": "Private Conservation Land: Reserve Land", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Private Conservation Land: Reserve Land", "source": "WHSE_LEGAL_ADMIN_BOUNDARIES.WCL_CONSERVATION_LANDS_SP", "source_type": "BCGW", "primary_key": null, - "name_column": "SITE_NAME", - "query": "CONSERVATION_LAND_TYPE = 'Reserve Lands'" + "query": "CONSERVATION_LAND_TYPE = 'Reserve Lands'", + "field_mapper": {"name": "SITE_NAME"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "uwr_conditional_harvest", - "designation": "Ungulate Winter Range: Conditional harvest", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Ungulate Winter Range: Conditional harvest", "source": "WHSE_WILDLIFE_MANAGEMENT.WCP_UNGULATE_WINTER_RANGE_SP", "source_type": "BCGW", "primary_key": null, - "name_column": "UWR_NUMBER", - "query": "TIMBER_HARVEST_CODE = 'CONDITIONAL HARVEST ZONE'" + "query": "TIMBER_HARVEST_CODE = 'CONDITIONAL HARVEST ZONE'", + "field_mapper": {"name": "UWR_NUMBER"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "wha_conditional_harvest", - "designation": "Wildlife Habitat Area: Conditional harvest", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Wildlife Habitat Area: Conditional harvest", "source": "WHSE_WILDLIFE_MANAGEMENT.WCP_WILDLIFE_HABITAT_AREA_POLY", "source_type": "BCGW", "primary_key": null, - "name_column": "COMMON_SPECIES_NAME", - "query": "(TIMBER_HARVEST_CODE = 'CONDITIONAL HARVEST ZONE' AND FEATURE_NOTES NOT LIKE '%not a legal boundary%') OR (TIMBER_HARVEST_CODE = 'NO HARVEST ZONE' AND FEATURE_NOTES IS NULL)" + "query": "(TIMBER_HARVEST_CODE = 'CONDITIONAL HARVEST ZONE' AND FEATURE_NOTES NOT LIKE '%not a legal boundary%') OR (TIMBER_HARVEST_CODE = 'NO HARVEST ZONE' AND FEATURE_NOTES IS NULL)", + "field_mapper": {"name": "COMMON_SPECIES_NAME"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "vqo_partretain", - "designation": "Visual Quality Objective (VQO): Partial Retention", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Visual Quality Objective (VQO): Partial Retention", "source": "WHSE_FOREST_VEGETATION.REC_VISUAL_LANDSCAPE_INVENTORY", "source_type": "BCGW", "primary_key": null, - "name_column": "PROJECT_NAME", - "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_RVQC_CODE = 'pr'" + "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_RVQC_CODE = 'pr'", + "field_mapper": {"name": "PROJECT_NAME"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "vqo_modify", - "designation": "Visual Quality Objective (VQO): Modification", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Visual Quality Objective (VQO): Modification", "source": "WHSE_FOREST_VEGETATION.REC_VISUAL_LANDSCAPE_INVENTORY", "source_type": "BCGW", "primary_key": null, - "name_column": "PROJECT_NAME", - "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_RVQC_CODE = 'm'" + "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_RVQC_CODE = 'm'", + "field_mapper": {"name": "PROJECT_NAME"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "community_watershed", - "designation": "Community Watershed", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Community Watershed", "source": "WHSE_WATER_MANAGEMENT.WLS_COMMUNITY_WS_PUB_SVW", "source_type": "BCGW", "primary_key": null, - "name_column": "CW_NAME", - "query": null + "query": null, + "field_mapper": {"name": "CW_NAME"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "national_wildlife_area", - "designation": "National Wildlife Areas", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "National Wildlife Areas", "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/CPCAD-BDCAPC_Nov2023.gdb.zip", "source_type": "FILE", "layer": "CPCAD_Nov2023_Albers", "primary_key": null, - "name_column": "PARENT_ID", - "query": "LOC = 2 AND BIOME = 'T' AND TYPE_E = 'National Wildlife Area'" + "query": "LOC = 2 AND BIOME = 'T' AND TYPE_E = 'National Wildlife Area'", + "field_mapper": {"name": "PARENT_ID"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "for_rec_site", - "designation": "Forest Recreation Site", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Forest Recreation Site", "source": "WHSE_FOREST_TENURE.FTEN_RECREATION_POLY_SVW", "source_type": "BCGW", "primary_key": null, - "name_column": "PROJECT_NAME", - "query": "PROJECT_TYPE in ('Recreation Site', 'Recreation Trail','Recreation Reserve', 'Interpretative Forest') and FILE_STATUS_CODE = 'HI' and RETIREMENT_DATE IS NULL" + "query": "PROJECT_TYPE in ('Recreation Site', 'Recreation Trail','Recreation Reserve', 'Interpretative Forest') and FILE_STATUS_CODE = 'HI' and RETIREMENT_DATE IS NULL", + "field_mapper": {"name": "PROJECT_NAME"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "lakes_corridors", - "designation": "Lakes South Landscape Corridor", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Lakes South Landscape Corridor", "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_LEGAL_POLY_SVW", "source_type": "BCGW", "primary_key": null, - "name_column": "LEGAL_FEAT_ID", - "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Lakes South Sustainable Resource Management Plan' AND LEGAL_FEAT_OBJECTIVE = 'Landscape Corridors'" + "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Lakes South Sustainable Resource Management Plan' AND LEGAL_FEAT_OBJECTIVE = 'Landscape Corridors'", + "field_mapper": {"name": "LEGAL_FEAT_ID"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "great_bear_fisheries_watersheds", - "designation": "Important Fisheries Watersheds (Great Bear Rainforest LUO)", - "harvest_restriction": 4, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Important Fisheries Watersheds (Great Bear Rainforest LUO)", "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/GBRO_Schedules_20160120.gdb.zip", "source_type": "FILE", "layer": "GBRSchE_IFW_20160104", "primary_key": null, - "name_column": "WS_NUM", - "query": null + "query": null, + "field_mapper": {"name": "WS_NUM"}, + "data": {"harvest_restriction": 4, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "great_bear_ebm_area", - "designation": "Great Bear Rainforest (GBR) Ecosystem-based Management (EBM) Area", - "harvest_restriction": 5, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Great Bear Rainforest (GBR) Ecosystem-based Management (EBM) Area", "source": "WHSE_ADMIN_BOUNDARIES.FADM_SPECIAL_PROTECTION_AREA", "source_type": "BCGW", "primary_key": null, - "name_column": "PROTECTED_AREA_NAME", - "query": "PROTECTED_AREA_NAME = 'Great Bear Rainforest'" + "query": "PROTECTED_AREA_NAME = 'Great Bear Rainforest'", + "field_mapper": {"name": "PROTECTED_AREA_NAME"}, + "data": {"harvest_restriction": 5, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "muskwa_kechika", - "designation": "Muskwa Kechika Management Area", - "harvest_restriction": 5, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Muskwa Kechika Management Area", "source": "WHSE_LAND_USE_PLANNING.RMP_PLAN_NON_LEGAL_POLY_SVW", "source_type": "BCGW", "primary_key": null, - "name_column": "NON_LEGAL_FEAT_ID", - "query": "NON_LEGAL_FEAT_ID IN (65244,65246,65251,65265,65274,219519,219520,219521,219522,219523,219524,219525,219526,219527,219528,219529,219530,219531,219532,219534,219535,219552,65494,65495)" + "query": "NON_LEGAL_FEAT_ID IN (65244,65246,65251,65265,65274,219519,219520,219521,219522,219523,219524,219525,219526,219527,219528,219529,219530,219531,219532,219534,219535,219552,65494,65495)", + "field_mapper": {"name": "NON_LEGAL_FEAT_ID"}, + "data": {"harvest_restriction": 5, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "fsw", - "designation": "Fisheries Sensitive Watersheds", - "harvest_restriction": 5, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Fisheries Sensitive Watersheds", "source": "WHSE_WILDLIFE_MANAGEMENT.WCP_FISH_SENSITIVE_WS_POLY", "source_type": "BCGW", "primary_key": null, - "name_column": null, - "query": null + "query": null, + "field_mapper": {"name": null}, + "data": {"harvest_restriction": 5, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "vqo_maxmodify", - "designation": "Visual Quality Objective (VQO): Maximum Modification", - "harvest_restriction": 5, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Visual Quality Objective (VQO): Maximum Modification", "source": "WHSE_FOREST_VEGETATION.REC_VISUAL_LANDSCAPE_INVENTORY", "source_type": "BCGW", "primary_key": null, - "name_column": "PROJECT_NAME", - "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_EVQO_CODE = 'MM'" + "query": "REC_MADE_KNOWN_CODE = 'Y' AND REC_EVQO_CODE = 'MM'", + "field_mapper": {"name": "PROJECT_NAME"}, + "data": {"harvest_restriction": 5, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "lrmp_cc_nc_hg", - "designation": "Haida Gwaii, Central Coast, North Coast Planning Areas", - "harvest_restriction": 5, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Haida Gwaii, Central Coast, North Coast Planning Areas", "source": "WHSE_LAND_USE_PLANNING.RMP_STRGC_LAND_RSRCE_PLAN_SVW", "source_type": "BCGW", "primary_key": null, - "name_column": "STRGC_LAND_RSRCE_PLAN_NAME", - "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Haida Gwaii Land Use Objectives Order'" + "query": "STRGC_LAND_RSRCE_PLAN_NAME = 'Haida Gwaii Land Use Objectives Order'", + "field_mapper": {"name": "STRGC_LAND_RSRCE_PLAN_NAME"}, + "data": {"harvest_restriction": 5, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "great_bear_grizzly_class2", - "designation": "Class2 Coast Grizzly Bear Habitat", - "harvest_restriction": 5, - "og_restriction": 0, - "mining_restriction": 0, + "description": "Class2 Coast Grizzly Bear Habitat", "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/GBRO_Schedules_20160120.gdb.zip", "source_type": "FILE", "layer": "GBRSchD_GB_20151105", "primary_key": null, - "name_column": "GB_CLASS", - "query": "GB_CLASS = 2" + "query": "GB_CLASS = 2", + "field_mapper": {"name": "GB_CLASS"}, + "data": {"harvest_restriction": 5, "og_restriction": 0, "mining_restriction": 0} }, { "alias": "no_special_restriction", - "designation": "No Special Restriction", - "harvest_restriction": 6, - "og_restriction": 0, - "mining_restriction": 0, + "description": "No Special Restriction", "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/BC_Boundary_Terrestrial.gpkg.zip", "source_type": "FILE", "layer": "BC_Boundary_Terrestrial_Singlepart", "primary_key": null, - "name_column": null, - "query": null + "query": null, + "field_mapper": {"name": null}, + "data": {"harvest_restriction": 6, "og_restriction": 0, "mining_restriction": 0} } ] \ No newline at end of file diff --git a/sql/tiles.sql b/sql/tiles.sql new file mode 100644 index 0000000..4a317e4 --- /dev/null +++ b/sql/tiles.sql @@ -0,0 +1,75 @@ +-- Create a tiling layer. +-- Primary tiles is the 20k bcgs grid, but they don't cover everything, add: +-- - 250k tiles in marine areas +-- - 250m buffer at northern border (60deg) + +DROP TABLE IF EXISTS tiles; +CREATE TABLE tiles (tile_id serial primary key, map_tile text, geom geometry); + +CREATE INDEX ON tiles (map_tile); +CREATE INDEX tiles_gidx ON tiles USING GIST (geom) ; + +-- add 20k tiles +INSERT INTO tiles (map_tile, geom) +SELECT map_tile, geom FROM whse_basemapping.bcgs_20k_grid; + +-- Add 250k tiles for marine areas +-- (except for SSOG to avoid confusion with USA border) + +INSERT INTO tiles (map_tile, geom) +SELECT a.map_tile||'000', +ST_Multi(ST_CollectionExtract(ST_Difference(a.geom, ST_Union(b.geom)), 3)) +FROM whse_basemapping.nts_250k_grid a +INNER JOIN whse_basemapping.bcgs_20k_grid b +ON ST_Intersects(a.geom, b.geom) +WHERE a.map_tile IN + ('092E','092C','102I','102O','102P','103A','103B', + '103F','103G', '103H', '103C','103J','103K') +GROUP BY a.map_tile, a.geom; + +-- Add 250m buffer along 60deg north to include all of the official bc boundary +-- first - grab 20k tiles along border by id +WITH north_tiles AS +(SELECT ST_Union(geom) as geom +FROM whse_basemapping.bcgs_20k_grid +WHERE (map_tile LIKE '114O%%' + OR map_tile LIKE '114P%%' + OR map_tile LIKE '104M%%' + OR map_tile LIKE '104N%%' + OR map_tile LIKE '104O%%' + OR map_tile LIKE '104P%%' + OR map_tile LIKE '094M%%' + OR map_tile LIKE '094N%%' + OR map_tile LIKE '094O%%' + OR map_tile LIKE '094P%%') + AND (substring(map_tile from 5 for 2) = '09' OR + substring(map_tile from 5 for 3) = '100') + AND (substring(map_tile from 5 for 3) != '090') + ), + +-- then buffer the aggregated tiles by 250m +buff AS +(SELECT + ST_Buffer(geom, 250) as geom +FROM north_tiles), + +-- finally, insert the difference of the buffer and exising tiles (use 250k +-- for speed) into our tile layer +diff as ( +SELECT st_union(geom) as geom FROM tiles_250k +WHERE (map_tile LIKE '114O%%' + OR map_tile LIKE '114P%%' + OR map_tile LIKE '104M%%' + OR map_tile LIKE '104N%%' + OR map_tile LIKE '104O%%' + OR map_tile LIKE '104P%%' + OR map_tile LIKE '094M%%' + OR map_tile LIKE '094N%%' + OR map_tile LIKE '094O%%' + OR map_tile LIKE '094P%%')) + +INSERT INTO tiles (map_tile, geom) +SELECT + '0000000' as map_tile, + ST_Difference(a.geom, b.geom) as geom +FROM buff a, diff b; \ No newline at end of file diff --git a/test_harvest_restrictions.py b/test_harvest_restrictions.py index 85e0fe1..8be25a8 100644 --- a/test_harvest_restrictions.py +++ b/test_harvest_restrictions.py @@ -2,36 +2,40 @@ import pytest -from harvest_restrictions import download_source, validate_sources, parse_sources +from harvest_restrictions import download, validate_sources, parse_sources @pytest.fixture def test_data(): return [ { - "designation": "National Park", "alias": "park_national", - "harvest_restriction": 1, - "og_restriction": 0, - "mining_restriction": 0, - "source": "WHSE_ADMIN_BOUNDARIES.CLAB_NATIONAL_PARKS", + "description": "National Park", "source_type": "BCGW", - "primary_key": None, - "name_column": "ENGLISH_NAME", + "source": "WHSE_ADMIN_BOUNDARIES.CLAB_NATIONAL_PARKS", "query": None, + "primary_key": None, + "field_mapper": {"name": "ENGLISH_NAME"}, + "data": { + "harvest_restriction": 1, + "og_restriction": 0, + "mining_restriction": 0, + }, }, { - "designation": "CRD Water Supply Area", "alias": "crd_water_supply_area", - "harvest_restriction": 3, - "og_restriction": 0, - "mining_restriction": 0, - "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/CRD.gdb.zip", + "description": "CRD Water Supply Area", "source_type": "FILE", + "source": "/vsizip//vsis3/$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/sources/CRD.gdb.zip", "layer": "WSA_Boundary", - "primary_key": None, - "name_column": "Name", "query": None, + "primary_key": None, + "field_mapper": {"name": "Name"}, + "data": { + "harvest_restriction": 3, + "og_restriction": 0, + "mining_restriction": 0, + }, }, ] @@ -48,29 +52,16 @@ def test_parse_alias(test_data): assert sources[0]["alias"] == "national_reserve" -def test_download_bcgw(test_data, tmpdir): - sources = [s for s in parse_sources(test_data) if s["alias"] == "park_national"] - sources = validate_sources(sources) - download_source(sources[0], out_path=tmpdir, out_table="test") - assert os.path.exists(os.path.join(tmpdir, "rr_01_park_national.parquet")) - - -def test_download_to_s3(test_data): +def test_download_bcgw(test_data): sources = [s for s in parse_sources(test_data) if s["alias"] == "park_national"] sources = validate_sources(sources) - download_source( - sources[0], - out_path=os.path.expandvars( - "s3://$OBJECTSTORE_BUCKET/dss_projects_2024/harvest_restrictions/test" - ), - out_table="test", - ) - # presume this succeeds if no error is raised + df = download(sources[0]) + assert len(df) > 0 def test_invalid_bcgw(test_data): sources = [s for s in parse_sources(test_data) if s["alias"] == "park_national"] - sources[0]["name_column"] = "INVALID_COLUMN" + sources[0]["field_mapper"] = {"name": "INVALID_COLUMN"} with pytest.raises(ValueError): sources = validate_sources(sources) @@ -80,14 +71,14 @@ def test_download_file(test_data, tmpdir): s for s in parse_sources(test_data) if s["alias"] == "crd_water_supply_area" ] sources = validate_sources(sources) - download_source(sources[0], out_path=tmpdir, out_table="test") - assert os.path.exists(os.path.join(tmpdir, "rr_02_crd_water_supply_area.parquet")) + df = download(sources[0]) + assert len(df) > 0 def test_invalid_file(test_data): sources = [ s for s in parse_sources(test_data) if s["alias"] == "crd_water_supply_area" ] - sources[0]["name_column"] = "INVALID_COLUMN" + sources[0]["field_mapper"] = {"name": "INVALID_COLUMN"} with pytest.raises(ValueError): sources = validate_sources(sources)