From 33f629472fe8b716a4d56dacb721b9c139adb71a Mon Sep 17 00:00:00 2001 From: Frederique Date: Mon, 11 Dec 2023 15:06:22 +0100 Subject: [PATCH 01/11] Added ground floor height and part of damages connection in API --- hydromt_fiat/api/data_types.py | 7 +++++ hydromt_fiat/api/exposure_vm.py | 37 ++++++++++++++++++++++- hydromt_fiat/fiat.py | 6 ++-- hydromt_fiat/workflows/exposure_vector.py | 10 ++++-- 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/hydromt_fiat/api/data_types.py b/hydromt_fiat/api/data_types.py index f0d4f89b..19eca868 100644 --- a/hydromt_fiat/api/data_types.py +++ b/hydromt_fiat/api/data_types.py @@ -70,6 +70,13 @@ class ExposureBuildingsSettings(BaseModel): occupancy_type: str max_potential_damage: str ground_floor_height: str + attribute_name_gfh: Union[str, List[str], None] + method_gfh: Union[str, List[str], None] + max_dist_gfh: Union[float, int, None] + damages: Union[str, None] + attribute_name_damages: Union[str, List[str], None] + method_damages: Union[str, List[str], None] + max_dist_damages: Union[float, int, None] unit: Units extraction_method: ExtractionMethod damage_types : Union[List[str], None] diff --git a/hydromt_fiat/api/exposure_vm.py b/hydromt_fiat/api/exposure_vm.py index e66ab02e..74411dae 100644 --- a/hydromt_fiat/api/exposure_vm.py +++ b/hydromt_fiat/api/exposure_vm.py @@ -51,6 +51,9 @@ def create_interest_area(self, **kwargs: str): def set_asset_locations_source( self, source: str, + ground_floor_height: str, + attribute_name_gfh: Union[str, List[str], None] = None, + method_gfh: Union[str, List[str], None] = "nearest", fiat_key_maps: Optional[Dict[str, str]] = None, crs: Union[str, int] = None, ): @@ -60,8 +63,10 @@ def set_asset_locations_source( asset_locations=source, occupancy_type=source, max_potential_damage=source, - ground_floor_height=source, + ground_floor_height=ground_floor_height, unit=Units.ft.value, # TODO: make flexible + attribute_name_gfh=attribute_name_gfh, + method_gfh=method_gfh, extraction_method=ExtractionMethod.centroid.value, damage_types=["structure", "content"], ) @@ -119,6 +124,36 @@ def setup_extraction_method(self, extraction_method): if self.exposure: self.exposure.setup_extraction_method(extraction_method) + def set_ground_floor_height( + self, + ground_floor_height: str, + attribute_name_gfh: Union[str, List[str], None] = None, + method_gfh: Union[str, List[str], None] = "nearest", + max_dist_gfh: Union[float, int, None] = 10, + ): + self.exposure_buildings_model.ground_floor_height = ground_floor_height + self.exposure_buildings_model.attribute_name_gfh = attribute_name_gfh + self.exposure_buildings_model.method_gfh = method_gfh + self.exposure_buildings_model.max_dist_gfh = max_dist_gfh + + def set_damages( + self, + damages: str, + attribute_name_damages: Union[str, List[str], None] = None, + method_damages: Union[str, List[str], None] = "nearest", + max_dist_damages: Union[float, int, None] = 10, + ): + self.exposure_buildings_model.damages = damages + self.exposure_buildings_model.attribute_name_damages = attribute_name_damages + self.exposure_buildings_model.method_damages = method_damages + self.exposure_buildings_model.max_dist_damages = max_dist_damages + + def set_max_dist_gfh(self, max_dist_gfh: Union[float, int, None]): + self.exposure_buildings_model.max_dist_gfh = max_dist_gfh + + def set_method_gfh(self, method_gfh: Union[str, List[str], None] = "nearest"): + self.exposure_buildings_model.method_gfh = method_gfh + def get_osm_roads( self, road_types: List[str] = [ diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 43ba3659..9fbfa63a 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -280,6 +280,8 @@ def setup_exposure_buildings( max_potential_damage: Union[str, Path], ground_floor_height: Union[int, float, str, Path, None], unit: str, + # attr_name_gfh: Union[str, List[str], None] = None, + # method: Union[str, List[str], None] = "nearest", occupancy_type_field: Union[str, None] = None, extraction_method: str = "centroid", damage_types: List[str] = ["structure", "content"], @@ -320,12 +322,12 @@ def setup_exposure_buildings( """ self.exposure = ExposureVector(self.data_catalog, self.logger, self.region) - if asset_locations == occupancy_type == max_potential_damage == ground_floor_height: + if asset_locations == occupancy_type == max_potential_damage: # The source for the asset locations, occupancy type and maximum potential # damage is the same, use one source to create the exposure data. self.exposure.setup_buildings_from_single_source( asset_locations, - ground_floor_height, + ground_floor_height, extraction_method, ground_elevation_file=ground_elevation_file, ) diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index b949b82d..2ba0de7a 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -127,6 +127,9 @@ def setup_buildings_from_single_source( source: Union[str, Path], ground_floor_height: Union[int, float, str, Path, None], extraction_method: str, + # attr_name_gfh: Union[str, List[str], None] = None, + # method_gfh: Union[str, List[str], None] = "nearest", + # max_dist: Union[int, float, None] = 10, ground_elevation_file: Union[int, float, str, Path, None] = None, ) -> None: """Set up asset locations and other available data from a single source. @@ -190,7 +193,7 @@ def setup_buildings_from_single_source( self.exposure_db["Object ID"] = range(1, len(self.exposure_db.index) + 1) # Set the ground floor height if not yet set - if "Ground Floor Height" not in self.exposure_db.columns: + if ground_floor_height != source: self.setup_ground_floor_height(ground_floor_height) # Set the extraction method @@ -279,13 +282,16 @@ def setup_buildings_from_multiple_sources( occupancy_type_field: Union[str, None] = None, damage_types: Union[List[str], None] = None, country: Union[str, None] = None, + attr_name_gfh: Union[str, List[str], None] = None, + method_gfh: Union[str, List[str], None] = "nearest", + max_dist: Union[int, float, None] = 10, ground_elevation_file: Union[int, float, str, Path, None] = None, ): self.logger.info("Setting up exposure data from multiple sources...") self.setup_asset_locations(asset_locations) self.setup_occupancy_type(occupancy_source, occupancy_type_field) self.setup_max_potential_damage(max_potential_damage, damage_types, country) - self.setup_ground_floor_height(ground_floor_height) + self.setup_ground_floor_height(ground_floor_height, attr_name_gfh, method_gfh, max_dist) self.setup_extraction_method(extraction_method) self.setup_ground_elevation( ground_elevation_file, self.exposure_db, self.get_full_gdf(self.exposure_db) From f31b30adb48a5a82e487ccba23852181adced30d Mon Sep 17 00:00:00 2001 From: Frederique Date: Mon, 11 Dec 2023 17:00:06 +0100 Subject: [PATCH 02/11] Adding separate function for the ground floor height and max potential damage updates --- hydromt_fiat/api/data_types.py | 22 +++++--- hydromt_fiat/api/exposure_vm.py | 59 +++++++++++++--------- hydromt_fiat/api/hydromt_fiat_vm.py | 8 ++- hydromt_fiat/fiat.py | 61 ++++++++++++++++++----- hydromt_fiat/workflows/exposure_vector.py | 3 -- 5 files changed, 105 insertions(+), 48 deletions(-) diff --git a/hydromt_fiat/api/data_types.py b/hydromt_fiat/api/data_types.py index 19eca868..c4b3e3b4 100644 --- a/hydromt_fiat/api/data_types.py +++ b/hydromt_fiat/api/data_types.py @@ -69,19 +69,25 @@ class ExposureBuildingsSettings(BaseModel): asset_locations: str occupancy_type: str max_potential_damage: str - ground_floor_height: str - attribute_name_gfh: Union[str, List[str], None] - method_gfh: Union[str, List[str], None] - max_dist_gfh: Union[float, int, None] - damages: Union[str, None] - attribute_name_damages: Union[str, List[str], None] - method_damages: Union[str, List[str], None] - max_dist_damages: Union[float, int, None] unit: Units extraction_method: ExtractionMethod damage_types : Union[List[str], None] +class ExposureSetupGroundFloorHeight(BaseModel): + source: str + attribute_name: Union[str, List[str], None] + method: Union[str, List[str], None] + max_dist: Union[float, int, None] + + +class ExposureSetupDamages(BaseModel): + source: str + attribute_name: Union[str, List[str], None] + method: Union[str, List[str], None] + max_dist: Union[float, int, None] + + class RoadVulnerabilitySettings(BaseModel): threshold_value: float min_hazard_value: float diff --git a/hydromt_fiat/api/exposure_vm.py b/hydromt_fiat/api/exposure_vm.py index 74411dae..e6d3a6a5 100644 --- a/hydromt_fiat/api/exposure_vm.py +++ b/hydromt_fiat/api/exposure_vm.py @@ -13,6 +13,8 @@ DataType, Driver, ExposureBuildingsSettings, + ExposureSetupGroundFloorHeight, + ExposureSetupDamages, ExposureRoadsSettings, ExtractionMethod, AggregationAreaSettings, @@ -27,6 +29,8 @@ def __init__( self.exposure_buildings_model = None self.exposure_roads_model = None self.aggregation_areas_model = None + self.exposure_ground_floor_height_model = None + self.exposure_damages_model = None self.database: IDatabase = database self.data_catalog: DataCatalog = data_catalog @@ -65,8 +69,6 @@ def set_asset_locations_source( max_potential_damage=source, ground_floor_height=ground_floor_height, unit=Units.ft.value, # TODO: make flexible - attribute_name_gfh=attribute_name_gfh, - method_gfh=method_gfh, extraction_method=ExtractionMethod.centroid.value, damage_types=["structure", "content"], ) @@ -82,8 +84,7 @@ def set_asset_locations_source( self.exposure.setup_buildings_from_single_source( source, - self.exposure_buildings_model.ground_floor_height, - "centroid", # TODO: MAKE FLEXIBLE + "centroid", ) primary_object_types = ( self.exposure.exposure_db["Primary Object Type"].unique().tolist() @@ -126,33 +127,43 @@ def setup_extraction_method(self, extraction_method): def set_ground_floor_height( self, - ground_floor_height: str, - attribute_name_gfh: Union[str, List[str], None] = None, - method_gfh: Union[str, List[str], None] = "nearest", - max_dist_gfh: Union[float, int, None] = 10, + source: str, + attribute_name: Union[str, List[str], None] = None, + method: Union[str, List[str], None] = "nearest", + max_dist: Union[float, int, None] = 10, ): - self.exposure_buildings_model.ground_floor_height = ground_floor_height - self.exposure_buildings_model.attribute_name_gfh = attribute_name_gfh - self.exposure_buildings_model.method_gfh = method_gfh - self.exposure_buildings_model.max_dist_gfh = max_dist_gfh + self.exposure_ground_floor_height_model = ExposureSetupGroundFloorHeight( + source=source, + attribute_name=attribute_name, + method=method, + max_dist=max_dist, + ) def set_damages( self, - damages: str, - attribute_name_damages: Union[str, List[str], None] = None, - method_damages: Union[str, List[str], None] = "nearest", - max_dist_damages: Union[float, int, None] = 10, + source: str, + attribute_name: Union[str, List[str], None] = None, + method: Union[str, List[str], None] = "nearest", + max_dist: Union[float, int, None] = 10, ): - self.exposure_buildings_model.damages = damages - self.exposure_buildings_model.attribute_name_damages = attribute_name_damages - self.exposure_buildings_model.method_damages = method_damages - self.exposure_buildings_model.max_dist_damages = max_dist_damages + self.exposure_damages_model = ExposureSetupDamages( + source=source, + attribute_name=attribute_name, + method=method, + max_dist=max_dist, + ) - def set_max_dist_gfh(self, max_dist_gfh: Union[float, int, None]): - self.exposure_buildings_model.max_dist_gfh = max_dist_gfh + def set_max_dist_gfh(self, max_dist: Union[float, int, None]): + self.exposure_ground_floor_height_model.max_dist = max_dist + + def set_method_gfh(self, method: Union[str, List[str], None] = "nearest"): + self.exposure_ground_floor_height_model.method = method + + def set_max_dist_damages(self, max_dist: Union[float, int, None]): + self.exposure_damages_model.max_dist = max_dist - def set_method_gfh(self, method_gfh: Union[str, List[str], None] = "nearest"): - self.exposure_buildings_model.method_gfh = method_gfh + def set_method_damages(self, method: Union[str, List[str], None] = "nearest"): + self.exposure_damages_model.method = method def get_osm_roads( self, diff --git a/hydromt_fiat/api/hydromt_fiat_vm.py b/hydromt_fiat/api/hydromt_fiat_vm.py index 3be6a582..2c9c7ad0 100644 --- a/hydromt_fiat/api/hydromt_fiat_vm.py +++ b/hydromt_fiat/api/hydromt_fiat_vm.py @@ -74,6 +74,12 @@ def build_config_yaml(self): if self.exposure_vm.exposure_roads_model: config_yaml.setup_exposure_roads = self.exposure_vm.exposure_roads_model + if self.exposure_vm.exposure_damages_model: + config_yaml.update_max_potential_damage = self.exposure_vm.exposure_damages_model + + if self.exposure_vm.exposure_ground_floor_height_model: + config_yaml.update_ground_floor_height = self.exposure_vm.exposure_ground_floor_height_model + if self.vulnerability_vm.vulnerability_roads_model: config_yaml.setup_road_vulnerability = self.vulnerability_vm.vulnerability_roads_model @@ -82,7 +88,7 @@ def build_config_yaml(self): if self.svi_vm.equity_model: config_yaml.setup_equity_data = self.svi_vm.equity_model - + database_path = self.__class__.database.drive with open(database_path / "config.yaml", "wb") as f: diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 9fbfa63a..93e45331 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -24,7 +24,10 @@ create_risk_dataset, ) from hydromt_fiat.workflows.equity_data import EquityData -from hydromt_fiat.workflows.social_vulnerability_index import SocialVulnerabilityIndex, list_of_states +from hydromt_fiat.workflows.social_vulnerability_index import ( + SocialVulnerabilityIndex, + list_of_states, +) from hydromt_fiat.workflows.vulnerability import Vulnerability from hydromt_fiat.workflows.aggregation_areas import join_exposure_aggregation_areas from hydromt_fiat.workflows.building_footprints import join_exposure_building_footprints @@ -280,8 +283,6 @@ def setup_exposure_buildings( max_potential_damage: Union[str, Path], ground_floor_height: Union[int, float, str, Path, None], unit: str, - # attr_name_gfh: Union[str, List[str], None] = None, - # method: Union[str, List[str], None] = "nearest", occupancy_type_field: Union[str, None] = None, extraction_method: str = "centroid", damage_types: List[str] = ["structure", "content"], @@ -326,10 +327,10 @@ def setup_exposure_buildings( # The source for the asset locations, occupancy type and maximum potential # damage is the same, use one source to create the exposure data. self.exposure.setup_buildings_from_single_source( - asset_locations, + asset_locations, ground_floor_height, extraction_method, - ground_elevation_file=ground_elevation_file, + ground_elevation_file=ground_elevation_file, ) else: @@ -382,13 +383,48 @@ def setup_exposure_roads( List of road types to include in the exposure data, by default True """ if not self.exposure: - self.exposure = ExposureVector(self.data_catalog, self.logger, self.region, unit=unit) + self.exposure = ExposureVector( + self.data_catalog, self.logger, self.region, unit=unit + ) self.exposure.setup_roads(roads_fn, road_damage, road_types) # Link to vulnerability curves # Combine the exposure database with pre-existing exposure data if available + def update_ground_floor_height( + self, + source: Union[int, float, str, Path, None], + attr_name: Union[str, List[str], None] = None, + method: Union[str, List[str], None] = "nearest", + max_dist: float = 10, + ): + if self.exposure: + self.exposure.setup_ground_floor_height(source, attr_name, method, max_dist) + + def update_max_potential_damage( + self, + source: Union[ + int, float, str, Path, List[str], List[Path], pd.DataFrame + ] = None, + damage_types: Union[List[str], str, None] = None, + country: Union[str, None] = None, + target_attribute: Union[str, List[str], None] = None, + attr_name: Union[str, List[str], None] = None, + method: Union[str, List[str], None] = "nearest", + max_dist: float = 10, + ): + if self.exposure: + self.exposure.setup_max_potential_damage( + max_potential_damage=source, + damage_types=damage_types, + country=country, + target_attribute=target_attribute, + attr_name=attr_name, + method=method, + max_dist=max_dist, + ) + def setup_exposure_raster(self): """Setup raster exposure data for Delft-FIAT. This function will be implemented at a later stage. @@ -586,7 +622,7 @@ def setup_social_vulnerability_index( Census data codebook_fn : Union[str, Path] The path to the codebook excel - year_data: int + year_data: int The year of which the census data should be downloaded, 2020, 2021, or 2022 save_all: bool If True, all (normalized) data variables are saved, if False, only the SVI @@ -594,7 +630,7 @@ def setup_social_vulnerability_index( """ # Check if the exposure data exists if self.exposure: - # First find the state(s) and county/counties where the exposure data is + # First find the state(s) and county/counties where the exposure data is # located in us_states_counties = self.data_catalog.get_dataframe("us_states_counties") counties, states = locate_from_bounding_box(self.exposure.bounding_box()) @@ -654,13 +690,15 @@ def setup_social_vulnerability_index( # Only save the SVI_key_domain and composite_svi_z cols_to_save = ["SVI_key_domain", "composite_svi_z", "geometry"] - svi_exp_joined = gpd.sjoin(exposure_data, svi.svi_data_shp[cols_to_save], how="left") + svi_exp_joined = gpd.sjoin( + exposure_data, svi.svi_data_shp[cols_to_save], how="left" + ) svi_exp_joined.drop(columns=["geometry"], inplace=True) svi_exp_joined = pd.DataFrame(svi_exp_joined) svi_exp_joined.rename(columns={"composite_svi_z": "SVI"}, inplace=True) del svi_exp_joined["index_right"] self.exposure.exposure_db = svi_exp_joined - + def setup_equity_data( self, census_key: str, @@ -679,7 +717,7 @@ def setup_equity_data( path : Union[str, Path] The path to the codebook excel """ - # First find the state(s) and county/counties where the exposure data is + # First find the state(s) and county/counties where the exposure data is # located in us_states_counties = self.data_catalog.get_dataframe("us_states_counties") counties, states = locate_from_bounding_box(self.exposure.bounding_box()) @@ -708,7 +746,6 @@ def setup_equity_data( self.set_tables(df=equity.equity_data_shp, name="equity_data") - def setup_aggregation_areas( self, aggregation_area_fn: Union[List[str], List[Path], str, Path], diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index 2ba0de7a..bf5fe73a 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -127,9 +127,6 @@ def setup_buildings_from_single_source( source: Union[str, Path], ground_floor_height: Union[int, float, str, Path, None], extraction_method: str, - # attr_name_gfh: Union[str, List[str], None] = None, - # method_gfh: Union[str, List[str], None] = "nearest", - # max_dist: Union[int, float, None] = 10, ground_elevation_file: Union[int, float, str, Path, None] = None, ) -> None: """Set up asset locations and other available data from a single source. From a802e4161d18584b4e7954964b8014fe755c09da Mon Sep 17 00:00:00 2001 From: Frederique Date: Mon, 11 Dec 2023 17:15:14 +0100 Subject: [PATCH 03/11] Bug fix for finding counties and parishes (are there other names for counties?) --- hydromt_fiat/workflows/gis.py | 2 +- hydromt_fiat/workflows/social_vulnerability_index.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hydromt_fiat/workflows/gis.py b/hydromt_fiat/workflows/gis.py index 2ccaefb6..26bf9b4d 100644 --- a/hydromt_fiat/workflows/gis.py +++ b/hydromt_fiat/workflows/gis.py @@ -297,7 +297,7 @@ def locate_from_bounding_box(bounding_box): locations = [geolocator.reverse(s) for s in search] locations_list = [location[0].split(", ") for location in locations] locations_list_no_numbers = [[y for y in x if not y.isnumeric()] for x in locations_list] - counties = [y for x in locations_list for y in x if "county" in y.lower()] + counties = [y for x in locations_list for y in x if ("county" in y.lower()) or ("parish" in y.lower())] states = [x[-2] for x in locations_list_no_numbers] counties_states_combo = set(list(zip(counties, states))) diff --git a/hydromt_fiat/workflows/social_vulnerability_index.py b/hydromt_fiat/workflows/social_vulnerability_index.py index 31d1d62a..59436152 100644 --- a/hydromt_fiat/workflows/social_vulnerability_index.py +++ b/hydromt_fiat/workflows/social_vulnerability_index.py @@ -167,7 +167,7 @@ def download_census_data(self, year_data): ) dfs.append(pd.DataFrame(download_census_codes)) - self.pd_census_data = pd.concat(dfs,ignore_index=True) + self.pd_census_data = pd.concat(dfs, ignore_index=True) self.logger.info( "The census data was successfully downloaded using the specified variables in the codebook Excel" ) From 8d0c1ec5676dc3393d618e486a93f79aa926aad1 Mon Sep 17 00:00:00 2001 From: Frederique Date: Mon, 11 Dec 2023 17:15:27 +0100 Subject: [PATCH 04/11] api update, separated the update functions --- hydromt_fiat/api/data_types.py | 1 + hydromt_fiat/api/exposure_vm.py | 1 + 2 files changed, 2 insertions(+) diff --git a/hydromt_fiat/api/data_types.py b/hydromt_fiat/api/data_types.py index c4b3e3b4..465a0a9c 100644 --- a/hydromt_fiat/api/data_types.py +++ b/hydromt_fiat/api/data_types.py @@ -69,6 +69,7 @@ class ExposureBuildingsSettings(BaseModel): asset_locations: str occupancy_type: str max_potential_damage: str + ground_floor_height: str unit: Units extraction_method: ExtractionMethod damage_types : Union[List[str], None] diff --git a/hydromt_fiat/api/exposure_vm.py b/hydromt_fiat/api/exposure_vm.py index e6d3a6a5..dcfb3946 100644 --- a/hydromt_fiat/api/exposure_vm.py +++ b/hydromt_fiat/api/exposure_vm.py @@ -84,6 +84,7 @@ def set_asset_locations_source( self.exposure.setup_buildings_from_single_source( source, + ground_floor_height, "centroid", ) primary_object_types = ( From a72c0cdde2c6120e7bc0d93db88512eadcd2b524 Mon Sep 17 00:00:00 2001 From: Frederique Date: Tue, 12 Dec 2023 12:10:54 +0100 Subject: [PATCH 05/11] deleted redundant code --- hydromt_fiat/api/exposure_vm.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/hydromt_fiat/api/exposure_vm.py b/hydromt_fiat/api/exposure_vm.py index dcfb3946..dc60ad1d 100644 --- a/hydromt_fiat/api/exposure_vm.py +++ b/hydromt_fiat/api/exposure_vm.py @@ -154,18 +154,6 @@ def set_damages( max_dist=max_dist, ) - def set_max_dist_gfh(self, max_dist: Union[float, int, None]): - self.exposure_ground_floor_height_model.max_dist = max_dist - - def set_method_gfh(self, method: Union[str, List[str], None] = "nearest"): - self.exposure_ground_floor_height_model.method = method - - def set_max_dist_damages(self, max_dist: Union[float, int, None]): - self.exposure_damages_model.max_dist = max_dist - - def set_method_damages(self, method: Union[str, List[str], None] = "nearest"): - self.exposure_damages_model.method = method - def get_osm_roads( self, road_types: List[str] = [ From 312ac2c8d16fb2593f40f33ec340dea085930c26 Mon Sep 17 00:00:00 2001 From: Frederique Date: Tue, 12 Dec 2023 14:25:16 +0100 Subject: [PATCH 06/11] Removed unnecessary lines --- hydromt_fiat/api/exposure_vm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hydromt_fiat/api/exposure_vm.py b/hydromt_fiat/api/exposure_vm.py index dc60ad1d..cd8ee95a 100644 --- a/hydromt_fiat/api/exposure_vm.py +++ b/hydromt_fiat/api/exposure_vm.py @@ -56,8 +56,6 @@ def set_asset_locations_source( self, source: str, ground_floor_height: str, - attribute_name_gfh: Union[str, List[str], None] = None, - method_gfh: Union[str, List[str], None] = "nearest", fiat_key_maps: Optional[Dict[str, str]] = None, crs: Union[str, int] = None, ): From 7a1d34a543ac21d72d24a934065de7cab180c618 Mon Sep 17 00:00:00 2001 From: Frederique Date: Tue, 12 Dec 2023 15:59:53 +0100 Subject: [PATCH 07/11] Implemented road damage to be set to 1 for road outages --- hydromt_fiat/api/data_types.py | 2 +- hydromt_fiat/api/exposure_vm.py | 4 ++-- hydromt_fiat/fiat.py | 2 +- hydromt_fiat/workflows/exposure_vector.py | 18 +++++++++++------- hydromt_fiat/workflows/roads.py | 8 ++++++++ 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/hydromt_fiat/api/data_types.py b/hydromt_fiat/api/data_types.py index 465a0a9c..0bbc9896 100644 --- a/hydromt_fiat/api/data_types.py +++ b/hydromt_fiat/api/data_types.py @@ -100,7 +100,7 @@ class RoadVulnerabilitySettings(BaseModel): class ExposureRoadsSettings(BaseModel): roads_fn: str road_types: Union[List[str], bool] - road_damage: str + road_damage: Union[str, int] unit: Units diff --git a/hydromt_fiat/api/exposure_vm.py b/hydromt_fiat/api/exposure_vm.py index cd8ee95a..a3c8357e 100644 --- a/hydromt_fiat/api/exposure_vm.py +++ b/hydromt_fiat/api/exposure_vm.py @@ -177,7 +177,7 @@ def get_osm_roads( self.exposure.setup_roads( source="OSM", - road_damage="default_road_max_potential_damages", + road_damage=1, road_types=road_types, ) roads = self.exposure.exposure_db.loc[ @@ -188,7 +188,7 @@ def get_osm_roads( self.exposure_roads_model = ExposureRoadsSettings( roads_fn="OSM", road_types=road_types, - road_damage="default_road_max_potential_damages", + road_damage=1, unit=Units.ft.value, ) diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 93e45331..6193e9c0 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -369,7 +369,7 @@ def setup_exposure_buildings( def setup_exposure_roads( self, roads_fn: Union[str, Path], - road_damage: Union[str, Path], + road_damage: Union[str, Path, int], road_types: Union[str, List[str], bool] = True, unit: str = "m", ): diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index bf5fe73a..9ab1f61c 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -31,7 +31,7 @@ ground_elevation_from_dem, ) -from hydromt_fiat.workflows.roads import get_max_potential_damage_roads +from hydromt_fiat.workflows.roads import get_max_potential_damage_roads, get_road_lengths class ExposureVector(Exposure): @@ -220,7 +220,7 @@ def setup_buildings_from_single_source( def setup_roads( self, source: Union[str, Path], - road_damage: Union[str, Path], + road_damage: Union[str, Path, int], road_types: Union[str, List[str], bool] = True, ): self.logger.info("Setting up roads...") @@ -255,11 +255,15 @@ def setup_roads( "The damage function 'roads' is selected for all of the structure damage to the roads." ) - # Add the max potential damage and the length of the segments to the roads - road_damage = self.data_catalog.get_dataframe(road_damage) - roads[ - ["Max Potential Damage: Structure", "Segment Length [m]"] - ] = get_max_potential_damage_roads(roads, road_damage) + if isinstance(road_damage, str): + # Add the max potential damage and the length of the segments to the roads + road_damage = self.data_catalog.get_dataframe(road_damage) + roads[ + ["Max Potential Damage: Structure", "Segment Length [m]"] + ] = get_max_potential_damage_roads(roads, road_damage) + elif isinstance(road_damage, int): + roads["Segment Length [m]"] = get_road_lengths(roads) + roads["Max Potential Damage: Structure"] = road_damage self.set_exposure_geoms(roads[["Object ID", "geometry"]]) self.set_geom_names("roads") diff --git a/hydromt_fiat/workflows/roads.py b/hydromt_fiat/workflows/roads.py index a7924f14..a6357505 100644 --- a/hydromt_fiat/workflows/roads.py +++ b/hydromt_fiat/workflows/roads.py @@ -40,3 +40,11 @@ def get_max_potential_damage_roads( ) return roads[["maximum_potential_damage", "segment_length"]] + + +def get_road_lengths(roads): + if roads.crs.is_geographic: + # If the CRS is geographic, reproject to the nearest UTM zone + nearest_utm = utm_crs(roads.total_bounds) + roads = roads.to_crs(nearest_utm) + return roads.length \ No newline at end of file From 22b24dcdc40d4529fe7de47060beb579bccbae9f Mon Sep 17 00:00:00 2001 From: Frederique Date: Tue, 12 Dec 2023 16:16:58 +0100 Subject: [PATCH 08/11] Added ground elevation function to fiat.py and updated, still to be added to the api --- hydromt_fiat/fiat.py | 7 +++++++ hydromt_fiat/workflows/exposure_vector.py | 12 +++++------- hydromt_fiat/workflows/gis.py | 4 +++- tests/test_ground_elevation.py | 2 -- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 6193e9c0..5ed499f2 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -424,6 +424,13 @@ def update_max_potential_damage( method=method, max_dist=max_dist, ) + + def update_ground_elevation( + self, + ground_elevation: Union[int, float, None, str, Path], + ): + if self.exposure: + self.exposure.setup_ground_elevation(ground_elevation) def setup_exposure_raster(self): """Setup raster exposure data for Delft-FIAT. diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index 9ab1f61c..3699f526 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -680,22 +680,20 @@ def setup_max_potential_damage( def setup_ground_elevation( self, - ground_elevation: Union[int, float, None, str, Path, List[str], List[Path]], - exposure_db: pd.DataFrame, - exposure_geoms: gpd.GeoDataFrame, + ground_elevation: Union[int, float, None, str, Path], ) -> None: if ground_elevation: - ground_elevation_from_dem( + self.exposure_db["Ground Elevation"] = ground_elevation_from_dem( ground_elevation=ground_elevation, - exposure_db=exposure_db, - exposure_geoms=exposure_geoms, + exposure_db=self.exposure_db, + exposure_geoms=self.exposure_geoms, ) else: print( "Ground elevation is not recognized by the setup_ground_elevation function\n Ground elevation will be set to 0" ) - exposure_db["Ground Elevation"] = 0 + self.exposure_db["Ground Elevation"] = 0 def update_max_potential_damage( self, updated_max_potential_damages: pd.DataFrame diff --git a/hydromt_fiat/workflows/gis.py b/hydromt_fiat/workflows/gis.py index 26bf9b4d..e2fb4ac2 100644 --- a/hydromt_fiat/workflows/gis.py +++ b/hydromt_fiat/workflows/gis.py @@ -242,7 +242,7 @@ def join_spatial_data( def ground_elevation_from_dem( - ground_elevation: Union[int, float, None, str, Path, List[str], List[Path]], + ground_elevation: Union[int, float, None, str, Path], exposure_db: pd.DataFrame, exposure_geoms: gpd.GeoDataFrame, ) -> None: @@ -284,6 +284,8 @@ def ground_elevation_from_dem( exposure_db["Ground Elevation"] = zonal_means exposure_db["Ground Elevation"].bfill(inplace=True) + return exposure_db["Ground Elevation"] + def locate_from_bounding_box(bounding_box): geolocator = Nominatim(user_agent="hydromt-fiat") diff --git a/tests/test_ground_elevation.py b/tests/test_ground_elevation.py index f437218a..e35c8ded 100644 --- a/tests/test_ground_elevation.py +++ b/tests/test_ground_elevation.py @@ -42,8 +42,6 @@ def test_ground_elevation(case): fm.exposure.setup_ground_elevation( _cases[case]["ground_elevation_file"], - fm.exposure.exposure_db, - fm.exposure.get_full_gdf(fm.exposure.exposure_db), ) # Remove the new root folder if it already exists From c09bfbdb43f226fa3d4f0a27b63a6e10390b3002 Mon Sep 17 00:00:00 2001 From: Frederique Date: Wed, 13 Dec 2023 10:56:44 +0100 Subject: [PATCH 09/11] Updates according to bugfixing after failing tests --- hydromt_fiat/api/data_types.py | 4 +++ hydromt_fiat/api/exposure_vm.py | 7 +++++ hydromt_fiat/workflows/exposure_vector.py | 30 +++++++++---------- ...t_update_max_potential_damage_user_data.py | 16 +++++----- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/hydromt_fiat/api/data_types.py b/hydromt_fiat/api/data_types.py index 0bbc9896..bbe2a94a 100644 --- a/hydromt_fiat/api/data_types.py +++ b/hydromt_fiat/api/data_types.py @@ -81,6 +81,10 @@ class ExposureSetupGroundFloorHeight(BaseModel): method: Union[str, List[str], None] max_dist: Union[float, int, None] + +class ExposureSetupGroundElevation(BaseModel): + source: Union[int, float, None, str] + class ExposureSetupDamages(BaseModel): source: str diff --git a/hydromt_fiat/api/exposure_vm.py b/hydromt_fiat/api/exposure_vm.py index a3c8357e..a805bcb2 100644 --- a/hydromt_fiat/api/exposure_vm.py +++ b/hydromt_fiat/api/exposure_vm.py @@ -15,6 +15,7 @@ ExposureBuildingsSettings, ExposureSetupGroundFloorHeight, ExposureSetupDamages, + ExposureSetupGroundElevation, ExposureRoadsSettings, ExtractionMethod, AggregationAreaSettings, @@ -31,6 +32,7 @@ def __init__( self.aggregation_areas_model = None self.exposure_ground_floor_height_model = None self.exposure_damages_model = None + self.exposure_ground_elevation_model = None self.database: IDatabase = database self.data_catalog: DataCatalog = data_catalog @@ -152,6 +154,11 @@ def set_damages( max_dist=max_dist, ) + def set_ground_elevation(self, source: Union[int, float, None, str]): + self.exposure_ground_elevation_model = ExposureSetupGroundElevation( + source=source + ) + def get_osm_roads( self, road_types: List[str] = [ diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index 3699f526..ee6d6178 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -31,7 +31,10 @@ ground_elevation_from_dem, ) -from hydromt_fiat.workflows.roads import get_max_potential_damage_roads, get_road_lengths +from hydromt_fiat.workflows.roads import ( + get_max_potential_damage_roads, + get_road_lengths, +) class ExposureVector(Exposure): @@ -209,8 +212,6 @@ def setup_buildings_from_single_source( if ground_elevation_file is not None: self.setup_ground_elevation( ground_elevation_file, - self.exposure_db, - gpd.GeoDataFrame(self.exposure_db[["Object ID", "geometry"]]), ) # Remove the geometry column from the exposure_db @@ -292,11 +293,11 @@ def setup_buildings_from_multiple_sources( self.setup_asset_locations(asset_locations) self.setup_occupancy_type(occupancy_source, occupancy_type_field) self.setup_max_potential_damage(max_potential_damage, damage_types, country) - self.setup_ground_floor_height(ground_floor_height, attr_name_gfh, method_gfh, max_dist) - self.setup_extraction_method(extraction_method) - self.setup_ground_elevation( - ground_elevation_file, self.exposure_db, self.get_full_gdf(self.exposure_db) + self.setup_ground_floor_height( + ground_floor_height, attr_name_gfh, method_gfh, max_dist ) + self.setup_extraction_method(extraction_method) + self.setup_ground_elevation(ground_elevation_file) def setup_asset_locations(self, asset_locations: str) -> None: """Set up the asset locations (points or polygons). @@ -576,11 +577,10 @@ def setup_max_potential_damage( int, float, str, Path, List[str], List[Path], pd.DataFrame ] = None, damage_types: Union[List[str], str, None] = None, - country: Union[str, None] = None, - target_attribute: Union[str, List[str], None] = None, attr_name: Union[str, List[str], None] = None, method: Union[str, List[str], None] = "nearest", max_dist: float = 10, + country: Union[str, None] = None, ) -> None: """Setup the max potential damage column of the exposure data in various ways. @@ -592,8 +592,6 @@ def setup_max_potential_damage( _description_, by default None country : Union[str, None], optional _description_, by default None - target_attribute : Union[str, List[str], None], optional - _description_, by default None attr_name : Union[str, List[str], None], optional _description_, by default None method : Union[str, List[str], None], optional @@ -675,7 +673,7 @@ def setup_max_potential_damage( gdf = self.get_full_gdf(self.exposure_db) gdf = join_spatial_data(gdf, gfh, attr_name, method, max_dist, self.logger) self.exposure_db = self._set_values_from_other_column( - gdf, target_attribute, attr_name + gdf, f"Max Potential Damage: {damage_types[0].capitalize()}", attr_name ) def setup_ground_elevation( @@ -686,7 +684,7 @@ def setup_ground_elevation( self.exposure_db["Ground Elevation"] = ground_elevation_from_dem( ground_elevation=ground_elevation, exposure_db=self.exposure_db, - exposure_geoms=self.exposure_geoms, + exposure_geoms=self.get_full_gdf(self.exposure_db), ) else: @@ -1081,9 +1079,6 @@ def setup_new_composite_areas( self.crs, ) - # Adding elevation data into the new objects - self.setup_ground_elevation(ground_elevation, new_objects, _new_exposure_geoms) - # Update the exposure_db self.exposure_db = pd.concat([self.exposure_db, new_objects]).reset_index( drop=True @@ -1092,6 +1087,9 @@ def setup_new_composite_areas( # Update the exposure_geoms self.set_exposure_geoms(_new_exposure_geoms) + # Adding elevation data into the new objects + self.setup_ground_elevation(ground_elevation) + def link_exposure_vulnerability( self, exposure_linking_table: pd.DataFrame, diff --git a/tests/test_update_max_potential_damage_user_data.py b/tests/test_update_max_potential_damage_user_data.py index 15a3ed0b..2824b21d 100644 --- a/tests/test_update_max_potential_damage_user_data.py +++ b/tests/test_update_max_potential_damage_user_data.py @@ -17,7 +17,7 @@ "data_catalog": DATADIR / "hydromt_fiat_catalog_USA.yml", "max_potential_damage_file": EXAMPLEDIR / "fake_max_potential_damage_points.gpkg", - "target_attribute": "Max Potential Damage: Content", + "damage_types": "content", "attribute": "maxpotential_content", "method": "nearest", "max_dist": 50, @@ -28,7 +28,7 @@ "data_catalog": DATADIR / "hydromt_fiat_catalog_USA.yml", "max_potential_damage_file": EXAMPLEDIR / "fake_max_potential_damage_polygons.gpkg", - "target_attribute": "Max Potential Damage: Structure", + "damage_types": "structure", "attribute": "maxpotential_structure", "method": "intersection", "max_dist": None, @@ -46,17 +46,17 @@ def test_update_max_potential_damage(case): ) fm.read() - target_column = _cases[case]["target_attribute"] + target_column = f"Max Potential Damage: {_cases[case]['damage_types'].capitalize()}" original_exposure = copy.deepcopy(fm.exposure.exposure_db) unique_mp_original = original_exposure[target_column].unique() fm.exposure.setup_max_potential_damage( - _cases[case]["max_potential_damage_file"], - _cases[case]["target_attribute"], - _cases[case]["attribute"], - _cases[case]["method"], - _cases[case]["max_dist"] + max_potential_damage=_cases[case]["max_potential_damage_file"], + damage_types=_cases[case]["damage_types"], + attr_name=_cases[case]["attribute"], + method=_cases[case]["method"], + max_dist=_cases[case]["max_dist"] ) # Remove the new root folder if it already exists From 8d7119ef377473845e5bdd4290f07a49fb223469 Mon Sep 17 00:00:00 2001 From: Frederique Date: Wed, 13 Dec 2023 13:05:44 +0100 Subject: [PATCH 10/11] Added output from testdata to the gitignore --- .gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d1942ede..faaa1226 100644 --- a/.gitignore +++ b/.gitignore @@ -135,4 +135,10 @@ examples/data/global_OSM_JRC/fiat_model/ examples/data/update_ground_floor_height/test_update_ground_floor_height_points/ examples/data/update_ground_floor_height/test_update_ground_floor_height_polygons/ examples/data/building_footprints/fiat_model_bfs/ -examples/data/aggregation_zones/output/ \ No newline at end of file +examples/data/aggregation_zones/output/ +examples/data/setup_exposure_buildings_with_dem_data/test_vulnerability_and_exposure_NSI_with_dem_data/ +examples/data/setup_new_composite_area/test_setup_new_composite_area_datum/ +examples/data/setup_new_composite_area/test_setup_new_composite_area_elevation/ +examples/data/setup_new_composite_area/test_setup_new_composite_area_geom/ +examples/data/update_max_potential_damage/test_update_max_potential_damage_points/ +examples/data/update_max_potential_damage/test_update_max_potential_damage_polygons/ \ No newline at end of file From 3dc3585bff1b06098d9ea4aa447641a0f24c41c2 Mon Sep 17 00:00:00 2001 From: Frederique Date: Wed, 13 Dec 2023 13:10:37 +0100 Subject: [PATCH 11/11] Updated path to equity data --- tests/test_equity_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_equity_data.py b/tests/test_equity_data.py index 005cac71..c8260a1c 100644 --- a/tests/test_equity_data.py +++ b/tests/test_equity_data.py @@ -88,7 +88,7 @@ def test_equity_data(case): assert root.joinpath("exposure", "region.gpkg").exists() # Check if the equity data exists - assert root.joinpath("exposure", "equity", "equity_data.csv").exists() + assert root.joinpath("equity", "equity_data.csv").exists() # Check if the vulnerability data exists assert root.joinpath("vulnerability", "vulnerability_curves.csv").exists()