From c0fa19f0a98f84ef9eab3659dba372f3f195589c Mon Sep 17 00:00:00 2001 From: Frederique Date: Fri, 20 Oct 2023 09:52:18 +0200 Subject: [PATCH 01/11] Start of integration into the code base --- hydromt_fiat/workflows/exposure_vector.py | 22 ++++++++++++++++++++++ hydromt_fiat/workflows/roads.py | 0 2 files changed, 22 insertions(+) create mode 100644 hydromt_fiat/workflows/roads.py diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index a9798266..6f3618e5 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -182,6 +182,28 @@ def setup_buildings_from_single_source( # Set the name to the geom_names self.set_geom_names("buildings") + def setup_roads(self, source: Union[str, Path]): + NotImplemented + self.logger.info("Setting up roads...") + # if str(source).upper() == "OSM": + # polygon = self.region.iloc[0].values[0] + # assets = get_assets_from_osm(polygon) + + # if assets.empty: + # self.logger.warning( + # "No assets found in the selected region from source " + # f"{source}." + # ) + + # # Rename the osmid column to Object ID + # assets.rename(columns={"osmid": "Object ID"}, inplace=True) + # else: + # assets = self.data_catalog.get_geodataframe( + # source, geom=self.region + # ) + # self.set_exposure_geoms(roads) + # self.set_geom_names("roads") + def setup_buildings_from_multiple_sources( self, asset_locations: Union[str, Path], diff --git a/hydromt_fiat/workflows/roads.py b/hydromt_fiat/workflows/roads.py new file mode 100644 index 00000000..e69de29b From baee6a199644f0b962fa13e7da8ed044a863fc0e Mon Sep 17 00:00:00 2001 From: Frederique Date: Fri, 20 Oct 2023 10:11:27 +0200 Subject: [PATCH 02/11] implemented empty functions for the roads and changed name of setup_exposure_vector function to setup_exposure_buildings --- docs/api/api_index.rst | 2 +- docs/data/hazus.rst | 2 +- docs/data/jrc.rst | 2 +- docs/data/national_structure_inventory.rst | 2 +- docs/data/openstreetmap.rst | 2 +- docs/user_guide/user_guide_overview.rst | 6 +++--- examples/global_OSM_JRC.ipynb | 2 +- hydromt_fiat/api/data_types.py | 2 +- hydromt_fiat/api/hydromt_fiat_vm.py | 2 +- hydromt_fiat/fiat.py | 7 +++++-- tests/test_SVI_exposure.py | 2 +- tests/test_integration.py | 4 ++-- tests/test_vulnerability_exposure.py | 4 ++-- tests/test_vulnerability_exposure_add_to_data_catalog.py | 4 ++-- tests/test_vulnerability_exposure_global_default.py | 2 +- 15 files changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/api/api_index.rst b/docs/api/api_index.rst index 0c12e278..4885d1cb 100644 --- a/docs/api/api_index.rst +++ b/docs/api/api_index.rst @@ -23,7 +23,7 @@ Build components fiat.FiatModel.setup_basemaps fiat.FiatModel.setup_vulnerability fiat.FiatModel.setup_vulnerability_from_csv - fiat.FiatModel.setup_exposure_vector + fiat.FiatModel.setup_exposure_buildings fiat.FiatModel.setup_hazard fiat.FiatModel.setup_social_vulnerability_index diff --git a/docs/data/hazus.rst b/docs/data/hazus.rst index b074c0e9..824f3c03 100644 --- a/docs/data/hazus.rst +++ b/docs/data/hazus.rst @@ -16,7 +16,7 @@ by providing the following values in the configuration file:: vulnerability_identifiers_and_linking_fn = "default_hazus_iwr_linking" unit = "ft" - [setup_exposure_vector] + [setup_exposure_buildings] max_potential_damage = "hazus_max_potential_damages" unit = "m" diff --git a/docs/data/jrc.rst b/docs/data/jrc.rst index c73551f0..c784b821 100644 --- a/docs/data/jrc.rst +++ b/docs/data/jrc.rst @@ -14,7 +14,7 @@ by providing the following values in the configuration file:: vulnerability_identifiers_and_linking_fn = "jrc_vulnerability_curves_linking" unit = "m" - [setup_exposure_vector] + [setup_exposure_buildings] max_potential_damage = "jrc_damage_values" unit = "m" diff --git a/docs/data/national_structure_inventory.rst b/docs/data/national_structure_inventory.rst index 6ebaae52..dd4806e1 100644 --- a/docs/data/national_structure_inventory.rst +++ b/docs/data/national_structure_inventory.rst @@ -8,7 +8,7 @@ For projects in the United States, users of **HydroMT-FIAT** can directly and ea of the `National Structure Inventory `_ (NSI). The user can access the data through providing 'NSI' in the configuration file as such:: - [setup_exposure_vector] + [setup_exposure_buildings] asset_locations = "NSI" occupancy_type = "NSI" max_potential_damage = "NSI" diff --git a/docs/data/openstreetmap.rst b/docs/data/openstreetmap.rst index cc9d3479..c1cc1d67 100644 --- a/docs/data/openstreetmap.rst +++ b/docs/data/openstreetmap.rst @@ -7,7 +7,7 @@ OpenStreetMap Building Footprints and Land Use Users of **HydroMT-FIAT** can directly and easily make use of the `OpenStreetMap `_ (OSM) initiative. The user can access the data through providing 'OSM' in the configuration file as such:: - [setup_exposure_vector] + [setup_exposure_buildings] asset_locations = "OSM" occupancy_type = "OSM" diff --git a/docs/user_guide/user_guide_overview.rst b/docs/user_guide/user_guide_overview.rst index 88d425c5..a9bdbb7c 100644 --- a/docs/user_guide/user_guide_overview.rst +++ b/docs/user_guide/user_guide_overview.rst @@ -32,9 +32,9 @@ to local data on the user's machine. The `asset_locations`, `occupancy_type`, an data should be provided as a vector file (e.g. *.shp* or *.gpkg*). The `ground_floor_height` can currently only be set to a single value (this will be updated soon!). The `damage_types` should be provided as a list of strings (e.g. ["structure", "content"]). The `unit` should be provided as a string (e.g. "m"). -See below how the `setup_exposure_vector` method can be used to build or update the exposure data:: +See below how the `setup_exposure_buildings` method can be used to build or update the exposure data:: - [setup_exposure_vector] + [setup_exposure_buildings] asset_locations = occupancy_type = damage_types = @@ -45,7 +45,7 @@ See below how the `setup_exposure_vector` method can be used to build or update The following method is used to build or update the **exposure** data: .. autosummary:: - FiatModel.setup_exposure_vector + FiatModel.setup_exposure_buildings For more information, see the :ref:`exposure_vector`. diff --git a/examples/global_OSM_JRC.ipynb b/examples/global_OSM_JRC.ipynb index 41efdc50..08afd705 100644 --- a/examples/global_OSM_JRC.ipynb +++ b/examples/global_OSM_JRC.ipynb @@ -242,7 +242,7 @@ " \"continent\": continent,\n", " \"unit\": unit,\n", " },\n", - " \"setup_exposure_vector\": {\n", + " \"setup_exposure_buildings\": {\n", " \"asset_locations\": asset_locations,\n", " \"occupancy_type\": occupancy_type,\n", " \"max_potential_damage\": max_potential_damage,\n", diff --git a/hydromt_fiat/api/data_types.py b/hydromt_fiat/api/data_types.py index f1421399..0bc6cbc1 100644 --- a/hydromt_fiat/api/data_types.py +++ b/hydromt_fiat/api/data_types.py @@ -85,4 +85,4 @@ class ConfigIni(BaseModel): setup_config: ModelIni setup_hazard: HazardIni setup_vulnerability: VulnerabilityIni - setup_exposure_vector: ExposureVectorIni + setup_exposure_buildings: ExposureVectorIni diff --git a/hydromt_fiat/api/hydromt_fiat_vm.py b/hydromt_fiat/api/hydromt_fiat_vm.py index 3c5f3072..affdcf3b 100644 --- a/hydromt_fiat/api/hydromt_fiat_vm.py +++ b/hydromt_fiat/api/hydromt_fiat_vm.py @@ -73,7 +73,7 @@ def build_config_ini(self): setup_config=self.model_vm.config_model, setup_hazard=self.hazard_vm.hazard_model, setup_vulnerability=self.vulnerability_vm.vulnerability_model, - setup_exposure_vector=self.exposure_vm.exposure_model, + setup_exposure_buildings=self.exposure_vm.exposure_model, ) database_path = self.__class__.database.drive diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index ab7128bd..6288fbbc 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -250,7 +250,7 @@ def setup_vulnerability_from_csv(self, csv_fn: Union[str, Path], unit: str) -> N ) self.vulnerability.from_csv(csv_fn) - def setup_exposure_vector( + def setup_exposure_buildings( self, asset_locations: Union[str, Path], occupancy_type: Union[str, Path], @@ -322,7 +322,7 @@ def setup_exposure_vector( except AssertionError: logging.error( "Please call the 'setup_vulnerability' function before " - "the 'setup_exposure_vector' function. Error message: {e}" + "the 'setup_exposure_buildings' function. Error message: {e}" ) self.exposure.link_exposure_vulnerability( self.vf_ids_and_linking_df, damage_types @@ -334,6 +334,9 @@ def setup_exposure_vector( self.set_config("exposure.geom.crs", self.exposure.crs) self.set_config("exposure.geom.unit", unit) + def setup_exposure_roads(self): + NotImplemented + def setup_exposure_raster(self): """Setup raster exposure data for Delft-FIAT. This function will be implemented at a later stage. diff --git a/tests/test_SVI_exposure.py b/tests/test_SVI_exposure.py index 3128ee7e..cd6edcb5 100644 --- a/tests/test_SVI_exposure.py +++ b/tests/test_SVI_exposure.py @@ -51,7 +51,7 @@ "functions_max": ["AGR1"], "unit": "feet", }, - "setup_exposure_vector": { + "setup_exposure_buildings": { "asset_locations": "NSI", "occupancy_type": "NSI", "max_potential_damage": "NSI", diff --git a/tests/test_integration.py b/tests/test_integration.py index 9b61a7f5..c6e0cec0 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -51,7 +51,7 @@ "unit": "feet", "step_size": 0.01, }, - "setup_exposure_vector": { + "setup_exposure_buildings": { "asset_locations": "NSI", "occupancy_type": "NSI", "max_potential_damage": "NSI", @@ -91,7 +91,7 @@ "unit": "feet", "step_size": 0.01, }, - "setup_exposure_vector": { + "setup_exposure_buildings": { "asset_locations": "NSI", "occupancy_type": "NSI", "max_potential_damage": "NSI", diff --git a/tests/test_vulnerability_exposure.py b/tests/test_vulnerability_exposure.py index dbeb376c..fdd35cc6 100644 --- a/tests/test_vulnerability_exposure.py +++ b/tests/test_vulnerability_exposure.py @@ -51,7 +51,7 @@ "unit": "feet", "step_size": 0.1, }, - "setup_exposure_vector": { + "setup_exposure_buildings": { "asset_locations": "NSI", "occupancy_type": "NSI", "max_potential_damage": "NSI", @@ -80,7 +80,7 @@ "unit": "feet", "step_size": 0.1, }, - "setup_exposure_vector": { + "setup_exposure_buildings": { "asset_locations": "NSI", "occupancy_type": "NSI", "max_potential_damage": "NSI", diff --git a/tests/test_vulnerability_exposure_add_to_data_catalog.py b/tests/test_vulnerability_exposure_add_to_data_catalog.py index 017c1b4b..74aaa84d 100644 --- a/tests/test_vulnerability_exposure_add_to_data_catalog.py +++ b/tests/test_vulnerability_exposure_add_to_data_catalog.py @@ -50,7 +50,7 @@ "vulnerability_identifiers_and_linking_fn": "tests/data/vulnerability_test_file_input_miami_landuse.csv", "unit": "feet", }, - "setup_exposure_vector": { + "setup_exposure_buildings": { "asset_locations": "OSM", "occupancy_type": "landuse_miami_dade", "occupancy_type_field": "DESCR", @@ -77,7 +77,7 @@ "vulnerability_identifiers_and_linking_fn": "tests/data/vulnerability_test_file_input_miami_landuse.csv", "unit": "feet", }, - "setup_exposure_vector": { + "setup_exposure_buildings": { "asset_locations": "osm_building_footprints", "occupancy_type": "landuse_miami_dade", "occupancy_type_field": "DESCR", diff --git a/tests/test_vulnerability_exposure_global_default.py b/tests/test_vulnerability_exposure_global_default.py index 2ca2b9ee..e8a203fa 100644 --- a/tests/test_vulnerability_exposure_global_default.py +++ b/tests/test_vulnerability_exposure_global_default.py @@ -55,7 +55,7 @@ "vulnerability_identifiers_and_linking_fn": "jrc_vulnerability_curves_linking", "unit": "m", }, - "setup_exposure_vector": { + "setup_exposure_buildings": { "asset_locations": "OSM", "occupancy_type": "OSM", "max_potential_damage": "jrc_damage_values", From 60852b16ed24f0aaf742e7bf94e330cbc8bd5304 Mon Sep 17 00:00:00 2001 From: Frederique Date: Fri, 20 Oct 2023 11:09:41 +0200 Subject: [PATCH 03/11] setting up functions --- hydromt_fiat/data_apis/open_street_maps.py | 34 +++++++++++++++++ hydromt_fiat/fiat.py | 31 ++++++++++++++-- hydromt_fiat/workflows/exposure_vector.py | 43 +++++++++++++--------- hydromt_fiat/workflows/vulnerability.py | 26 ++++++++++--- 4 files changed, 107 insertions(+), 27 deletions(-) diff --git a/hydromt_fiat/data_apis/open_street_maps.py b/hydromt_fiat/data_apis/open_street_maps.py index 868e5170..b53b07cd 100644 --- a/hydromt_fiat/data_apis/open_street_maps.py +++ b/hydromt_fiat/data_apis/open_street_maps.py @@ -25,6 +25,40 @@ def get_assets_from_osm(polygon: Polygon): return footprints +def get_roads_from_osm( + polygon: Union[Polygon, Path, str] +) ->gpd.GeoDataFrame: + tag = { + "highway": ["motorway"] + } # this is the tag we use to find the correct OSM data + if isinstance(polygon,(Path,str)): + gdf = gpd.read_file(polygon) + polygon =gdf.geometry.iloc[0] + else: + return polygon + + roads = ox.features.features_from_polygon( + polygon, tags=tag + ) # then we query the data + + if roads.empty: + logging.warning("No road network found from OSM") + return None + + logging.info(f"Total number of roads found from OSM: {len(roads)}") + + # Not sure if this is needed here and maybe filter for the columns that we need + roads = roads.loc[ + (roads.geometry.type == "Polygon") + | (roads.geometry.type == "MultiPolygon") + | (roads.geometry.type == "LineString") + ] + roads = roads.reset_index(drop=True) + roads.rename(columns={"element_type": "type"}, inplace=True) + + return roads + + def get_landuse_from_osm(polygon: Polygon): tags = {"landuse": True} # this is the tag we use to find the correct OSM data landuse = ox.features.features_from_polygon(polygon, tags) # then we query the data diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 6288fbbc..8abd933b 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -244,12 +244,28 @@ def setup_vulnerability_from_csv(self, csv_fn: Union[str, Path], unit: str) -> N (e.g. meter). """ # Process the vulnerability data - self.vulnerability = Vulnerability( - unit, - self.logger, - ) + if not self.vulnerability: + self.vulnerability = Vulnerability( + unit, + self.logger, + ) self.vulnerability.from_csv(csv_fn) + def setup_road_vulnerability( + self, + unit: str, + threshold_value: float = 0.6, + min_hazard_value: float = 0, + max_hazard_value: float = 10, + step_hazard_value: float = 1.0, + ): + if not self.vulnerability: + self.vulnerability = Vulnerability( + unit, + self.logger, + ) + self.vulnerability.create_step_function("roads", threshold_value, min_hazard_value, max_hazard_value, step_hazard_value) + def setup_exposure_buildings( self, asset_locations: Union[str, Path], @@ -337,6 +353,13 @@ def setup_exposure_buildings( def setup_exposure_roads(self): NotImplemented + # First you get the data + + # Link to vulnerability curves + + # Combine the exposure database with pre-existing exposure data if available + + def setup_exposure_raster(self): """Setup raster exposure data for Delft-FIAT. This function will be implemented at a later stage. diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index 6f3618e5..c6093426 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -185,24 +185,31 @@ def setup_buildings_from_single_source( def setup_roads(self, source: Union[str, Path]): NotImplemented self.logger.info("Setting up roads...") - # if str(source).upper() == "OSM": - # polygon = self.region.iloc[0].values[0] - # assets = get_assets_from_osm(polygon) - - # if assets.empty: - # self.logger.warning( - # "No assets found in the selected region from source " - # f"{source}." - # ) - - # # Rename the osmid column to Object ID - # assets.rename(columns={"osmid": "Object ID"}, inplace=True) - # else: - # assets = self.data_catalog.get_geodataframe( - # source, geom=self.region - # ) - # self.set_exposure_geoms(roads) - # self.set_geom_names("roads") + if str(source).upper() == "OSM": + polygon = self.region.iloc[0].values[0] + roads = get_roads_from_osm(polygon) + + if assets.empty: + self.logger.warning( + "No assets found in the selected region from source " + f"{source}." + ) + + # Rename the osmid column to Object ID + assets.rename(columns={"osmid": "Object ID"}, inplace=True) + else: + assets = self.data_catalog.get_geodataframe( + source, geom=self.region + ) + # add the function to segmentize the roads into certain segments + + # Primary Object Type = roads + # Secondary Object Type = road type + + # Add the max potential damage to the roads + + self.set_exposure_geoms(roads) + self.set_geom_names("roads") def setup_buildings_from_multiple_sources( self, diff --git a/hydromt_fiat/workflows/vulnerability.py b/hydromt_fiat/workflows/vulnerability.py index 09f7b469..ca3dbbc9 100644 --- a/hydromt_fiat/workflows/vulnerability.py +++ b/hydromt_fiat/workflows/vulnerability.py @@ -15,7 +15,7 @@ def __init__( logger: logging.Logger = None, fn: Union[str, Path] = None, ): - """A class for creating, adding, and updating damage functions to use in + """A class for creating, adding, and updating damage functions to use in Delft-FIAT. Attributes @@ -24,7 +24,7 @@ def __init__( The name of the hazard variable in the damage functions. hazard_values : list The hazard values of the damage functions. This variable is updated when a - new damage function is added which has different hazard values than the + new damage function is added which has different hazard values than the current. functions : dict A dictionary of the damage functions fractions. @@ -43,7 +43,7 @@ def __init__( read(self, fn) Reads the vulnerability data from a CSV file. from_csv(self, fn) - Reads in one or multiple CSVs containing damage functions and adds them to + Reads in one or multiple CSVs containing damage functions and adds them to the vulnerability functions variable. from_table(self, vulnerability_functions) Reads an existing Delft-FIAT damage function (i.e. vulnerability_curves.csv) @@ -57,7 +57,7 @@ def __init__( set_area_extraction_methods(self, method='mean') Sets the area extraction method for all damage functions. get_new_hazard_values(self, hazard_values) - Returns the new hazard values of the existing and to be added damage + Returns the new hazard values of the existing and to be added damage functions. """ self.hazard_name = f"water depth [{unit}]" @@ -279,6 +279,22 @@ def set_area_extraction_method( {k: method for k in function_selection if k in self.functions.keys()} ) + def create_step_function( + self, + name: str, + threshold_value: float = 0.6, + min_hazard_value: float = 0, + max_hazard_value: float = 10, + step_hazard_value: float = 1.0, + ): + # Create a list of hazard values from the min_hazard_value to the max_hazard_value with steps + # of step_hazard value + # 0 0.59 0.6 1 2 3 4 5 6 7 8 9 10 + # 0 0 1 1 1 1 1 1 1 1 + hazard_values = [] + fraction_values = [] + self.add(name, hazard_values, fraction_values) + def get_damage_function_names(self): return list(self.functions.keys()) @@ -583,7 +599,7 @@ def get_table(self): df = pd.DataFrame(vf_fiat_format, columns=column_names) return df - + def get_metadata(self) -> List[str]: """Retrieves the vulnerability metadata. From b6f580e1a8960f982584f8443d86a5181efc98b3 Mon Sep 17 00:00:00 2001 From: "sarah.rautenbach" Date: Fri, 20 Oct 2023 13:54:22 +0200 Subject: [PATCH 04/11] function for hazared and fraction values, create test for function --- hydromt_fiat/workflows/vulnerability.py | 26 +++++++++++++++++++------ tests/test_create_step_function.py | 14 +++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 tests/test_create_step_function.py diff --git a/hydromt_fiat/workflows/vulnerability.py b/hydromt_fiat/workflows/vulnerability.py index ca3dbbc9..0397b60b 100644 --- a/hydromt_fiat/workflows/vulnerability.py +++ b/hydromt_fiat/workflows/vulnerability.py @@ -6,6 +6,7 @@ from typing import List from pathlib import Path from typing import Union, Tuple +import math class Vulnerability: @@ -287,13 +288,26 @@ def create_step_function( max_hazard_value: float = 10, step_hazard_value: float = 1.0, ): - # Create a list of hazard values from the min_hazard_value to the max_hazard_value with steps - # of step_hazard value - # 0 0.59 0.6 1 2 3 4 5 6 7 8 9 10 - # 0 0 1 1 1 1 1 1 1 1 - hazard_values = [] - fraction_values = [] + #list before threshold + if threshold_value <= 1: + list_bt = np.arange(min_hazard_value,threshold_value, 0.01).tolist() + del list_bt[1:-1] + list_bt.append(threshold_value) + else: + list_bt = np.arange(0,threshold_value, step_hazard_value).tolist() + list_bt.append(threshold_value-0.01) + list_bt.append(threshold_value) + list_bt = [ round(elem, 2) for elem in list_bt] + + # list after threshold + list_at = np.arange(math.ceil(threshold_value), max_hazard_value+1, 1).tolist() + # merge list before and after threshold + hazard_values = list_bt +list_at + # create fraction_values + fraction_values = [0 if value < threshold_value else 1 for value in hazard_values] + self.add(name, hazard_values, fraction_values) + def get_damage_function_names(self): return list(self.functions.keys()) diff --git a/tests/test_create_step_function.py b/tests/test_create_step_function.py new file mode 100644 index 00000000..e9f23974 --- /dev/null +++ b/tests/test_create_step_function.py @@ -0,0 +1,14 @@ +# Unit test + +from hydromt_fiat.workflows.vulnerability import Vulnerability + +def test_create_step_function(): + x = Vulnerability() + name = "roads_2" + min_hazard_input = 0 + max_hazard_input = 15 + threshold_value = 0.5 + step_hazard_value = 2 + x.create_step_function() + x.create_step_function(name, threshold_value, min_hazard_input, max_hazard_input, step_hazard_value) + From 173dbf8e479816b2844b269bd57e4faac6b85262 Mon Sep 17 00:00:00 2001 From: Frederique Date: Fri, 20 Oct 2023 13:55:41 +0200 Subject: [PATCH 05/11] Update --- .../data/hydromt_fiat_catalog_USA.yml | 70 +++++++++------- .../max_potential_damages/us_road_damage.csv | 7 ++ hydromt_fiat/data_apis/open_street_maps.py | 42 ++++------ hydromt_fiat/fiat.py | 46 ++++++++--- hydromt_fiat/workflows/exposure_vector.py | 74 ++++++++++++----- hydromt_fiat/workflows/roads.py | 54 +++++++++++++ tests/test_roads.py | 80 +++++++++++++++++++ 7 files changed, 282 insertions(+), 91 deletions(-) create mode 100644 hydromt_fiat/data/max_potential_damages/us_road_damage.csv create mode 100644 tests/test_roads.py diff --git a/hydromt_fiat/data/hydromt_fiat_catalog_USA.yml b/hydromt_fiat/data/hydromt_fiat_catalog_USA.yml index 38ebb75e..72790b05 100644 --- a/hydromt_fiat/data/hydromt_fiat_catalog_USA.yml +++ b/hydromt_fiat/data/hydromt_fiat_catalog_USA.yml @@ -1,45 +1,57 @@ NSI: - path: "https://nsi.sec.usace.army.mil/nsiapi/structures?fmt=fc" - data_type: GeoDataFrame - driver: vector - crs: 4326 - translation_fn: "attribute_linking/NSI_attributes_to_FIAT.json" - meta: - category: exposure + path: "https://nsi.sec.usace.army.mil/nsiapi/structures?fmt=fc" + data_type: GeoDataFrame + driver: vector + crs: 4326 + translation_fn: "attribute_linking/NSI_attributes_to_FIAT.json" + meta: + category: exposure default_vulnerability_curves: - path: damage_functions/flooding/Hazus_IWR_curves.csv - data_type: DataFrame - driver: csv - meta: - category: vulnerability - source: HAZUS SQL database, USACE-IWR and FEMA expert elicitation curves. The source of these curves is the Draft Report Nonresidential Flood Depth-Damage Functions Derived from Expert Elicitation, April 2009, Revised 2013. FEMA Contract Number HSFEHQ-06-D-0162. Task Order HSFEHQ-08-J-0014. Not for distribution, but data shared by USACE. + path: damage_functions/flooding/Hazus_IWR_curves.csv + data_type: DataFrame + driver: csv + meta: + unit: ft + category: vulnerability + source: HAZUS SQL database, USACE-IWR and FEMA expert elicitation curves. The source of these curves is the Draft Report Nonresidential Flood Depth-Damage Functions Derived from Expert Elicitation, April 2009, Revised 2013. FEMA Contract Number HSFEHQ-06-D-0162. Task Order HSFEHQ-08-J-0014. Not for distribution, but data shared by USACE. jrc_vulnerability_curves: - path: damage_functions/flooding/JRC_damage_functions.xlsx - data_type: DataFrame - driver: xlsx - meta: - category: vulnerability + path: damage_functions/flooding/JRC_damage_functions.xlsx + data_type: DataFrame + driver: xlsx + meta: + unit: m + category: vulnerability hazus_max_potential_damages: - path: max_potential_damages/damage_values_fema_hazus-inventory-technical-manual-4.2.3.xlsx - data_type: DataFrame - driver: xlsx - meta: - category: vulnerability + path: max_potential_damages/damage_values_fema_hazus-inventory-technical-manual-4.2.3.xlsx + data_type: DataFrame + driver: xlsx + meta: + unit: ft + category: vulnerability social_vulnerability: - path: social_vulnerability/census_vulnerability_data_codebook.xlsx - data_type: DataFrame - driver: xlsx - meta: - category: social_vulnerability + path: social_vulnerability/census_vulnerability_data_codebook.xlsx + data_type: DataFrame + driver: xlsx + meta: + category: social_vulnerability default_hazus_iwr_linking: path: vulnerability_linking/default_hazus_iwr_curve_linking.csv data_type: DataFrame driver: csv meta: - category: vulnerability \ No newline at end of file + category: vulnerability + +default_road_max_potential_damages: + path: max_potential_damages/us_road_damage.csv + data_type: DataFrame + driver: csv + meta: + unit: ft + category: exposure + source: Bouwer, Laurens & Haasnoot, Marjolijn & Wagenaar, Dennis & Roscoe, Kathryn. (2018). Assessment of alternative flood mitigation strategies for the C-7 Basin in Miami, Florida. \ No newline at end of file diff --git a/hydromt_fiat/data/max_potential_damages/us_road_damage.csv b/hydromt_fiat/data/max_potential_damages/us_road_damage.csv new file mode 100644 index 00000000..54ceaf71 --- /dev/null +++ b/hydromt_fiat/data/max_potential_damages/us_road_damage.csv @@ -0,0 +1,7 @@ +lanes,cost [USD/ft] +1,240 +2,240 +3,360 +4,480 +5,600 +6,720 diff --git a/hydromt_fiat/data_apis/open_street_maps.py b/hydromt_fiat/data_apis/open_street_maps.py index b53b07cd..de09c87e 100644 --- a/hydromt_fiat/data_apis/open_street_maps.py +++ b/hydromt_fiat/data_apis/open_street_maps.py @@ -1,16 +1,17 @@ import osmnx as ox import logging from shapely.geometry import Polygon +import geopandas as gpd +from typing import Union, List -def get_assets_from_osm(polygon: Polygon): +def get_assets_from_osm(polygon: Polygon) -> gpd.GeoDataFrame: tags = {"building": True} # this is the tag we use to find the correct OSM data footprints = ox.features.features_from_polygon( polygon, tags ) # then we query the data if footprints.empty: - logging.warning("No buildings found from OSM") return None logging.info(f"Total number of buildings found from OSM: {len(footprints)}") @@ -26,40 +27,37 @@ def get_assets_from_osm(polygon: Polygon): def get_roads_from_osm( - polygon: Union[Polygon, Path, str] -) ->gpd.GeoDataFrame: + polygon: Polygon, + road_types: Union[str, List[str], bool] = True, +) -> gpd.GeoDataFrame: + if isinstance(road_types, str): + road_types = [road_types] + tag = { - "highway": ["motorway"] + "highway": road_types } # this is the tag we use to find the correct OSM data - if isinstance(polygon,(Path,str)): - gdf = gpd.read_file(polygon) - polygon =gdf.geometry.iloc[0] - else: - return polygon roads = ox.features.features_from_polygon( polygon, tags=tag ) # then we query the data if roads.empty: - logging.warning("No road network found from OSM") return None logging.info(f"Total number of roads found from OSM: {len(roads)}") # Not sure if this is needed here and maybe filter for the columns that we need roads = roads.loc[ - (roads.geometry.type == "Polygon") - | (roads.geometry.type == "MultiPolygon") - | (roads.geometry.type == "LineString") + (roads.geometry.type == "LineString") + | (roads.geometry.type == "MultiLineString") ] roads = roads.reset_index(drop=True) - roads.rename(columns={"element_type": "type"}, inplace=True) + roads = roads.loc[:, ["highway", "name", "lanes", "geometry"]] return roads -def get_landuse_from_osm(polygon: Polygon): +def get_landuse_from_osm(polygon: Polygon) -> gpd.GeoDataFrame: tags = {"landuse": True} # this is the tag we use to find the correct OSM data landuse = ox.features.features_from_polygon(polygon, tags) # then we query the data @@ -79,18 +77,6 @@ def get_landuse_from_osm(polygon: Polygon): return landuse -if __name__ == "__main__": - _polygon = Polygon( - [ - [-80.21997289327112, 25.83897611793664], - [-80.21997289327112, 25.86427542636784], - [-80.24609330530801, 25.86427542636784], - [-80.24609330530801, 25.83897611793664], - [-80.21997289327112, 25.83897611793664], - ] - ) - get_landuse_from_osm(_polygon) - # # Do a spatial join to connect the buildings with the classes # # (here we use a buffer area of 100 m around the classes in case there buildings falling completely out of the classes) # # a more comprehensive spatial join to account for overlapping area can be used as well diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 8abd933b..75d09603 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -252,19 +252,25 @@ def setup_vulnerability_from_csv(self, csv_fn: Union[str, Path], unit: str) -> N self.vulnerability.from_csv(csv_fn) def setup_road_vulnerability( - self, - unit: str, - threshold_value: float = 0.6, - min_hazard_value: float = 0, - max_hazard_value: float = 10, - step_hazard_value: float = 1.0, - ): + self, + unit: str, + threshold_value: float = 0.6, + min_hazard_value: float = 0, + max_hazard_value: float = 10, + step_hazard_value: float = 1.0, + ): if not self.vulnerability: self.vulnerability = Vulnerability( unit, self.logger, ) - self.vulnerability.create_step_function("roads", threshold_value, min_hazard_value, max_hazard_value, step_hazard_value) + self.vulnerability.create_step_function( + "roads", + threshold_value, + min_hazard_value, + max_hazard_value, + step_hazard_value, + ) def setup_exposure_buildings( self, @@ -278,7 +284,7 @@ def setup_exposure_buildings( damage_types: Union[List[str], None] = ["structure", "content"], country: Union[str, None] = None, ) -> None: - """Setup vector exposure data for Delft-FIAT. + """Setup building exposure (vector) data for Delft-FIAT. Parameters ---------- @@ -350,16 +356,30 @@ def setup_exposure_buildings( self.set_config("exposure.geom.crs", self.exposure.crs) self.set_config("exposure.geom.unit", unit) - def setup_exposure_roads(self): - NotImplemented + def setup_exposure_roads( + self, + roads_fn: Union[str, Path], + road_damage: Union[str, Path], + road_types: Union[str, List[str], bool] = True, + unit: str = "m", + ): + """Setup road exposure data for Delft-FIAT. - # First you get the data + Parameters + ---------- + roads_fn : Union[str, Path] + Path to the road network source (e.g., OSM) or file. + road_types : Union[str, List[str], bool], optional + List of road types to include in the exposure data, by default True + """ + if not self.exposure: + self.exposure = ExposureVector(self.data_catalog, self.logger, self.region, unit=unit) + self.exposure.setup_roads(roads_fn, road_damage, road_types) # Link to vulnerability curves # Combine the exposure database with pre-existing exposure data if available - def setup_exposure_raster(self): """Setup raster exposure data for Delft-FIAT. This function will be implemented at a later stage. diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index c6093426..975b52b8 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -13,13 +13,23 @@ from hydromt_fiat.data_apis.open_street_maps import ( get_assets_from_osm, get_landuse_from_osm, + get_roads_from_osm, ) -from hydromt_fiat.workflows.damage_values import preprocess_jrc_damage_values, preprocess_hazus_damage_values +from hydromt_fiat.workflows.damage_values import ( + preprocess_jrc_damage_values, + preprocess_hazus_damage_values, +) from hydromt_fiat.workflows.exposure import Exposure from hydromt_fiat.workflows.utils import detect_delimiter from hydromt_fiat.workflows.vulnerability import Vulnerability -from hydromt_fiat.workflows.gis import get_area, sjoin_largest_area, get_crs_str_from_gdf, join_spatial_data +from hydromt_fiat.workflows.gis import ( + get_area, + sjoin_largest_area, + get_crs_str_from_gdf, + join_spatial_data, +) +from hydromt_fiat.workflows.roads import add_max_potential_damage_roads class ExposureVector(Exposure): @@ -182,35 +192,54 @@ def setup_buildings_from_single_source( # Set the name to the geom_names self.set_geom_names("buildings") - def setup_roads(self, source: Union[str, Path]): - NotImplemented + def setup_roads( + self, + source: Union[str, Path], + road_damage: Union[str, Path], + road_types: Union[str, List[str], bool] = True, + ): self.logger.info("Setting up roads...") if str(source).upper() == "OSM": polygon = self.region.iloc[0].values[0] - roads = get_roads_from_osm(polygon) + roads = get_roads_from_osm(polygon, road_types) - if assets.empty: + if roads.empty: self.logger.warning( - "No assets found in the selected region from source " - f"{source}." + "No roads found in the selected region from source " f"{source}." ) - # Rename the osmid column to Object ID - assets.rename(columns={"osmid": "Object ID"}, inplace=True) - else: - assets = self.data_catalog.get_geodataframe( - source, geom=self.region + # Rename the columns to FIAT names + roads.rename( + columns={"highway": "Secondary Object Type", "name": "Object Name"}, + inplace=True, ) + + # Add an Object ID + roads["Object ID"] = range(1, len(roads.index) + 1) + else: + roads = self.data_catalog.get_geodataframe(source, geom=self.region) # add the function to segmentize the roads into certain segments - # Primary Object Type = roads - # Secondary Object Type = road type + # Add the Primary Object Type and damage function, which is currently not set up to be flexible + roads["Primary Object Type"] = "road" + roads["Damage Function: Structure"] = "road" + + self.logger.info( + "The damage function 'road' is selected for all of the structure damage to the roads." + ) # Add the max potential damage to the roads - - self.set_exposure_geoms(roads) + road_damage = self.data_catalog.get_dataframe(road_damage) + add_max_potential_damage_roads(roads, road_damage) + + self.set_exposure_geoms(roads[["Object ID", "geometry"]]) self.set_geom_names("roads") + del roads["geometry"] + + # Update the exposure_db + self.exposure_db = pd.concat([self.exposure_db, roads]).reset_index(drop=True) + def setup_buildings_from_multiple_sources( self, asset_locations: Union[str, Path], @@ -482,7 +511,9 @@ def setup_ground_floor_height( gfh = self.data_catalog.get_geodataframe(ground_floor_height) gdf = self.get_full_gdf(self.exposure_db) gdf = join_spatial_data(gdf, gfh, attr_name, method) - gdf = self._set_values_from_other_column(gdf, "Ground Floor Height", attr_name) + gdf = self._set_values_from_other_column( + gdf, "Ground Floor Height", attr_name + ) elif isinstance(ground_floor_height, list): # Multiple files are used to assign the ground floor height to the assets NotImplemented @@ -626,8 +657,9 @@ def raise_ground_floor_height( "Raising the ground floor height of the properties relative to Datum." ) self.exposure_db.loc[ - (self.exposure_db["Ground Floor Height"] < raise_by) & self.exposure_db.index.isin(idx), - "Ground Floor Height", + (self.exposure_db["Ground Floor Height"] < raise_by) + & self.exposure_db.index.isin(idx), + "Ground Floor Height", ] = raise_by elif height_reference.lower() in ["geom", "table"]: @@ -1329,7 +1361,7 @@ def set_height_relative_to_reference( def _set_values_from_other_column( df: Union[pd.DataFrame, gpd.GeoDataFrame], col_to_set: str, col_to_copy: str ) -> Union[pd.DataFrame, gpd.GeoDataFrame]: - """Sets the values of to where the values of are + """Sets the values of to where the values of are nan and deletes . """ df.loc[df[col_to_copy].notna(), col_to_set] = df.loc[ diff --git a/hydromt_fiat/workflows/roads.py b/hydromt_fiat/workflows/roads.py index e69de29b..c3d93963 100644 --- a/hydromt_fiat/workflows/roads.py +++ b/hydromt_fiat/workflows/roads.py @@ -0,0 +1,54 @@ +import pandas as pd +import geopandas as gpd +from hydromt.gis_utils import utm_crs +import numpy as np + + +def add_max_potential_damage_roads( + roads: gpd.GeoDataFrame, road_damage: pd.DataFrame +) -> gpd.GeoDataFrame: + if roads.crs.is_geographic: + # If the CRS is geographic, reproject to the nearest UTM zone + nearest_utm = utm_crs(roads.total_bounds) + roads_utm = roads.to_crs(nearest_utm) + + roads = gpd.GeoDataFrame( + { + "highway": roads["highway"], + "lanes": pd.to_numeric(roads["lanes"], errors="coerce"), + "segment_length": roads_utm.length, + "geometry": roads["geometry"], + } + ) + # Create dictionary of damages per number of lanes + damage_dic = dict(zip(road_damage["lanes"], road_damage["cost [USD/ft]"])) + + damage_per_foot_row = [] + # Step 3: Iterate through the DataFrame column and retrieve corresponding values + for index, row in roads.iterrows(): + # print(row['lanes']) + if np.isnan(row["lanes"]) or row["lanes"] == 0: + row["lanes"] = 1 + else: + row["lanes"] = row["lanes"] + if row["lanes"] in damage_dic: + damage_per_foot = damage_dic[row["lanes"]] + damage_per_foot_row.append(damage_per_foot) + else: + print("No lane found") + + # Potentially convert the length to meters + if unit == "foot" or unit == "feet" or unit == "ft": + roads["maximum_potential_damage"] = ( + damage_per_foot * roads["segment_length"] + ) + elif unit == "meter" or unit == "metre" or unit == "m": + roads["maximum_potential_damage"] = ( + damage_per_foot * roads["segment_length"] * 0.3048 + ) + else: + print( + "You are using wrong unit for the segment length. Please use <'foot'> or <'meter/metre/m'>" + ) + + return roads diff --git a/tests/test_roads.py b/tests/test_roads.py new file mode 100644 index 00000000..b81e1bc8 --- /dev/null +++ b/tests/test_roads.py @@ -0,0 +1,80 @@ +from hydromt_fiat.fiat import FiatModel +from hydromt.log import setuplog +from pathlib import Path +import pytest +import shutil +import geopandas as gpd + + +EXAMPLEDIR = Path( + "P:/11207949-dhs-phaseii-floodadapt/Model-builder/Delft-FIAT/local_test_database" +) +DATADIR = Path().absolute() / "hydromt_fiat" / "data" + +_region = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [-79.92169686568795, 32.768208904171374], + [-79.92169686568795, 32.77745096033627], + [-79.94881762529997, 32.77745096033627], + [-79.94881762529997, 32.768208904171374], + [-79.92169686568795, 32.768208904171374], + ] + ], + "type": "Polygon", + }, + } + ], +} +_cases = { + "roads_from_OSM": { + "data_catalogue": DATADIR / "hydromt_fiat_catalog_USA.yml", + "dir": "test_roads_from_OSM", + "configuration": { + "setup_exposure_roads": { + "roads_fn": "OSM", + "road_types": ["motorway", "primary", "secondary", "tertiary"], + "road_damage": "default_road_max_potential_damages", + "unit": "ft", + }, + }, + "region": _region, + }, +} + + +@pytest.mark.parametrize("case", list(_cases.keys())) +def test_setup_roads(case): + # Read model in examples folder. + root = EXAMPLEDIR.joinpath(_cases[case]["dir"]) + if root.exists(): + shutil.rmtree(root) + data_catalog_yml = str(_cases[case]["data_catalogue"]) + + logger = setuplog("hydromt_fiat", log_level=10) + + fm = FiatModel(root=root, mode="w", data_libs=[data_catalog_yml], logger=logger) + + region = gpd.GeoDataFrame.from_features(_cases[case]["region"], crs=4326) + fm.build(region={"geom": region}, opt=_cases[case]["configuration"]) + fm.write() + + # Check if the exposure data exists + assert root.joinpath("exposure", "roads.gpkg").exists() + assert root.joinpath("exposure", "exposure.csv").exists() + assert root.joinpath("exposure", "region.gpkg").exists() + + # Check if the vulnerability data exists + # assert root.joinpath("vulnerability", "vulnerability_curves.csv").exists() + + # Check if the hazard folder exists + assert root.joinpath("hazard").exists() + + # Check if the output data folder exists + assert root.joinpath("output").exists() From 84441b75db5354f81701413bf5d2205f949f78f0 Mon Sep 17 00:00:00 2001 From: Frederique Date: Fri, 20 Oct 2023 14:17:48 +0200 Subject: [PATCH 06/11] update test --- tests/test_roads.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_roads.py b/tests/test_roads.py index b81e1bc8..b35dcdd3 100644 --- a/tests/test_roads.py +++ b/tests/test_roads.py @@ -37,6 +37,9 @@ "data_catalogue": DATADIR / "hydromt_fiat_catalog_USA.yml", "dir": "test_roads_from_OSM", "configuration": { + "setup_road_vulnerability": { + "" + }, "setup_exposure_roads": { "roads_fn": "OSM", "road_types": ["motorway", "primary", "secondary", "tertiary"], From 1af4cc58338ed25e8ab1b9a4724532f4722db551 Mon Sep 17 00:00:00 2001 From: "sarah.rautenbach" Date: Fri, 20 Oct 2023 14:19:39 +0200 Subject: [PATCH 07/11] add unit test road --- hydromt_fiat/workflows/vulnerability.py | 6 +++--- tests/test_create_step_function.py | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hydromt_fiat/workflows/vulnerability.py b/hydromt_fiat/workflows/vulnerability.py index 0397b60b..45e4855c 100644 --- a/hydromt_fiat/workflows/vulnerability.py +++ b/hydromt_fiat/workflows/vulnerability.py @@ -307,7 +307,7 @@ def create_step_function( fraction_values = [0 if value < threshold_value else 1 for value in hazard_values] self.add(name, hazard_values, fraction_values) - + def get_damage_function_names(self): return list(self.functions.keys()) @@ -663,10 +663,10 @@ def interpolate_damage_function( df_output.interpolate(method="index", limit_area="inside", inplace=True) # Second: fill the nan values before the first value per column (with 0) - df_output.fillna(method="bfill", inplace=True) + df_output.bfill(inplace=True) # Third: fill the nan values after the later value per column (with the same value as the last value) - df_output.fillna(method="ffill", inplace=True) + df_output.ffill(inplace=True) # Reset the index to a column df_output.reset_index(inplace=True) diff --git a/tests/test_create_step_function.py b/tests/test_create_step_function.py index e9f23974..4e3b3b7d 100644 --- a/tests/test_create_step_function.py +++ b/tests/test_create_step_function.py @@ -9,6 +9,10 @@ def test_create_step_function(): max_hazard_input = 15 threshold_value = 0.5 step_hazard_value = 2 - x.create_step_function() + x.create_step_function("roads") x.create_step_function(name, threshold_value, min_hazard_input, max_hazard_input, step_hazard_value) - + + + + assert x.functions == {'roads': [0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'roads_2': [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]} + assert x.hazard_values == [0.0, 0.49, 0.5, 0.59, 0.6, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] \ No newline at end of file From 7634eb0c1225e26d2d6d3484a8b5b6c8750da2ca Mon Sep 17 00:00:00 2001 From: "sarah.rautenbach" Date: Fri, 20 Oct 2023 14:43:55 +0200 Subject: [PATCH 08/11] update function and test --- hydromt_fiat/workflows/vulnerability.py | 2 +- tests/test_roads.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hydromt_fiat/workflows/vulnerability.py b/hydromt_fiat/workflows/vulnerability.py index 45e4855c..6b3e1693 100644 --- a/hydromt_fiat/workflows/vulnerability.py +++ b/hydromt_fiat/workflows/vulnerability.py @@ -300,7 +300,7 @@ def create_step_function( list_bt = [ round(elem, 2) for elem in list_bt] # list after threshold - list_at = np.arange(math.ceil(threshold_value), max_hazard_value+1, 1).tolist() + list_at = np.arange(math.ceil(threshold_value), max_hazard_value+1, step_hazard_value).tolist() # merge list before and after threshold hazard_values = list_bt +list_at # create fraction_values diff --git a/tests/test_roads.py b/tests/test_roads.py index b35dcdd3..e0600339 100644 --- a/tests/test_roads.py +++ b/tests/test_roads.py @@ -38,7 +38,11 @@ "dir": "test_roads_from_OSM", "configuration": { "setup_road_vulnerability": { - "" + "name" : "roads", + "theshold_value" : 0.5, + "min_hazard__value" : 0, + "max_hazard_value" :15, + "step_hazard_value" : 1, }, "setup_exposure_roads": { "roads_fn": "OSM", @@ -81,3 +85,7 @@ def test_setup_roads(case): # Check if the output data folder exists assert root.joinpath("output").exists() + + # Check if the output gives the correct solution + assert fm.functions == {'roads': [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]} + assert fm.hazard_values == [0.0, 0.49, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] \ No newline at end of file From 14ec83aa8b3f40a6f16bce5eae25f26df756aa7f Mon Sep 17 00:00:00 2001 From: Frederique Date: Fri, 20 Oct 2023 14:51:40 +0200 Subject: [PATCH 09/11] update, roads test passed succesfully --- hydromt_fiat/workflows/exposure_vector.py | 4 +-- hydromt_fiat/workflows/roads.py | 38 ++++++++--------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index 975b52b8..ef442012 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -29,7 +29,7 @@ get_crs_str_from_gdf, join_spatial_data, ) -from hydromt_fiat.workflows.roads import add_max_potential_damage_roads +from hydromt_fiat.workflows.roads import get_max_potential_damage_roads class ExposureVector(Exposure): @@ -230,7 +230,7 @@ def setup_roads( # Add the max potential damage to the roads road_damage = self.data_catalog.get_dataframe(road_damage) - add_max_potential_damage_roads(roads, road_damage) + get_max_potential_damage_roads(roads, road_damage) self.set_exposure_geoms(roads[["Object ID", "geometry"]]) self.set_geom_names("roads") diff --git a/hydromt_fiat/workflows/roads.py b/hydromt_fiat/workflows/roads.py index c3d93963..9f2d84cd 100644 --- a/hydromt_fiat/workflows/roads.py +++ b/hydromt_fiat/workflows/roads.py @@ -4,51 +4,39 @@ import numpy as np -def add_max_potential_damage_roads( +def get_max_potential_damage_roads( roads: gpd.GeoDataFrame, road_damage: pd.DataFrame ) -> gpd.GeoDataFrame: if roads.crs.is_geographic: # If the CRS is geographic, reproject to the nearest UTM zone nearest_utm = utm_crs(roads.total_bounds) - roads_utm = roads.to_crs(nearest_utm) + roads = roads.to_crs(nearest_utm) + + unit = roads.crs.axis_info[0].unit_name roads = gpd.GeoDataFrame( { - "highway": roads["highway"], "lanes": pd.to_numeric(roads["lanes"], errors="coerce"), - "segment_length": roads_utm.length, + "segment_length": roads.length, "geometry": roads["geometry"], } ) # Create dictionary of damages per number of lanes damage_dic = dict(zip(road_damage["lanes"], road_damage["cost [USD/ft]"])) - damage_per_foot_row = [] # Step 3: Iterate through the DataFrame column and retrieve corresponding values - for index, row in roads.iterrows(): - # print(row['lanes']) - if np.isnan(row["lanes"]) or row["lanes"] == 0: - row["lanes"] = 1 - else: - row["lanes"] = row["lanes"] - if row["lanes"] in damage_dic: - damage_per_foot = damage_dic[row["lanes"]] - damage_per_foot_row.append(damage_per_foot) - else: - print("No lane found") + roads["lanes"] = [ + 1 if (np.isnan(lane)) or (lane == 0) else lane for lane in list(roads["lanes"]) + ] + roads["damage_value"] = roads["lanes"].map(damage_dic) # Potentially convert the length to meters - if unit == "foot" or unit == "feet" or unit == "ft": - roads["maximum_potential_damage"] = ( - damage_per_foot * roads["segment_length"] - ) - elif unit == "meter" or unit == "metre" or unit == "m": - roads["maximum_potential_damage"] = ( - damage_per_foot * roads["segment_length"] * 0.3048 - ) + roads["maximum_potential_damage"] = roads["damage_value"] * roads["segment_length"] + if unit == "meter" or unit == "metre" or unit == "m": + roads["maximum_potential_damage"] = roads["maximum_potential_damage"] * 0.3048 else: print( - "You are using wrong unit for the segment length. Please use <'foot'> or <'meter/metre/m'>" + "You are using the wrong unit for the segment length. Please use <'foot/feet/ft'> or <'meter/metre/m'>" ) return roads From 4d2460bce03983d318405d132d9f22b478a91f33 Mon Sep 17 00:00:00 2001 From: "sarah.rautenbach" Date: Fri, 20 Oct 2023 14:51:40 +0200 Subject: [PATCH 10/11] update test roads --- tests/test_roads.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_roads.py b/tests/test_roads.py index e0600339..04d55b3d 100644 --- a/tests/test_roads.py +++ b/tests/test_roads.py @@ -78,7 +78,7 @@ def test_setup_roads(case): assert root.joinpath("exposure", "region.gpkg").exists() # Check if the vulnerability data exists - # assert root.joinpath("vulnerability", "vulnerability_curves.csv").exists() + assert root.joinpath("vulnerability", "vulnerability_curves.csv").exists() # Check if the hazard folder exists assert root.joinpath("hazard").exists() @@ -87,5 +87,5 @@ def test_setup_roads(case): assert root.joinpath("output").exists() # Check if the output gives the correct solution - assert fm.functions == {'roads': [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]} - assert fm.hazard_values == [0.0, 0.49, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] \ No newline at end of file + assert fm.vulnerability.functions == {'roads': [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]} + assert fm.vulnerability.hazard_values == [0.0, 0.49, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] \ No newline at end of file From e396b5d0d8d9f4220ef0b005f8a0dcfd48b133ee Mon Sep 17 00:00:00 2001 From: Frederique Date: Fri, 20 Oct 2023 15:13:20 +0200 Subject: [PATCH 11/11] Tested and complete for now --- hydromt_fiat/fiat.py | 4 ++-- hydromt_fiat/workflows/exposure_vector.py | 10 +++++----- hydromt_fiat/workflows/roads.py | 2 +- hydromt_fiat/workflows/vulnerability.py | 2 ++ tests/test_roads.py | 18 +++++++++++++----- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/hydromt_fiat/fiat.py b/hydromt_fiat/fiat.py index 75d09603..b834729d 100644 --- a/hydromt_fiat/fiat.py +++ b/hydromt_fiat/fiat.py @@ -253,7 +253,7 @@ def setup_vulnerability_from_csv(self, csv_fn: Union[str, Path], unit: str) -> N def setup_road_vulnerability( self, - unit: str, + vertical_unit: str, threshold_value: float = 0.6, min_hazard_value: float = 0, max_hazard_value: float = 10, @@ -261,7 +261,7 @@ def setup_road_vulnerability( ): if not self.vulnerability: self.vulnerability = Vulnerability( - unit, + vertical_unit, self.logger, ) self.vulnerability.create_step_function( diff --git a/hydromt_fiat/workflows/exposure_vector.py b/hydromt_fiat/workflows/exposure_vector.py index ef442012..f5fa3010 100644 --- a/hydromt_fiat/workflows/exposure_vector.py +++ b/hydromt_fiat/workflows/exposure_vector.py @@ -221,16 +221,16 @@ def setup_roads( # add the function to segmentize the roads into certain segments # Add the Primary Object Type and damage function, which is currently not set up to be flexible - roads["Primary Object Type"] = "road" - roads["Damage Function: Structure"] = "road" + roads["Primary Object Type"] = "roads" + roads["Damage Function: Structure"] = "roads" self.logger.info( - "The damage function 'road' is selected for all of the structure damage to the roads." + "The damage function 'roads' is selected for all of the structure damage to the roads." ) - # Add the max potential damage to the roads + # Add the max potential damage and the length of the segments to the roads road_damage = self.data_catalog.get_dataframe(road_damage) - get_max_potential_damage_roads(roads, road_damage) + roads[["Max Potential Damage: Structure", "Segment Length [m]"]] = get_max_potential_damage_roads(roads, road_damage) self.set_exposure_geoms(roads[["Object ID", "geometry"]]) self.set_geom_names("roads") diff --git a/hydromt_fiat/workflows/roads.py b/hydromt_fiat/workflows/roads.py index 9f2d84cd..a7924f14 100644 --- a/hydromt_fiat/workflows/roads.py +++ b/hydromt_fiat/workflows/roads.py @@ -39,4 +39,4 @@ def get_max_potential_damage_roads( "You are using the wrong unit for the segment length. Please use <'foot/feet/ft'> or <'meter/metre/m'>" ) - return roads + return roads[["maximum_potential_damage", "segment_length"]] diff --git a/hydromt_fiat/workflows/vulnerability.py b/hydromt_fiat/workflows/vulnerability.py index 6b3e1693..d36bf891 100644 --- a/hydromt_fiat/workflows/vulnerability.py +++ b/hydromt_fiat/workflows/vulnerability.py @@ -308,6 +308,8 @@ def create_step_function( self.add(name, hazard_values, fraction_values) + # Set the area extraction method + self.set_area_extraction_methods() def get_damage_function_names(self): return list(self.functions.keys()) diff --git a/tests/test_roads.py b/tests/test_roads.py index 04d55b3d..8d8d48c1 100644 --- a/tests/test_roads.py +++ b/tests/test_roads.py @@ -4,6 +4,7 @@ import pytest import shutil import geopandas as gpd +import pandas as pd EXAMPLEDIR = Path( @@ -38,11 +39,11 @@ "dir": "test_roads_from_OSM", "configuration": { "setup_road_vulnerability": { - "name" : "roads", - "theshold_value" : 0.5, - "min_hazard__value" : 0, - "max_hazard_value" :15, - "step_hazard_value" : 1, + "vertical_unit": "ft", + "threshold_value": 0.5, + "min_hazard_value": 0, + "max_hazard_value": 15, + "step_hazard_value": 1, }, "setup_exposure_roads": { "roads_fn": "OSM", @@ -77,6 +78,13 @@ def test_setup_roads(case): assert root.joinpath("exposure", "exposure.csv").exists() assert root.joinpath("exposure", "region.gpkg").exists() + # Read the resulting exposure data and check if the required columns exist + exposure = pd.read_csv(root.joinpath("exposure", "exposure.csv")) + required_columns = ['Secondary Object Type', 'Object Name', 'lanes', 'Object ID', + 'Primary Object Type', 'Damage Function: Structure', + 'Max Potential Damage: Structure', 'Segment Length [m]'] + assert set(required_columns) == set(exposure.columns) + # Check if the vulnerability data exists assert root.joinpath("vulnerability", "vulnerability_curves.csv").exists()