diff --git a/doc/tutorial/climada_engine_Forecast.ipynb b/doc/tutorial/climada_engine_Forecast.ipynb index 2ab5bb841..29c9a5930 100644 --- a/doc/tutorial/climada_engine_Forecast.ipynb +++ b/doc/tutorial/climada_engine_Forecast.ipynb @@ -257,7 +257,9 @@ "### generate exposure\n", "# find out which hazard coord to consider\n", "CHE_borders = u_plot._get_borders(\n", - " np.stack([exposure.latitude, exposure.longitude], axis=1)\n", + " np.stack(\n", + " [exposure.gdf[\"latitude\"].values, exposure.gdf[\"longitude\"].values], axis=1\n", + " )\n", ")\n", "centroid_selection = np.logical_and(\n", " np.logical_and(\n", diff --git a/doc/tutorial/climada_engine_Impact.ipynb b/doc/tutorial/climada_engine_Impact.ipynb index a342a43b3..37b2b2f7f 100644 --- a/doc/tutorial/climada_engine_Impact.ipynb +++ b/doc/tutorial/climada_engine_Impact.ipynb @@ -1486,7 +1486,9 @@ "\n", "# Set Hazard in Exposures points\n", "# set centroids from exposures coordinates\n", - "centr_pnt = Centroids.from_lat_lon(exp_pnt.latitude, exp_pnt.longitude, exp_pnt.crs)\n", + "centr_pnt = Centroids.from_lat_lon(\n", + " exp_pnt.gdf[\"latitude\"].values, exp_pnt.gdf[\"longitude\"].values, exp_pnt.crs\n", + ")\n", "# compute Hazard in that centroids\n", "tr_pnt = TCTracks.from_ibtracs_netcdf(storm_id=\"2007314N10093\")\n", "tc_pnt = TropCyclone.from_tracks(tr_pnt, centroids=centr_pnt)\n", @@ -2005,7 +2007,9 @@ "\n", "# compute sequence of hazards using TropCyclone video_intensity method\n", "exp_sea = add_sea(exp_video, (100, 5))\n", - "centr_video = Centroids.from_lat_lon(exp_sea.latitude, exp_sea.longitude)\n", + "centr_video = Centroids.from_lat_lon(\n", + " exp_sea.gdf[\"latitude\"].values, exp_sea.gdf[\"longitude\"].values\n", + ")\n", "centr_video.check()\n", "\n", "track_name = \"2017242N16333\"\n", diff --git a/doc/tutorial/climada_entity_Exposures.ipynb b/doc/tutorial/climada_entity_Exposures.ipynb index a57079ef2..f8b93eec9 100644 --- a/doc/tutorial/climada_entity_Exposures.ipynb +++ b/doc/tutorial/climada_entity_Exposures.ipynb @@ -173,7 +173,7 @@ "exp_df = DataFrame()\n", "n_exp = 100 * 100\n", "# provide value\n", - "exp_df[\"value\"] = np.random.random_sample(n_exp)\n", + "exp_df[\"value\"] = np.arange(n_exp)\n", "# provide latitude and longitude\n", "lat, lon = np.mgrid[\n", " 15 : 35 : complex(0, np.sqrt(n_exp)), 20 : 40 : complex(0, np.sqrt(n_exp))\n", @@ -324,13 +324,39 @@ "# In this case, we only specify the IDs for tropical cyclone (TC); here, each exposure entry will be treated with\n", "# the same impact function: the one that has ID '1':\n", "# Of course, this will only be relevant at later steps during impact calculations.\n", - "exp_df[\"impf_TC\"] = np.ones(n_exp, int)\n", - "exp_df" + "exp_df[\"impf_TC\"] = np.ones(n_exp, int)" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "exp_df is a DataFrame: \n", + "exp_df looks like:\n", + " value latitude longitude impf_TC\n", + "0 0 15.0 20.000000 1\n", + "1 1 15.0 20.202020 1\n", + "2 2 15.0 20.404040 1\n", + "3 3 15.0 20.606061 1\n", + "4 4 15.0 20.808081 1\n" + ] + } + ], + "source": [ + "# Let's have a look at the pandas DataFrame\n", + "print(\"exp_df is a DataFrame:\", str(type(exp_df)))\n", + "print(\"exp_df looks like:\")\n", + "print(exp_df.head())" + ] + }, + { + "cell_type": "code", + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -379,6 +405,7 @@ ], "source": [ "# let's have a look at the Exposures instance we created!\n", + "print(\"\\n\" + \"exp looks like:\")\n", "print(exp)" ] }, @@ -413,7 +440,10 @@ "from climada.entity import Exposures\n", "\n", "# Read spatial info from an external file into GeoDataFrame\n", - "world = gpd.read_file(gpd.datasets.get_path(\"naturalearth_cities\"))" + "world = gpd.read_file(gpd.datasets.get_path(\"naturalearth_cities\"))\n", + "print(\"World is a GeoDataFrame:\", str(type(world)))\n", + "print(\"World looks like:\")\n", + "print(world.head())" ] }, { @@ -447,7 +477,11 @@ "world[\"value\"] = np.arange(n_exp)\n", "# Convert GeoDataFrame into Exposure instance\n", "exp_gpd = Exposures(world)\n", - "print(exp_gpd)" + "print(\"\\n\" + \"exp_gpd is an Exposures:\", str(type(exp_gpd)))\n", + "# add random values to entries\n", + "exp_gpd.gdf[\"value\"] = np.arange(world.shape[0])\n", + "# set latitude and longitude attributes from geometry\n", + "exp_gpd.set_lat_lon()" ] }, { @@ -481,7 +515,73 @@ "# In this case, we only specify the IDs for tropical cyclone (TC); here, each exposure entry will be treated with\n", "# the same impact function: the one that has ID '1':\n", "# Of course, this will only be relevant at later steps during impact calculations.\n", - "exp_gpd.data[\"impf_TC\"] = np.ones(world.shape[0], int)\n", + "exp_gpd.gdf[\"impf_TC\"] = np.ones(world.shape[0], int)\n", + "print(\"\\n\" + \"check method logs:\")\n", + "\n", + "# as always, run check method to assign meta-data and check for missing mandatory variables.\n", + "exp_gpd.check()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\u001b[1;03;30;30mexp_gpd looks like:\u001b[0m\n", + "ref_year: 2018\n", + "value_unit: USD\n", + "meta: {'crs': \n", + "Name: WGS 84\n", + "Axis Info [ellipsoidal]:\n", + "- Lat[north]: Geodetic latitude (degree)\n", + "- Lon[east]: Geodetic longitude (degree)\n", + "Area of Use:\n", + "- name: World.\n", + "- bounds: (-180.0, -90.0, 180.0, 90.0)\n", + "Datum: World Geodetic System 1984\n", + "- Ellipsoid: WGS 84\n", + "- Prime Meridian: Greenwich\n", + "}\n", + "crs: epsg:4326\n", + "data:\n", + " name geometry value latitude longitude \\\n", + "0 Vatican City POINT (12.45339 41.90328) 0 41.903282 12.453387 \n", + "1 San Marino POINT (12.44177 43.93610) 1 43.936096 12.441770 \n", + "2 Vaduz POINT (9.51667 47.13372) 2 47.133724 9.516669 \n", + "3 Luxembourg POINT (6.13000 49.61166) 3 49.611660 6.130003 \n", + "4 Palikir POINT (158.14997 6.91664) 4 6.916644 158.149974 \n", + ".. ... ... ... ... ... \n", + "197 Cairo POINT (31.24802 30.05191) 197 30.051906 31.248022 \n", + "198 Tokyo POINT (139.74946 35.68696) 198 35.686963 139.749462 \n", + "199 Paris POINT (2.33139 48.86864) 199 48.868639 2.331389 \n", + "200 Santiago POINT (-70.66899 -33.44807) 200 -33.448068 -70.668987 \n", + "201 Singapore POINT (103.85387 1.29498) 201 1.294979 103.853875 \n", + "\n", + " impf_TC \n", + "0 1 \n", + "1 1 \n", + "2 1 \n", + "3 1 \n", + "4 1 \n", + ".. ... \n", + "197 1 \n", + "198 1 \n", + "199 1 \n", + "200 1 \n", + "201 1 \n", + "\n", + "[202 rows x 6 columns]\n" + ] + } + ], + "source": [ + "# let's have a look at the Exposures instance we created!\n", + "print(\"\\n\" + \"\\x1b[1;03;30;30m\" + \"exp_gpd looks like:\" + \"\\x1b[0m\")\n", "print(exp_gpd)" ] }, @@ -728,7 +828,7 @@ "sel_exp.data = sel_exp.data.cx[:, -5:5]\n", "\n", "print(\"\\n\" + \"sel_exp contains a subset of the original data\")\n", - "sel_exp.data" + "sel_exp.gdf.head()" ] }, { @@ -844,7 +944,7 @@ "\n", "# Let's have a look. Again, the sub-selection is a GeoDataFrame!\n", "print(\"\\n\" + \"sel_exp contains a subset of the original data\")\n", - "sel_polygon.data" + "sel_polygon.gdf" ] }, { @@ -958,7 +1058,7 @@ "print(\n", " \"the values for latitude and longitude are now according to the new coordinate system: \"\n", ")\n", - "sel_polygon.data" + "sel_polygon.gdf" ] }, { @@ -1293,7 +1393,177 @@ "source": [ "# Generate an Exposures instance from the dataframe.\n", "exp_templ = Exposures(exp_templ)\n", - "print(\"\\n\" + \"exp_templ is now an Exposures:\", exp_templ)" + "print(\"\\n\" + \"exp_templ is now an Exposures:\", str(type(exp_templ)))\n", + "\n", + "# set geometry attribute (shapely Points) from GeoDataFrame from latitude and longitude\n", + "print(\"\\n\" + \"set_geometry logs:\")\n", + "exp_templ.set_geometry_points()\n", + "# as always, run check method to include metadata and check for missing mandatory parameters\n", + "\n", + "print(\"\\n\" + \"check exp_templ:\")\n", + "exp_templ.check()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "exp_templ.gdf looks like:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
latitudelongitudevaluedeductiblecoverregion_idcategory_idimpf_TCcentr_TCimpf_FLcentr_FLgeometry
026.933899-80.1287991.392750e+1001.392750e+10111111POINT (-80.12880 26.93390)
126.957203-80.0982841.259606e+1001.259606e+10111212POINT (-80.09828 26.95720)
226.783846-80.7489471.259606e+1001.259606e+10111313POINT (-80.74895 26.78385)
326.645524-80.5507041.259606e+1001.259606e+10111414POINT (-80.55070 26.64552)
426.897796-80.5969291.259606e+1001.259606e+10111515POINT (-80.59693 26.89780)
\n", + "
" + ], + "text/plain": [ + " latitude longitude value deductible cover region_id \\\n", + "0 26.933899 -80.128799 1.392750e+10 0 1.392750e+10 1 \n", + "1 26.957203 -80.098284 1.259606e+10 0 1.259606e+10 1 \n", + "2 26.783846 -80.748947 1.259606e+10 0 1.259606e+10 1 \n", + "3 26.645524 -80.550704 1.259606e+10 0 1.259606e+10 1 \n", + "4 26.897796 -80.596929 1.259606e+10 0 1.259606e+10 1 \n", + "\n", + " category_id impf_TC centr_TC impf_FL centr_FL \\\n", + "0 1 1 1 1 1 \n", + "1 1 1 2 1 2 \n", + "2 1 1 3 1 3 \n", + "3 1 1 4 1 4 \n", + "4 1 1 5 1 5 \n", + "\n", + " geometry \n", + "0 POINT (-80.12880 26.93390) \n", + "1 POINT (-80.09828 26.95720) \n", + "2 POINT (-80.74895 26.78385) \n", + "3 POINT (-80.55070 26.64552) \n", + "4 POINT (-80.59693 26.89780) " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Let's have a look at our Exposures instance!\n", + "print(\"\\n\" + \"exp_templ.gdf looks like:\")\n", + "exp_templ.gdf.head()" ] }, { @@ -1339,7 +1609,35 @@ "name": "stdout", "output_type": "stream", "text": [ - "2024-10-04 17:19:03,725 - climada.util.coordinates - INFO - Raster from resolution 0.009000000000000341 to 0.009000000000000341.\n" + "2024-04-12 14:44:53,848 - climada.entity.exposures.base - INFO - Setting impf_ to default impact functions ids 1.\n", + "2024-04-12 14:44:53,849 - climada.entity.exposures.base - INFO - category_id not set.\n", + "2024-04-12 14:44:53,850 - climada.entity.exposures.base - INFO - cover not set.\n", + "2024-04-12 14:44:53,850 - climada.entity.exposures.base - INFO - deductible not set.\n", + "2024-04-12 14:44:53,851 - climada.entity.exposures.base - INFO - geometry not set.\n", + "2024-04-12 14:44:53,851 - climada.entity.exposures.base - INFO - region_id not set.\n", + "2024-04-12 14:44:53,852 - climada.entity.exposures.base - INFO - centr_ not set.\n", + "Meta: {'driver': 'GSBG', 'dtype': 'float32', 'nodata': 1.701410009187828e+38, 'width': 50, 'height': 60, 'count': 1, 'crs': CRS.from_epsg(4326), 'transform': Affine(0.009000000000000341, 0.0, -69.2471495969998,\n", + " 0.0, -0.009000000000000341, 10.248220966978932)}\n" + ] + } + ], + "source": [ + "# As always, run the check method, such that metadata can be assigned and checked for missing mandatory parameters.\n", + "exp_raster.check()\n", + "print(\"Meta:\", exp_raster.meta)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "exp_raster looks like:\n" ] }, { @@ -1368,7 +1666,9 @@ } ], "source": [ - "exp_raster.derive_raster()" + "# Let's have a look at the Exposures instance!\n", + "print(\"\\n\" + \"exp_raster looks like:\")\n", + "exp_raster.gdf.head()" ] }, { @@ -1708,6 +2008,51 @@ "# this generates a results folder in the current path and stores the output there\n", "save(\"exp_templ.pkl.p\", exp_templ) # creates results folder and stores there" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Dask - improving performance for big exposure\n", + "\n", + "Dask is used in some methods of CLIMADA and can be activated easily by proving the scheduler." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " value latitude longitude impf_TC\n", + "0 0 15.0 20.000000 1\n", + "1 1 15.0 20.202020 1\n", + "2 2 15.0 20.404040 1\n", + "3 3 15.0 20.606061 1\n", + "4 4 15.0 20.808081 1\n", + "CPU times: user 243 ms, sys: 116 ms, total: 359 ms\n", + "Wall time: 2.52 s\n", + " value latitude longitude impf_TC geometry\n", + "0 0 15.0 20.000000 1 POINT (20.00000 15.00000)\n", + "1 1 15.0 20.202020 1 POINT (20.20202 15.00000)\n", + "2 2 15.0 20.404040 1 POINT (20.40404 15.00000)\n", + "3 3 15.0 20.606061 1 POINT (20.60606 15.00000)\n", + "4 4 15.0 20.808081 1 POINT (20.80808 15.00000)\n" + ] + } + ], + "source": [ + "# set_geometry_points is expensive for big exposures\n", + "# for small amount of data, the execution time might be even greater when using dask\n", + "exp.gdf.drop(columns=[\"geometry\"], inplace=True)\n", + "print(exp.gdf.head())\n", + "%time exp.set_geometry_points(scheduler='processes')\n", + "print(exp.gdf.head())" + ] } ], "metadata": { diff --git a/doc/tutorial/climada_entity_LitPop.ipynb b/doc/tutorial/climada_entity_LitPop.ipynb index b41728bf2..5ad9a5ce3 100644 --- a/doc/tutorial/climada_entity_LitPop.ipynb +++ b/doc/tutorial/climada_entity_LitPop.ipynb @@ -754,6 +754,12 @@ "ent_adm0 = LitPop.from_countries(\n", " \"CHE\", res_arcsec=120, fin_mode=\"gdp\", admin1_calc=False\n", ")\n", + "ent_adm0.set_geometry_points()\n", + "\n", + "ent_adm1 = LitPop.from_countries(\n", + " \"CHE\", res_arcsec=120, fin_mode=\"gdp\", admin1_calc=True\n", + ")\n", + "\n", "ent_adm0.check()\n", "\n", "ent_adm1 = LitPop.from_countries(\n", diff --git a/doc/tutorial/climada_hazard_TropCyclone.ipynb b/doc/tutorial/climada_hazard_TropCyclone.ipynb index 47df87fb7..480d5c0b4 100644 --- a/doc/tutorial/climada_hazard_TropCyclone.ipynb +++ b/doc/tutorial/climada_hazard_TropCyclone.ipynb @@ -1895,6 +1895,7 @@ "# construct centroids\n", "min_lat, max_lat, min_lon, max_lon = 16.99375, 21.95625, -72.48125, -61.66875\n", "cent = Centroids.from_pnt_bounds((min_lon, min_lat, max_lon, max_lat), res=0.12)\n", + "cent.check()\n", "cent.plot()\n", "\n", "# construct tropical cyclones\n", diff --git a/requirements/env_climada.yml b/requirements/env_climada.yml index 9ebc16c2b..c3e9762c4 100644 --- a/requirements/env_climada.yml +++ b/requirements/env_climada.yml @@ -8,9 +8,9 @@ dependencies: - cfgrib>=0.9.9,<0.9.10 # 0.9.10 cannot read the icon_grib files from https://opendata.dwd.de - contextily>=1.6 - dask>=2024.5 - - eccodes>=2.27,<2.28 # 2.28 changed some labels, in particular: gust -> i20fg + - eccodes>=2.27 # 2.28 changed some labels, in particular: gust -> i10fg (i20fg?) - gdal>=3.6 - - geopandas>=0.14 + - geopandas>=0.14,<1.0 - h5py>=3.8 - haversine>=2.8 - matplotlib-base>=3.9