From 29ba3a3c56efdbf20220b9a6060494d3b6036a6f Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Mon, 13 May 2024 15:29:52 -0700 Subject: [PATCH 01/88] add test file --- tests/test_drcdm.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/test_drcdm.py diff --git a/tests/test_drcdm.py b/tests/test_drcdm.py new file mode 100644 index 000000000..a8a8219c2 --- /dev/null +++ b/tests/test_drcdm.py @@ -0,0 +1,2 @@ +def test_pass(): + pass From f1bcdeb6bc505107c4667279159e4becb1289633 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Mon, 13 May 2024 15:31:34 -0700 Subject: [PATCH 02/88] add driver script --- pcmdi_metrics/drcdm/drcdm_driver.py | 288 ++++++++++++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 pcmdi_metrics/drcdm/drcdm_driver.py diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py new file mode 100644 index 000000000..07fbe01de --- /dev/null +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python +import glob +import os + +import xcdat + +from pcmdi_metrics.drcdm.lib import ( + compute_metrics, + create_extremes_parser, + metadata, + region_utilities, + utilities, +) + +########## +# Set up +########## + +parser = create_extremes_parser.create_extremes_parser() +parameter = parser.get_parameter(argparse_vals_only=False) + +# Parameters +# I/O settings +case_id = parameter.case_id +model_list = parameter.test_data_set +realization = parameter.realization +variable_list = parameter.vars +filename_template = parameter.filename_template +sftlf_filename_template = parameter.sftlf_filename_template +test_data_path = parameter.test_data_path +reference_data_path = parameter.reference_data_path +reference_data_set = parameter.reference_data_set +reference_sftlf_template = parameter.reference_sftlf_template +metrics_output_path = parameter.metrics_output_path +ModUnitsAdjust = parameter.ModUnitsAdjust +ObsUnitsAdjust = parameter.ObsUnitsAdjust +plots = parameter.plots +debug = parameter.debug +cmec = parameter.cmec +msyear = parameter.msyear +meyear = parameter.meyear +osyear = parameter.osyear +oeyear = parameter.oeyear +generate_sftlf = parameter.generate_sftlf +regrid = parameter.regrid +cov_file = parameter.covariate_path +cov_name = parameter.covariate +return_period = parameter.return_period +# Block extrema related settings +annual_strict = parameter.annual_strict +exclude_leap = parameter.exclude_leap +dec_mode = parameter.dec_mode +drop_incomplete_djf = parameter.drop_incomplete_djf +# Region masking +shp_path = parameter.shp_path +col = parameter.attribute +region_name = parameter.region_name +coords = parameter.coords + +# Check the region masking parameters, if provided +use_region_mask, region_name, coords = region_utilities.check_region_params( + shp_path, coords, region_name, col, "land" +) + +# Verifying output directory +metrics_output_path = utilities.verify_output_path(metrics_output_path, case_id) + +if isinstance(reference_data_set, list): + # Fix a command line issue + reference_data_set = reference_data_set[0] + +# Verify years +ok_mod = utilities.verify_years( + msyear, + meyear, + msg="Error: Model msyear and meyear must both be set or both be None (unset).", +) +ok_obs = utilities.verify_years( + osyear, + oeyear, + msg="Error: Obs osyear and oeyear must both be set or both be None (unset).", +) + +# Initialize output.json file +meta = metadata.MetadataFile(metrics_output_path) + +# Initialize other directories +# Not sure if needed so commented for now. +# nc_dir = os.path.join(metrics_output_path, "netcdf") +# os.makedirs(nc_dir, exist_ok=True) +# if plots: +# plot_dir_maps = os.path.join(metrics_output_path, "plots", "maps") +# os.makedirs(plot_dir_maps, exist_ok=True) + +# Setting up model realization list +find_all_realizations, realizations = utilities.set_up_realizations(realization) + +# Only include reference data in loop if it exists +if reference_data_path is not None: + model_loop_list = ["Reference"] + model_list +else: + model_loop_list = model_list + +# Initialize output JSON structures +# FYI: if the analysis output JSON is changed, remember to update this function! +metrics_dict = compute_metrics.init_metrics_dict( + model_loop_list, + variable_list, + dec_mode, + drop_incomplete_djf, + annual_strict, + region_name, +) + +obs = {} + +############## +# Run Analysis +############## + +# Loop over models +for model in model_loop_list: + if model == "Reference": + list_of_runs = [str(reference_data_set)] + elif find_all_realizations: + tags = {"%(model)": model, "%(model_version)": model, "%(realization)": "*"} + test_data_full_path = os.path.join(test_data_path, filename_template) + test_data_full_path = utilities.replace_multi(test_data_full_path, tags) + ncfiles = glob.glob(test_data_full_path) + realizations = [] + for ncfile in ncfiles: + realizations.append(ncfile.split("/")[-1].split(".")[3]) + print("=================================") + print("model, runs:", model, realizations) + list_of_runs = realizations + else: + list_of_runs = realizations + + metrics_dict["RESULTS"][model] = {} + + # Loop over realizations + for run in list_of_runs: + # Finding land/sea mask + sftlf_exists = True + if run == reference_data_set: + if reference_sftlf_template is not None and os.path.exists( + reference_sftlf_template + ): + sftlf_filename = reference_sftlf_template + else: + print("No reference sftlf file template provided.") + if not generate_sftlf: + print("Skipping reference data") + else: + # Set flag to generate sftlf after loading data + sftlf_exists = False + else: + try: + tags = { + "%(model)": model, + "%(model_version)": model, + "%(realization)": run, + } + sftlf_filename_list = utilities.replace_multi( + sftlf_filename_template, tags + ) + sftlf_filename = glob.glob(sftlf_filename_list)[0] + except (AttributeError, IndexError): + print("No sftlf file found for", model, run) + if not generate_sftlf: + print("Skipping realization", run) + continue + else: + # Set flag to generate sftlf after loading data + sftlf_exists = False + if sftlf_exists: + sftlf = xcdat.open_dataset(sftlf_filename, decode_times=False) + # Note: we should expect SFTLF to be between 0-1 like the rest of the metrics + # in the PMP do, not 0-100 like extremes. + if use_region_mask: + print("\nCreating region mask for land/sea mask.") + sftlf = region_utilities.mask_region( + sftlf, region_name, coords=coords, shp_path=shp_path, column=col + ) + + if run == reference_data_set: + units_adjust = ObsUnitsAdjust + else: + units_adjust = ModUnitsAdjust + + # Initialize results dictionary for this mode/run + # Presumably we will be populating this will metrics as we go + metrics_dict["RESULTS"][model][run] = {} + + # In extremes we are looping over different variables. I am really + # not sure what will be the best way to approach this, if we should loop + # over variables, take in multiple variables at once, etc. + for varname in variable_list: + # Populate the filename templates to get actual data path + if run == reference_data_set: + test_data_full_path = reference_data_path + start_year = osyear + end_year = oeyear + else: + tags = { + "%(variable)": varname, + "%(model)": model, + "%(model_version)": model, + "%(realization)": run, + } + test_data_full_path = os.path.join(test_data_path, filename_template) + test_data_full_path = utilities.replace_multi(test_data_full_path, tags) + start_year = msyear + end_year = meyear + yrs = [str(start_year), str(end_year)] # for output file names + test_data_full_path = glob.glob(test_data_full_path) + test_data_full_path.sort() + if len(test_data_full_path) == 0: + print("") + print("-----------------------") + print("Not found: model, run, variable:", model, run, varname) + continue + else: + print("") + print("-----------------------") + print("model, run, variable:", model, run, varname) + print("test_data (model in this case) full_path:") + for t in test_data_full_path: + print(" ", t) + + # Load and prep data + # TODO: substitute PMP version of loading + ds = utilities.load_dataset(test_data_full_path) + + if not sftlf_exists and generate_sftlf: + print("Generating land sea mask.") + sftlf = utilities.generate_land_sea_mask(ds, debug=debug) + if use_region_mask: + # TODO: any way to not create the land/sea mask in two different places? + print("\nCreating region mask for land/sea mask.") + sftlf = region_utilities.mask_region( + sftlf, region_name, coords=coords, shp_path=shp_path, column=col + ) + + # Mask out Antarctica + sftlf["sftlf"] = sftlf["sftlf"].where(sftlf.lat > -60) + + if use_region_mask: + print("Creating dataset mask.") + ds = region_utilities.mask_region( + ds, region_name, coords=coords, shp_path=shp_path, column=col + ) + + # Get time slice if year parameters exist + if start_year is not None: + ds = utilities.slice_dataset(ds, start_year, end_year) + else: + # Get labels for start/end years from dataset + yrs = [str(int(ds.time.dt.year[0])), str(int(ds.time.dt.year[-1]))] + + # If any of the metrics use daily data we'll want to keep + # something like this calendar handling + if ds.time.encoding["calendar"] != "noleap" and exclude_leap: + ds = ds.convert_calendar("noleap") + + # ------------------------------- + # Metrics go here + # ------------------------------- + + +# ------------------------------- +# Output JSON with metrics here +# ------------------------------- + +# Update and write metadata file +# Will want something similar to below code I've left commented +# try: +# with open(fname, "r") as f: +# tmp = json.load(f) +# meta.update_provenance("environment", tmp["provenance"]) +# except Exception: +# # Skip provenance if there's an issue +# print("Error: Could not get provenance from extremes json for output.json.") + +meta.update_provenance("modeldata", test_data_path) +if reference_data_path is not None: + meta.update_provenance("obsdata", reference_data_path) +meta.write() From 8e3019022bd87ffef5921e9b5341df8e764cbd15 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Mon, 13 May 2024 15:32:35 -0700 Subject: [PATCH 03/88] change import --- pcmdi_metrics/drcdm/drcdm_driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 07fbe01de..c4bd9dd99 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -6,7 +6,7 @@ from pcmdi_metrics.drcdm.lib import ( compute_metrics, - create_extremes_parser, + create_drcdm_parser, metadata, region_utilities, utilities, @@ -16,7 +16,7 @@ # Set up ########## -parser = create_extremes_parser.create_extremes_parser() +parser = create_drcdm_parser.create_extremes_parser() parameter = parser.get_parameter(argparse_vals_only=False) # Parameters From 743a7f94499ed638de2fa3b939fb16d8c62ede68 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Mon, 13 May 2024 15:33:05 -0700 Subject: [PATCH 04/88] add lib --- pcmdi_metrics/drcdm/lib/__init__.py | 0 pcmdi_metrics/drcdm/lib/compute_metrics.py | 709 ++++++++++++++++++ .../drcdm/lib/create_drcdm_parser.py | 291 +++++++ pcmdi_metrics/drcdm/lib/metadata.py | 49 ++ pcmdi_metrics/drcdm/lib/region_utilities.py | 112 +++ pcmdi_metrics/drcdm/lib/utilities.py | 141 ++++ 6 files changed, 1302 insertions(+) create mode 100644 pcmdi_metrics/drcdm/lib/__init__.py create mode 100644 pcmdi_metrics/drcdm/lib/compute_metrics.py create mode 100644 pcmdi_metrics/drcdm/lib/create_drcdm_parser.py create mode 100644 pcmdi_metrics/drcdm/lib/metadata.py create mode 100644 pcmdi_metrics/drcdm/lib/region_utilities.py create mode 100644 pcmdi_metrics/drcdm/lib/utilities.py diff --git a/pcmdi_metrics/drcdm/lib/__init__.py b/pcmdi_metrics/drcdm/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py new file mode 100644 index 000000000..649fe58c7 --- /dev/null +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -0,0 +1,709 @@ +#!/usr/bin/env python +import datetime + +import cftime +import numpy as np +import xarray as xr +import xcdat as xc + +from pcmdi_metrics.mean_climate.lib import compute_statistics + + +class TimeSeriesData: + # Track years and calendar for time series grids + # Store methods to act on time series grids + def __init__(self, ds, ds_var): + self.ds = ds + self.ds_var = ds_var + self.freq = xr.infer_freq(ds.time) + self._set_years() + self.calendar = ds.time.encoding["calendar"] + self.time_units = ds.time.encoding["units"] + + def _set_years(self): + self.year_beg = self.ds.isel({"time": 0}).time.dt.year.item() + self.year_end = self.ds.isel({"time": -1}).time.dt.year.item() + + if self.year_end < self.year_beg + 1: + raise Exception("Error: Final year must be greater than beginning year.") + + self.year_range = np.arange(self.year_beg, self.year_end + 1, 1) + + def return_data_array(self): + return self.ds[self.ds_var] + + def rolling_5day(self): + # Use on daily data + return self.ds[self.ds_var].rolling(time=5).mean() + + +class SeasonalAverager: + # Make seasonal averages of data in TimeSeriesData class + + def __init__( + self, TSD, sftlf, dec_mode="DJF", drop_incomplete_djf=True, annual_strict=True + ): + self.TSD = TSD + self.dec_mode = dec_mode + self.drop_incomplete_djf = drop_incomplete_djf + self.annual_strict = annual_strict + self.del1d = datetime.timedelta(days=1) + self.del0d = datetime.timedelta(days=0) + self.pentad = None + self.sftlf = sftlf["sftlf"] + + def masked_ds(self, ds): + # Mask land where 50<=sftlf<=100 + return ds.where(self.sftlf >= 50).where(self.sftlf <= 100) + + def calc_5day_mean(self): + # Get the 5-day mean dataset + self.pentad = self.TSD.rolling_5day() + + def fix_time_coord(self, ds): + cal = self.TSD.calendar + ds = ds.rename({"year": "time"}) + y_to_cft = [cftime.datetime(y, 1, 1, calendar=cal) for y in ds.time] + ds["time"] = y_to_cft + ds.time.attrs["axis"] = "T" + ds["time"].encoding["calendar"] = cal + ds["time"].attrs["standard_name"] = "time" + ds.time.encoding["units"] = self.TSD.time_units + return ds + + def annual_stats(self, stat, pentad=False): + # Acquire annual statistics + # Arguments: + # stat: Can be "max", "min" + # pentad: True to run on 5-day mean + # Returns: + # ds_ann: Dataset containing annual max or min grid + + if pentad: + if self.pentad is None: + self.calc_5day_mean() + ds = self.pentad + else: + ds = self.TSD.return_data_array() + cal = self.TSD.calendar + + if self.annual_strict and pentad: + # This setting is for means using 5 day rolling average values, where + # we do not want to include any data from the prior year + year_range = self.TSD.year_range + hr = int(ds.time[0].dt.hour) # get hour to help with selecting nearest time + + # Only use data from that year - start on Jan 5 avg + date_range = [ + xr.cftime_range( + start=cftime.datetime(year, 1, 5, hour=hr, calendar=cal) + - self.del0d, + end=cftime.datetime(year + 1, 1, 1, hour=hr, calendar=cal) + - self.del1d, + freq="D", + calendar=cal, + ) + for year in year_range + ] + date_range = [item for sublist in date_range for item in sublist] + if stat == "max": + ds_ann = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .max(dim="time") + ) + elif stat == "min": + ds_ann = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .min(dim="time") + ) + else: + # Group by date + if stat == "max": + ds_ann = ds.groupby("time.year").max(dim="time") + elif stat == "min": + ds_ann = ds.groupby("time.year").min(dim="time") + + # Need to fix time axis if groupby operation happened + if "year" in ds_ann.coords: + ds_ann = self.fix_time_coord(ds_ann) + return self.masked_ds(ds_ann) + + def seasonal_stats(self, season, stat, pentad=False): + # Acquire statistics for a given season + # Arguments: + # season: Can be "DJF","MAM","JJA","SON" + # stat: Can be "max", "min" + # pentad: True to run on 5-day mean + # Returns: + # ds_stat: Dataset containing seasonal max or min grid + + year_range = self.TSD.year_range + + if pentad: + if self.pentad is None: + self.calc_5day_mean() + ds = self.pentad + else: + ds = self.TSD.return_data_array() + cal = self.TSD.calendar + + hr = int(ds.time[0].dt.hour) # help with selecting nearest time + + if season == "DJF" and self.dec_mode == "DJF": + # Resample DJF to count prior DJF in current year + if stat == "max": + ds_stat = ds.resample(time="QS-DEC").max(dim="time") + elif stat == "min": + ds_stat = ds.resample(time="QS-DEC").min(dim="time") + + ds_stat = ds_stat.isel(time=ds_stat.time.dt.month.isin([12])) + + # Deal with inconsistencies between QS-DEC calendar and block exremes calendar + if self.drop_incomplete_djf: + ds_stat = ds_stat.sel( + {"time": slice(str(year_range[0]), str(year_range[-1] - 1))} + ) + ds_stat["time"] = [ + cftime.datetime(y, 1, 1, calendar=cal) + for y in np.arange(year_range[0] + 1, year_range[-1] + 1) + ] + else: + ds_stat = ds_stat.sel( + {"time": slice(str(year_range[0] - 1), str(year_range[-1] - 1))} + ) + ds_stat["time"] = [ + cftime.datetime(y, 1, 1, calendar=cal) + for y in np.arange(year_range[0], year_range[-1] + 1) + ] + + elif season == "DJF" and self.dec_mode == "JFD": + # Make date lists that capture JF and D in all years, then merge and sort + if self.annual_strict and pentad: + # Only use data from that year - start on Jan 5 avg + date_range_1 = [ + xr.cftime_range( + start=cftime.datetime(year, 1, 5, hour=hr, calendar=cal) + - self.del0d, + end=cftime.datetime(year, 3, 1, hour=hr, calendar=cal) + - self.del1d, + freq="D", + calendar=cal, + ) + for year in year_range + ] + else: + date_range_1 = [ + xr.cftime_range( + start=cftime.datetime(year, 1, 1, hour=hr, calendar=cal) + - self.del0d, + end=cftime.datetime(year, 3, 1, hour=hr, calendar=cal) + - self.del1d, + freq="D", + calendar=cal, + ) + for year in year_range + ] + date_range_1 = [item for sublist in date_range_1 for item in sublist] + date_range_2 = [ + xr.cftime_range( + start=cftime.datetime(year, 12, 1, hour=hr, calendar=cal) + - self.del0d, + end=cftime.datetime(year + 1, 1, 1, hour=hr, calendar=cal) + - self.del1d, + freq="D", + calendar=cal, + ) + for year in year_range + ] + date_range_2 = [item for sublist in date_range_2 for item in sublist] + date_range = sorted(date_range_1 + date_range_2) + + if stat == "max": + ds_stat = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .max(dim="time") + ) + elif stat == "min": + ds_stat = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .min(dim="time") + ) + + else: # Other 3 seasons + dates = { # Month/day tuples + "MAM": [(3, 1), (6, 1)], + "JJA": [(6, 1), (9, 1)], + "SON": [(9, 1), (12, 1)], + } + + mo_st = dates[season][0][0] + day_st = dates[season][0][1] + mo_en = dates[season][1][0] + day_en = dates[season][1][1] + + cal = self.TSD.calendar + + date_range = [ + xr.cftime_range( + start=cftime.datetime(year, mo_st, day_st, hour=hr, calendar=cal) + - self.del0d, + end=cftime.datetime(year, mo_en, day_en, hour=hr, calendar=cal) + - self.del1d, + freq="D", + calendar=cal, + ) + for year in year_range + ] + date_range = [item for sublist in date_range for item in sublist] + + if stat == "max": + ds_stat = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .max(dim="time") + ) + elif stat == "min": + ds_stat = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .min(dim="time") + ) + + # Need to fix time axis if groupby operation happened + if "year" in ds_stat.coords: + ds_stat = self.fix_time_coord(ds_stat) + return self.masked_ds(ds_stat) + + +def update_nc_attrs(ds, dec_mode, drop_incomplete_djf, annual_strict): + # Add bounds and record user settings in attributes + # Use this function for any general dataset updates. + ds.lat.attrs["standard_name"] = "Y" + ds.lon.attrs["standard_name"] = "X" + bnds_dict = {"lat": "Y", "lon": "X", "time": "T"} + for item in bnds_dict: + if "bounds" in ds[item].attrs: + bnds_var = ds[item].attrs["bounds"] + if bnds_var not in ds.keys(): + ds[item].attrs["bounds"] = "" + ds = ds.bounds.add_missing_bounds([bnds_dict[item]]) + else: + ds = ds.bounds.add_missing_bounds([bnds_dict[item]]) + ds.attrs["december_mode"] = str(dec_mode) + ds.attrs["drop_incomplete_djf"] = str(drop_incomplete_djf) + ds.attrs["annual_strict"] = str(annual_strict) + + # Update fill value encoding + ds.lat.encoding["_FillValue"] = None + ds.lon.encoding["_FillValue"] = None + ds.time.encoding["_FillValue"] = None + ds.lat_bnds.encoding["_FillValue"] = None + ds.lon_bnds.encoding["_FillValue"] = None + ds.time_bnds.encoding["_FillValue"] = None + for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: + ds[season].encoding["_FillValue"] = float(1e20) + + # Drop type attribute that comes from land mask + if "type" in ds: + ds = ds.drop("type") + return ds + + +def convert_units(data, units_adjust): + # Convert the units of the input data + # units_adjust is a tuple of form + # (flag (bool), operation (str), value (float), units (str)). + # For example, (True, "multiply", 86400., "mm/day") + # If flag is False, data is returned unaltered. + if bool(units_adjust[0]): + op_dict = {"add": "+", "subtract": "-", "multiply": "*", "divide": "/"} + if str(units_adjust[1]) not in op_dict: + print( + "Error in units conversion. Operation must be add, subtract, multiply, or divide." + ) + print("Skipping units conversion.") + return data + op = op_dict[str(units_adjust[1])] + val = float(units_adjust[2]) + operation = "data {0} {1}".format(op, val) + data = eval(operation) + data.attrs["units"] = str(units_adjust[3]) + else: + # No adjustment, but check that units attr is populated + if "units" not in data.attrs: + data.attrs["units"] = "" + return data + + +def temperature_indices( + ds, varname, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict +): + # Returns annual max and min of provided temperature dataset + # Temperature input can be "tasmax" or "tasmin". + + print("Generating temperature block extrema.") + + ds[varname] = convert_units(ds[varname], units_adjust) + + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + Tmax = xr.Dataset() + Tmin = xr.Dataset() + Tmax["ANN"] = S.annual_stats("max") + Tmin["ANN"] = S.annual_stats("min") + + for season in ["DJF", "MAM", "JJA", "SON"]: + Tmax[season] = S.seasonal_stats(season, "max") + Tmin[season] = S.seasonal_stats(season, "min") + + Tmax = update_nc_attrs(Tmax, dec_mode, drop_incomplete_djf, annual_strict) + Tmin = update_nc_attrs(Tmin, dec_mode, drop_incomplete_djf, annual_strict) + + return Tmax, Tmin + + +def precipitation_indices( + ds, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict +): + # Returns annual Rx1day and Rx5day of provided precipitation dataset. + # Precipitation variable must be called "pr". + # Input data expected to have units of kg/m2/s + + print("Generating precipitation block extrema.") + + ds["pr"] = convert_units(ds["pr"], units_adjust) + + PR = TimeSeriesData(ds, "pr") + S = SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + # Rx1day + P1day = xr.Dataset() + P1day["ANN"] = S.annual_stats("max", pentad=False) + # Can end up with very small negative values that should be 0 + # Possibly related to this issue? https://github.com/pydata/bottleneck/issues/332 + # (from https://github.com/pydata/xarray/issues/3855) + P1day["ANN"] = ( + P1day["ANN"].where(P1day["ANN"] > 0, 0).where(~np.isnan(P1day["ANN"]), np.nan) + ) + for season in ["DJF", "MAM", "JJA", "SON"]: + P1day[season] = S.seasonal_stats(season, "max", pentad=False) + P1day[season] = ( + P1day[season] + .where(P1day[season] > 0, 0) + .where(~np.isnan(P1day[season]), np.nan) + ) + P1day = update_nc_attrs(P1day, dec_mode, drop_incomplete_djf, annual_strict) + + # Rx5day + P5day = xr.Dataset() + P5day["ANN"] = S.annual_stats("max", pentad=True) + P5day["ANN"] = ( + P5day["ANN"].where(P5day["ANN"] > 0, 0).where(~np.isnan(P5day["ANN"]), np.nan) + ) + for season in ["DJF", "MAM", "JJA", "SON"]: + P5day[season] = S.seasonal_stats(season, "max", pentad=True) + P5day[season] = ( + P5day[season] + .where(P5day[season] > 0, 0) + .where(~np.isnan(P5day[season]), np.nan) + ) + P5day = update_nc_attrs(P5day, dec_mode, drop_incomplete_djf, annual_strict) + + return P1day, P5day + + +# A couple of statistics that aren't being loaded from mean_climate +def mean_xy(data, varname): + # Spatial mean of single dataset + mean_xy = data.spatial.average(varname)[varname].mean() + return float(mean_xy) + + +def percent_difference(ref, bias_xy, varname, weights): + # bias as percentage of reference dataset "ref" + pct_dif = float( + 100.0 + * bias_xy + / ref.spatial.average(varname, axis=["X", "Y"], weights=weights)[varname] + ) + return pct_dif + + +def init_metrics_dict( + model_list, var_list, dec_mode, drop_incomplete_djf, annual_strict, region_name +): + # Return initial version of the metrics dictionary + metrics = { + "DIMENSIONS": { + "json_structure": [ + "model", + "realization", + "index", + "region", + "statistic", + "season", + ], + "region": {region_name: "Areas where 50<=sftlf<=100"}, + "season": ["ANN", "DJF", "MAM", "JJA", "SON"], + "index": {}, + "statistic": { + "mean": compute_statistics.mean_xy(None), + "std_xy": compute_statistics.std_xy(None, None), + "bias_xy": compute_statistics.bias_xy(None, None), + "cor_xy": compute_statistics.cor_xy(None, None), + "mae_xy": compute_statistics.meanabs_xy(None, None), + "rms_xy": compute_statistics.rms_xy(None, None), + "rmsc_xy": compute_statistics.rmsc_xy(None, None), + "std-obs_xy": compute_statistics.std_xy(None, None), + "pct_dif": { + "Abstract": "Bias xy as a percentage of the Observed mean.", + "Contact": "pcmdi-metrics@llnl.gov", + "Name": "Spatial Difference Percentage", + }, + }, + "model": model_list, + "realization": [], + }, + "RESULTS": {}, + "RUNTIME_CALENDAR_SETTINGS": { + "december_mode": str(dec_mode), + "drop_incomplete_djf": str(drop_incomplete_djf), + "annual_strict": str(annual_strict), + }, + } + + # Only include the definitions for the indices in this particular analysis. + for v in var_list: + if v == "tasmax": + metrics["DIMENSIONS"]["index"].update( + {"TXx": "Maximum value of daily maximum temperature"} + ) + metrics["DIMENSIONS"]["index"].update( + {"TXn": "Minimum value of daily maximum temperature"} + ) + if v == "tasmin": + metrics["DIMENSIONS"]["index"].update( + {"TNx": "Maximum value of daily minimum temperature"} + ) + metrics["DIMENSIONS"]["index"].update( + {"TNn": "Minimum value of daily minimum temperature"} + ) + if v in ["pr", "PRECT", "precip"]: + metrics["DIMENSIONS"]["index"].update( + {"Rx5day": "Maximum consecutive 5-day mean precipitation, mm/day"} + ) + metrics["DIMENSIONS"]["index"].update( + {"Rx1day": "Maximum daily precipitation, mm/day"} + ) + + return metrics + + +def metrics_json(data_dict, obs_dict={}, region="land", regrid=True): + # Format, calculate, and return the global mean value over land + # for all datasets in the input dictionary + # Arguments: + # data_dict: Dictionary containing block extrema datasets + # obs_dict: Dictionary containing block extrema for + # reference dataset + # region: Name of region. + # Returns: + # met_dict: A dictionary containing metrics + + met_dict = {} + seasons_dict = {"ANN": "", "DJF": "", "MAM": "", "JJA": "", "SON": ""} + + # Looping over each type of extrema in data_dict + for m in data_dict: + met_dict[m] = { + region: {"mean": seasons_dict.copy(), "std_xy": seasons_dict.copy()} + } + # If obs available, add metrics comparing with obs + # If new statistics are added, be sure to update + # "statistic" entry in init_metrics_dict() + if len(obs_dict) > 0: + for k in [ + "std-obs_xy", + "pct_dif", + "bias_xy", + "cor_xy", + "mae_xy", + "rms_xy", + "rmsc_xy", + ]: + met_dict[m][region][k] = seasons_dict.copy() + + ds_m = data_dict[m] + for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: + # Global mean over land + met_dict[m][region]["mean"][season] = mean_xy(ds_m, season) + a = ds_m.temporal.average(season) + std_xy = compute_statistics.std_xy(a, season) + met_dict[m][region]["std_xy"][season] = std_xy + + if len(obs_dict) > 0 and not obs_dict[m].equals(ds_m): + # Regrid obs to model grid + if regrid: + target = xc.create_grid(ds_m.lat, ds_m.lon) + target = target.bounds.add_missing_bounds(["X", "Y"]) + obs_m = obs_dict[m].regridder.horizontal( + season, target, tool="regrid2" + ) + else: + obs_m = obs_dict[m] + shp1 = (len(ds_m[season].lat), len(ds_m[season].lon)) + shp2 = (len(obs_m[season].lat), len(obs_m[season].lon)) + assert ( + shp1 == shp2 + ), "Model and Reference data dimensions 'lat' and 'lon' must match." + + # Get xy stats for temporal average + a = ds_m.temporal.average(season) + b = obs_m.temporal.average(season) + weights = ds_m.spatial.get_weights(axis=["X", "Y"]) + rms_xy = compute_statistics.rms_xy(a, b, var=season, weights=weights) + meanabs_xy = compute_statistics.meanabs_xy( + a, b, var=season, weights=weights + ) + bias_xy = compute_statistics.bias_xy(a, b, var=season, weights=weights) + cor_xy = compute_statistics.cor_xy(a, b, var=season, weights=weights) + rmsc_xy = compute_statistics.rmsc_xy(a, b, var=season, weights=weights) + std_obs_xy = compute_statistics.std_xy(b, season) + pct_dif = percent_difference(b, bias_xy, season, weights) + + met_dict[m][region]["pct_dif"][season] = pct_dif + met_dict[m][region]["rms_xy"][season] = rms_xy + met_dict[m][region]["mae_xy"][season] = meanabs_xy + met_dict[m][region]["bias_xy"][season] = bias_xy + met_dict[m][region]["cor_xy"][season] = cor_xy + met_dict[m][region]["rmsc_xy"][season] = rmsc_xy + met_dict[m][region]["std-obs_xy"][season] = std_obs_xy + + return met_dict + + +def metrics_json_return_value( + rv, blockex, obs, blockex_obs, stat, region="land", regrid=True +): + # Generate metrics for stationary return value comparing model and obs + # Arguments: + # rv: dataset + # blockex: dataset + # obs: dataset + # stat: string + # region: string + # regrid: bool + # Returns: + # met_dict: dictionary + met_dict = {stat: {}} + seasons_dict = {"ANN": "", "DJF": "", "MAM": "", "JJA": "", "SON": ""} + + # Looping over each type of extrema in data_dict + met_dict[stat] = { + region: {"mean": seasons_dict.copy(), "std_xy": seasons_dict.copy()} + } + # If obs available, add metrics comparing with obs + # If new statistics are added, be sure to update + # "statistic" entry in init_metrics_dict() + if obs is not None: + obs = obs.bounds.add_missing_bounds() + for k in [ + "std-obs_xy", + "pct_dif", + "bias_xy", + "cor_xy", + "mae_xy", + "rms_xy", + "rmsc_xy", + ]: + met_dict[stat][region][k] = seasons_dict.copy() + + rv_tmp = rv.copy(deep=True) + + for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: + # Global mean over land + rv_tmp[season] = remove_outliers(rv[season], blockex[season]) + met_dict[stat][region]["mean"][season] = mean_xy(rv_tmp, season) + std_xy = compute_statistics.std_xy(rv_tmp, season) + met_dict[stat][region]["std_xy"][season] = std_xy + + if obs is not None and not obs[season].equals(rv_tmp): + obs[season] = remove_outliers(obs[season], blockex_obs[season]) + # Regrid obs to model grid + if regrid: + target = xc.create_grid(rv_tmp.lat, rv_tmp.lon) + target = target.bounds.add_missing_bounds(["X", "Y"]) + obs_m = obs.regridder.horizontal(season, target, tool="regrid2") + else: + obs_m = obs + shp1 = (len(rv_tmp.lat), len(rv_tmp.lon)) + shp2 = (len(obs.lat), len(obs.lon)) + assert ( + shp1 == shp2 + ), "Model and Reference data dimensions 'lat' and 'lon' must match." + + # Get xy stats for temporal average + weights = rv_tmp.spatial.get_weights(axis=["X", "Y"]) + rms_xy = compute_statistics.rms_xy( + rv_tmp, obs_m, var=season, weights=weights + ) + meanabs_xy = compute_statistics.meanabs_xy( + rv_tmp, obs_m, var=season, weights=weights + ) + bias_xy = compute_statistics.bias_xy( + rv_tmp, obs_m, var=season, weights=weights + ) + cor_xy = compute_statistics.cor_xy( + rv_tmp, obs_m, var=season, weights=weights + ) + rmsc_xy = compute_statistics.rmsc_xy( + rv_tmp, obs_m, var=season, weights=weights + ) + std_obs_xy = compute_statistics.std_xy(rv_tmp, season) + pct_dif = percent_difference(obs_m, bias_xy, season, weights) + + met_dict[stat][region]["pct_dif"][season] = pct_dif + met_dict[stat][region]["rms_xy"][season] = rms_xy + met_dict[stat][region]["mae_xy"][season] = meanabs_xy + met_dict[stat][region]["bias_xy"][season] = bias_xy + met_dict[stat][region]["cor_xy"][season] = cor_xy + met_dict[stat][region]["rmsc_xy"][season] = rmsc_xy + met_dict[stat][region]["std-obs_xy"][season] = std_obs_xy + + return met_dict + + +def remove_outliers(rv, blockex): + # Remove outlier return values for metrics computation + # filtering by comparing to the block extreme values + # rv: data array + # blckex: data array + block_max = blockex.max("time", skipna=True).data + block_min = blockex.min("time", skipna=True).data + block_std = blockex.std("time", skipna=True).data + + # Remove values that are either: + # 8 standard deviations above the max value in block extema + # 8 standard deviations below the min value in block extrema + tol = 8 * block_std + plussig = block_max + tol + minsig = block_min - tol + rv_remove_outliers = rv.where((rv < plussig) & (rv > minsig)) + return rv_remove_outliers diff --git a/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py b/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py new file mode 100644 index 000000000..8992bb128 --- /dev/null +++ b/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +from pcmdi_metrics.mean_climate.lib import pmp_parser + + +def create_extremes_parser(): + parser = pmp_parser.PMPMetricsParser() + parser.add_argument( + "--case_id", + dest="case_id", + help="Defines a subdirectory to the metrics output, so multiple" + + "cases can be compared", + required=False, + ) + + parser.add_argument( + "-v", + "--vars", + type=str, + nargs="+", + dest="vars", + help="Variables to use", + required=False, + ) + + parser.add_argument( + "-r", + "--reference_data_set", + default=None, + type=str, + nargs="+", + dest="reference_data_set", + help="List of observations or models that are used as a " + + "reference against the test_data_set", + required=False, + ) + + parser.add_argument( + "--reference_data_path", + default=None, + dest="reference_data_path", + help="Path for the reference climitologies", + required=False, + ) + parser.add_argument( + "--reference_sftlf_template", + default=None, + dest="reference_sftlf_template", + help="Path for the reference sftlf file", + required=False, + ) + + parser.add_argument( + "-t", + "--test_data_set", + type=str, + nargs="+", + dest="test_data_set", + help="List of observations or models to test " + + "against the reference_data_set", + required=False, + ) + + parser.add_argument( + "--test_data_path", + dest="test_data_path", + help="Path for the test climitologies", + required=False, + ) + + parser.add_argument( + "--realization", + dest="realization", + help="A simulation parameter", + required=False, + ) + + parser.add_argument( + "--dry_run", + # If input is 'True' or 'true', return True. Otherwise False. + type=lambda x: x.lower() == "true", + dest="dry_run", + help="True if output is to be created, False otherwise", + required=False, + ) + + parser.add_argument( + "--filename_template", + dest="filename_template", + help="Template for climatology files", + required=False, + ) + + parser.add_argument( + "--sftlf_filename_template", + dest="sftlf_filename_template", + help='Filename template for landsea masks ("sftlf")', + required=False, + ) + + parser.add_argument( + "--metrics_output_path", + dest="metrics_output_path", + default=None, + help="Directory of where to put the results", + required=False, + ) + + parser.add_argument( + "--filename_output_template", + dest="filename_output_template", + help="Filename for the interpolated test climatologies", + required=False, + ) + + parser.add_argument( + "--output_json_template", + help="Filename template for results json files", + required=False, + ) + + parser.add_argument( + "--user_notes", + dest="user_notes", + help="Provide a short description to help identify this run of the PMP mean climate.", + required=False, + ) + + parser.add_argument( + "--debug", + dest="debug", + action="store_true", + help="Turn on debugging mode by printing more information to track progress", + required=False, + ) + + parser.add_argument( + "--cmec", + dest="cmec", + action="store_true", + help="Save metrics in CMEC format", + required=False, + ) + + parser.add_argument( + "--no_cmec", + dest="cmec", + action="store_false", + help="Option to not save metrics in CMEC format", + required=False, + ) + + parser.add_argument( + "--chunk_size", + dest="chunk_size", + default=None, + help="Chunk size for latitude/longitude", + required=False, + ) + + parser.add_argument( + "--annual_strict", + dest="annual_strict", + action="store_true", + help="Flag to only include current year in rolling data calculations", + ) + + parser.add_argument( + "--exclude_leap_day", + dest="exclude_leap", + action="store_true", + help="Flag to exclude leap days", + ) + + parser.add_argument( + "--keep_incomplete_djf", + dest="drop_incomplete_djf", + action="store_false", + help="Flag to include data from incomplete DJF seasons", + ) + + parser.add_argument( + "--dec_mode", + dest="dec_mode", + default="DJF", + help="'DJF' or 'JFD' format for December/January/February season", + ) + + parser.add_argument( + "--year_range", + type=list, + default=[None, None], + help="List containing the start and end year", + ), + parser.add_argument( + "--covariate_path", type=str, default=None, help="Covariate file path" + ) + parser.add_argument( + "--covariate", type=str, default="CO2mass", help="Covariate variable name" + ) + + parser.add_argument( + "--shp_path", + type=str, + default=None, + help="Region shapefile path. Must also provide --column and --region_name. Only one of --shp_path, --coords can be used.", + required=False, + ) + + parser.add_argument( + "--attribute", + type=str, + default=None, + help="Name of region attribute column in shapefile", + required=False, + ) + parser.add_argument( + "--region_name", + type=str, + default=None, + help="Name of region. If from shapefile, value must be found under attribute given by --column", + required=False, + ) + + parser.add_argument( + "--coords", + type=list, + default=None, + help="List of coordinates for region bounds. Must be provided in consecutive order around shape perimeter. Only one of --shp_path, --coords can be used.", + required=False, + ) + + parser.add_argument( + "--generate_sftlf", + action="store_true", + help="Flag to generate land sea mask if not found.", + required=False, + ) + + parser.add_argument( + "--regrid", + type=bool, + default=True, + help="Set to False if model and reference data all use same grid.", + required=False, + ) + + parser.add_argument( + "--plots", + action="store_true", + help="Set to True to generate figures.", + required=False, + ) + parser.add_argument( + "--osyear", dest="osyear", type=int, help="Start year for reference data set" + ) + parser.add_argument( + "--msyear", dest="msyear", type=int, help="Start year for model data set" + ) + parser.add_argument( + "--oeyear", dest="oeyear", type=int, help="End year for reference data set" + ) + parser.add_argument( + "--meyear", dest="meyear", type=int, help="End year for model data set" + ) + parser.add_argument( + "--ObsUnitsAdjust", + type=tuple, + default=(False, 0, 0, None), + help="For unit adjust for OBS dataset. For example:\n" + "- (True, 'divide', 100.0, 'hPa') # Pa to hPa\n" + "- (True, 'subtract', 273.15, 'C') # degK to degC\n" + "- (False, 0, 0, None) # No adjustment (default)", + ) + parser.add_argument( + "--ModUnitsAdjust", + type=tuple, + default=(False, 0, 0, None), + help="For unit adjust for model dataset. For example:\n" + "- (True, 'divide', 100.0, 'hPa') # Pa to hPa\n" + "- (True, 'subtract', 273.15, 'C') # degK to degC\n" + "- (False, 0, 0, None) # No adjustment (default)", + ) + + parser.add_argument( + "--return_period", + type=int, + default=20, + help="Return period, in years, for obtaining return values.", + ) + + return parser diff --git a/pcmdi_metrics/drcdm/lib/metadata.py b/pcmdi_metrics/drcdm/lib/metadata.py new file mode 100644 index 000000000..df7edc791 --- /dev/null +++ b/pcmdi_metrics/drcdm/lib/metadata.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import json +import os + + +class MetadataFile: + # This class organizes the contents for the CMEC + # metadata file called output.json, which describes + # the other files in the output bundle. + + def __init__(self, metrics_output_path): + self.outfile = os.path.join(metrics_output_path, "output.json") + self.json = { + "provenance": { + "environment": "", + "modeldata": "", + "obsdata": "", + "log": "", + }, + "metrics": {}, + "data": {}, + "plots": {}, + } + + def update_metrics(self, kw, filename, longname, desc): + tmp = {"filename": filename, "longname": longname, "description": desc} + self.json["metrics"].update({kw: tmp}) + return + + def update_data(self, kw, filename, longname, desc): + tmp = {"filename": filename, "longname": longname, "description": desc} + self.json["data"].update({kw: tmp}) + return + + def update_plots(self, kw, filename, longname, desc): + tmp = {"filename": filename, "longname": longname, "description": desc} + self.json["plots"].update({kw: tmp}) + + def update_provenance(self, kw, data): + self.json["provenance"].update({kw: data}) + return + + def update_index(self, val): + self.json["index"] = val + return + + def write(self): + with open(self.outfile, "w") as f: + json.dump(self.json, f, indent=4) diff --git a/pcmdi_metrics/drcdm/lib/region_utilities.py b/pcmdi_metrics/drcdm/lib/region_utilities.py new file mode 100644 index 000000000..d2f599a55 --- /dev/null +++ b/pcmdi_metrics/drcdm/lib/region_utilities.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +import os +import sys + +import geopandas as gpd +import numpy as np +import regionmask + + +def check_region_params(shp_path, coords, region_name, col, default): + use_region_mask = False + + # Underscore will mess with file name slicing in extremes driver + if region_name is not None and "_" in region_name: + print("Error: Underscore character not permitted in region_name.") + sys.exit() + + if shp_path is not None: + use_region_mask = True + if not os.path.exists(shp_path): + print("Error: Shapefile path does not exist.") + print("Must provide valid shapefile path.") + sys.exit() + if region_name is None: + print( + "Error: Region name parameter --region_name must be provided with shapefile." + ) + sys.exit() + if col is None: + print( + "Error: Column name parameter --column must be provided with shapefile." + ) + sys.exit() + print("Region settings are:") + print(" Shapefile:", shp_path) + print(" Column name:", col) + print(" Region name:", region_name) + elif coords is not None: + use_region_mask = True + # Coords is a list that might be ingested badly + # from command line, so some cleanup is done here if needed. + if isinstance(coords, list): + if coords[0] == "[": + tmp = "" + for n in range(0, len(coords)): + tmp = tmp + str(coords[n]) + coords = tmp + if isinstance(coords, str): + tmp = coords.replace("[", "").replace("]", "").split(",") + coords = [ + [float(tmp[n]), float(tmp[n + 1])] for n in range(0, len(tmp) - 1, 2) + ] + if region_name is None: + print("No region name provided. Using 'custom'.") + region_name = "custom" + print("Region settings are:") + print(" Coordinates:", coords) + print(" Region name:", region_name) + else: + region_name = default + + return use_region_mask, region_name, coords + + +def mask_region(data, name, coords=None, shp_path=None, column=None): + # Return data masked from coordinate list or shapefile. + # Masks a single region + + lon = data["lon"].data + lat = data["lat"].data + + # Option 1: Region is defined by coord pairs + if coords is not None: + try: + names = [name] + regions = regionmask.Regions([np.array(coords)], names=names) + mask = regions.mask(lon, lat) + val = 0 + except Exception as e: + print("Error in creating mask from provided coordinates:") + print(" ", e) + sys.exit() + + # Option 2: region is defined by shapefile + elif shp_path is not None: + try: + regions_file = gpd.read_file(shp_path) + if column is not None: + regions = regionmask.from_geopandas(regions_file, names=column) + else: + print("Column name not provided.") + regions = regionmask.from_geopandas(regions_file) + mask = regions.mask(lon, lat) + # Can't match mask by name, rather index of name + val = list(regions_file[column]).index(name) + except Exception as e: + print("Error in creating mask from shapefile:") + print(" ", e) + sys.exit() + + else: + print("Error in masking: Region coordinates or shapefile must be provided.") + sys.exit() + + try: + masked_data = data.where(mask == val) + except Exception as e: + print("Error: Masking failed.") + print(" ", e) + sys.exit() + + return masked_data diff --git a/pcmdi_metrics/drcdm/lib/utilities.py b/pcmdi_metrics/drcdm/lib/utilities.py new file mode 100644 index 000000000..701dad016 --- /dev/null +++ b/pcmdi_metrics/drcdm/lib/utilities.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +import datetime +import os +import sys + +import cftime +import xcdat + +from pcmdi_metrics.io import xcdat_openxml +from pcmdi_metrics.io.base import Base +from pcmdi_metrics.utils import create_land_sea_mask + + +def load_dataset(filepath): + # Load an xarray dataset from the given filepath. + # If list of netcdf files, opens mfdataset. + # If list of xmls, open last file in list. + if filepath[-1].endswith(".xml"): + # Final item of sorted list would have most recent version date + ds = xcdat_openxml.xcdat_openxml(filepath[-1]) + elif len(filepath) > 1: + ds = xcdat.open_mfdataset(filepath, chunks=None) + else: + ds = xcdat.open_dataset(filepath[0]) + return ds + + +def slice_dataset(ds, start_year, end_year): + cal = ds.time.encoding["calendar"] + start_time = cftime.datetime(start_year, 1, 1, calendar=cal) - datetime.timedelta( + days=0 + ) + end_time = cftime.datetime(end_year + 1, 1, 1, calendar=cal) - datetime.timedelta( + days=1 + ) + ds = ds.sel(time=slice(start_time, end_time)) + return ds + + +def replace_multi(string, rdict): + # Replace multiple keyworks in a string template + # based on key-value pairs in 'rdict'. + for k in rdict.keys(): + string = string.replace(k, rdict[k]) + return string + + +def write_to_nc(data, model, run, region_name, index, years, ncdir, desc, meta): + # Consolidating some netcdf writing code here to streamline main function + yrs = "-".join(years) + filepath = os.path.join( + ncdir, "_".join([model, run, region_name, index, yrs]) + ".nc" + ) + write_netcdf_file(filepath, data) + meta.update_data(os.path.basename(filepath), filepath, index, desc) + return meta + + +def write_netcdf_file(filepath, ds): + try: + ds.to_netcdf(filepath, mode="w") + except PermissionError as e: + if os.path.exists(filepath): + print(" Permission error. Removing existing file", filepath) + os.remove(filepath) + print(" Writing new netcdf file", filepath) + ds.to_netcdf(filepath, mode="w") + else: + print(" Permission error. Could not write netcdf file", filepath) + print(" ", e) + except Exception as e: + print(" Error: Could not write netcdf file", filepath) + print(" ", e) + + +def write_to_json(outdir, json_filename, json_dict): + # Open JSON + JSON = Base(outdir, json_filename) + json_structure = json_dict["DIMENSIONS"]["json_structure"] + + JSON.write( + json_dict, + json_structure=json_structure, + sort_keys=True, + indent=4, + separators=(",", ": "), + ) + return + + +def verify_years(start_year, end_year, msg="Error: Invalid start or end year"): + if start_year is None and end_year is None: + return + elif start_year is None or end_year is None: + # If only one of the two is set, exit. + print(msg) + print("Exiting") + sys.exit() + + +def verify_output_path(metrics_output_path, case_id): + if metrics_output_path is None: + metrics_output_path = datetime.datetime.now().strftime("v%Y%m%d") + if case_id is not None: + metrics_output_path = metrics_output_path.replace("%(case_id)", case_id) + if not os.path.exists(metrics_output_path): + print("\nMetrics output path not found.") + print("Creating metrics output directory", metrics_output_path) + try: + os.makedirs(metrics_output_path) + except Exception as e: + print("\nError: Could not create metrics output path", metrics_output_path) + print(e) + print("Exiting.") + sys.exit() + return metrics_output_path + + +def set_up_realizations(realization): + find_all_realizations = False + if realization is None: + realization = "" + realizations = [realization] + elif isinstance(realization, str): + if realization.lower() in ["all", "*"]: + find_all_realizations = True + else: + realizations = [realization] + elif isinstance(realization, list): + realizations = realization + + return find_all_realizations, realizations + + +def generate_land_sea_mask(data, debug=False): + # generate sftlf if not provided. + sft = create_land_sea_mask(data) + sftlf = data.copy(data=None) + sftlf["sftlf"] = sft * 100 + + return sftlf From bd01ff01046c45177cf9818d5231ed0f726782d3 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 13:05:03 -0700 Subject: [PATCH 05/88] clean up and add todos --- pcmdi_metrics/drcdm/drcdm_driver.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index c4bd9dd99..d7a06ab36 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -24,6 +24,7 @@ case_id = parameter.case_id model_list = parameter.test_data_set realization = parameter.realization +# Expected variables - pr, tasmax, tasmin, tas variable_list = parameter.vars filename_template = parameter.filename_template sftlf_filename_template = parameter.sftlf_filename_template @@ -32,20 +33,16 @@ reference_data_set = parameter.reference_data_set reference_sftlf_template = parameter.reference_sftlf_template metrics_output_path = parameter.metrics_output_path +# Do we require variables to use certain units? ModUnitsAdjust = parameter.ModUnitsAdjust ObsUnitsAdjust = parameter.ObsUnitsAdjust plots = parameter.plots -debug = parameter.debug -cmec = parameter.cmec +# TODO: Some metrics require a baseline period. Do we use obs for that? Allow two model date ranges? msyear = parameter.msyear meyear = parameter.meyear osyear = parameter.osyear oeyear = parameter.oeyear generate_sftlf = parameter.generate_sftlf -regrid = parameter.regrid -cov_file = parameter.covariate_path -cov_name = parameter.covariate -return_period = parameter.return_period # Block extrema related settings annual_strict = parameter.annual_strict exclude_leap = parameter.exclude_leap @@ -112,8 +109,6 @@ region_name, ) -obs = {} - ############## # Run Analysis ############## @@ -234,7 +229,8 @@ if not sftlf_exists and generate_sftlf: print("Generating land sea mask.") - sftlf = utilities.generate_land_sea_mask(ds, debug=debug) + # TODO: are we using PMP version of land/sea mask? + sftlf = utilities.generate_land_sea_mask(ds, debug=False) if use_region_mask: # TODO: any way to not create the land/sea mask in two different places? print("\nCreating region mask for land/sea mask.") @@ -266,6 +262,15 @@ # ------------------------------- # Metrics go here # ------------------------------- + # Maybe start with the metrics from this paper: https://climatemodeling.science.energy.gov/sites/default/files/2023-11/Validation%20of%20LOCA2%20and%20STAR-ESDM%20Statistically%20Downscaled%20Products%20v2.pdf + # + # pr: annualmean_pr, seasonalmean_pr, pr_q50, pr_q99p9, annual pxx + # tasmax: annualmean_tasmax, seasonalmean_tasmax, annual txx, annual_tasmax_ge_95F, + # annual_tasmax_ge_100F, annual_tasmax_ge_105F, tasmax_q50, tasmax_q99p9 + # tasmin: annualmean_tasmin, annual_tasmin_le_32F, annual_tnn + # ETTCDI has some metrics for tas as well + # Putting a placeholder here for now: + print(model, run, varname) # ------------------------------- From c5a18ddd265eab6b025f2d871bef1a8f6b15538d Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 13:05:43 -0700 Subject: [PATCH 06/88] add drcdm driver --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ddd0b9c6b..32bb58c89 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ "pcmdi_metrics/cloud_feedback/cloud_feedback_driver.py", "pcmdi_metrics/extremes/extremes_driver.py", "pcmdi_metrics/sea_ice/sea_ice_driver.py", + "pcmdi_metrics/drcdm/drcdm_driver.py", ] entry_points = { From 75f7f2348b145b54fd411220550726df13292ad9 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 13:06:16 -0700 Subject: [PATCH 07/88] add init --- pcmdi_metrics/drcdm/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pcmdi_metrics/drcdm/__init__.py diff --git a/pcmdi_metrics/drcdm/__init__.py b/pcmdi_metrics/drcdm/__init__.py new file mode 100644 index 000000000..e69de29bb From 2d1c8b8ace7958b836f8ef2f5b86c3d67790e83a Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 13:58:08 -0700 Subject: [PATCH 08/88] update sftlf --- pcmdi_metrics/drcdm/drcdm_driver.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index d7a06ab36..388c5bfb3 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -11,6 +11,7 @@ region_utilities, utilities, ) +from pcmdi_metrics.utils import create_land_sea_mask ########## # Set up @@ -170,8 +171,6 @@ sftlf_exists = False if sftlf_exists: sftlf = xcdat.open_dataset(sftlf_filename, decode_times=False) - # Note: we should expect SFTLF to be between 0-1 like the rest of the metrics - # in the PMP do, not 0-100 like extremes. if use_region_mask: print("\nCreating region mask for land/sea mask.") sftlf = region_utilities.mask_region( @@ -229,10 +228,10 @@ if not sftlf_exists and generate_sftlf: print("Generating land sea mask.") - # TODO: are we using PMP version of land/sea mask? - sftlf = utilities.generate_land_sea_mask(ds, debug=False) + sft = create_land_sea_mask(ds) + sftlf = ds.copy(data=None) + sftlf["sftlf"] = sft if use_region_mask: - # TODO: any way to not create the land/sea mask in two different places? print("\nCreating region mask for land/sea mask.") sftlf = region_utilities.mask_region( sftlf, region_name, coords=coords, shp_path=shp_path, column=col From 938bbcb23e11b878024dfcfa765b125b135f6ddb Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 13:58:41 -0700 Subject: [PATCH 09/88] update sftlf --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 649fe58c7..61eff7dc2 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -53,8 +53,8 @@ def __init__( self.sftlf = sftlf["sftlf"] def masked_ds(self, ds): - # Mask land where 50<=sftlf<=100 - return ds.where(self.sftlf >= 50).where(self.sftlf <= 100) + # Mask land where 0.5<=sftlf<=1 + return ds.where(self.sftlf >= 0.5).where(self.sftlf <= 1) def calc_5day_mean(self): # Get the 5-day mean dataset @@ -460,7 +460,7 @@ def init_metrics_dict( "statistic", "season", ], - "region": {region_name: "Areas where 50<=sftlf<=100"}, + "region": {region_name: "Areas where 0.5<=sftlf<=1"}, "season": ["ANN", "DJF", "MAM", "JJA", "SON"], "index": {}, "statistic": { From 3f3cb7a2cbb83000dff60cbcee7df74663b217fe Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 14:07:13 -0700 Subject: [PATCH 10/88] update stats --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 56 +++++++++------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 61eff7dc2..0b640e37e 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -6,7 +6,7 @@ import xarray as xr import xcdat as xc -from pcmdi_metrics.mean_climate.lib import compute_statistics +from pcmdi_metrics.stats import compute_statistics_dataset as pmp_stats class TimeSeriesData: @@ -464,14 +464,14 @@ def init_metrics_dict( "season": ["ANN", "DJF", "MAM", "JJA", "SON"], "index": {}, "statistic": { - "mean": compute_statistics.mean_xy(None), - "std_xy": compute_statistics.std_xy(None, None), - "bias_xy": compute_statistics.bias_xy(None, None), - "cor_xy": compute_statistics.cor_xy(None, None), - "mae_xy": compute_statistics.meanabs_xy(None, None), - "rms_xy": compute_statistics.rms_xy(None, None), - "rmsc_xy": compute_statistics.rmsc_xy(None, None), - "std-obs_xy": compute_statistics.std_xy(None, None), + "mean": pmp_stats.mean_xy(None), + "std_xy": pmp_stats.std_xy(None, None), + "bias_xy": pmp_stats.bias_xy(None, None), + "cor_xy": pmp_stats.cor_xy(None, None), + "mae_xy": pmp_stats.meanabs_xy(None, None), + "rms_xy": pmp_stats.rms_xy(None, None), + "rmsc_xy": pmp_stats.rmsc_xy(None, None), + "std-obs_xy": pmp_stats.std_xy(None, None), "pct_dif": { "Abstract": "Bias xy as a percentage of the Observed mean.", "Contact": "pcmdi-metrics@llnl.gov", @@ -555,7 +555,7 @@ def metrics_json(data_dict, obs_dict={}, region="land", regrid=True): # Global mean over land met_dict[m][region]["mean"][season] = mean_xy(ds_m, season) a = ds_m.temporal.average(season) - std_xy = compute_statistics.std_xy(a, season) + std_xy = pmp_stats.std_xy(a, season) met_dict[m][region]["std_xy"][season] = std_xy if len(obs_dict) > 0 and not obs_dict[m].equals(ds_m): @@ -578,14 +578,12 @@ def metrics_json(data_dict, obs_dict={}, region="land", regrid=True): a = ds_m.temporal.average(season) b = obs_m.temporal.average(season) weights = ds_m.spatial.get_weights(axis=["X", "Y"]) - rms_xy = compute_statistics.rms_xy(a, b, var=season, weights=weights) - meanabs_xy = compute_statistics.meanabs_xy( - a, b, var=season, weights=weights - ) - bias_xy = compute_statistics.bias_xy(a, b, var=season, weights=weights) - cor_xy = compute_statistics.cor_xy(a, b, var=season, weights=weights) - rmsc_xy = compute_statistics.rmsc_xy(a, b, var=season, weights=weights) - std_obs_xy = compute_statistics.std_xy(b, season) + rms_xy = pmp_stats.rms_xy(a, b, var=season, weights=weights) + meanabs_xy = pmp_stats.meanabs_xy(a, b, var=season, weights=weights) + bias_xy = pmp_stats.bias_xy(a, b, var=season, weights=weights) + cor_xy = pmp_stats.cor_xy(a, b, var=season, weights=weights) + rmsc_xy = pmp_stats.rmsc_xy(a, b, var=season, weights=weights) + std_obs_xy = pmp_stats.std_xy(b, season) pct_dif = percent_difference(b, bias_xy, season, weights) met_dict[m][region]["pct_dif"][season] = pct_dif @@ -641,7 +639,7 @@ def metrics_json_return_value( # Global mean over land rv_tmp[season] = remove_outliers(rv[season], blockex[season]) met_dict[stat][region]["mean"][season] = mean_xy(rv_tmp, season) - std_xy = compute_statistics.std_xy(rv_tmp, season) + std_xy = pmp_stats.std_xy(rv_tmp, season) met_dict[stat][region]["std_xy"][season] = std_xy if obs is not None and not obs[season].equals(rv_tmp): @@ -661,22 +659,14 @@ def metrics_json_return_value( # Get xy stats for temporal average weights = rv_tmp.spatial.get_weights(axis=["X", "Y"]) - rms_xy = compute_statistics.rms_xy( - rv_tmp, obs_m, var=season, weights=weights - ) - meanabs_xy = compute_statistics.meanabs_xy( - rv_tmp, obs_m, var=season, weights=weights - ) - bias_xy = compute_statistics.bias_xy( - rv_tmp, obs_m, var=season, weights=weights - ) - cor_xy = compute_statistics.cor_xy( - rv_tmp, obs_m, var=season, weights=weights - ) - rmsc_xy = compute_statistics.rmsc_xy( + rms_xy = pmp_stats.rms_xy(rv_tmp, obs_m, var=season, weights=weights) + meanabs_xy = pmp_stats.meanabs_xy( rv_tmp, obs_m, var=season, weights=weights ) - std_obs_xy = compute_statistics.std_xy(rv_tmp, season) + bias_xy = pmp_stats.bias_xy(rv_tmp, obs_m, var=season, weights=weights) + cor_xy = pmp_stats.cor_xy(rv_tmp, obs_m, var=season, weights=weights) + rmsc_xy = pmp_stats.rmsc_xy(rv_tmp, obs_m, var=season, weights=weights) + std_obs_xy = pmp_stats.std_xy(rv_tmp, season) pct_dif = percent_difference(obs_m, bias_xy, season, weights) met_dict[stat][region]["pct_dif"][season] = pct_dif From 5e2c861b415bfbb8183727c905d0228cd028d2a5 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 14:14:55 -0700 Subject: [PATCH 11/88] add param --- pcmdi_metrics/drcdm/param/drcdm_param.py | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 pcmdi_metrics/drcdm/param/drcdm_param.py diff --git a/pcmdi_metrics/drcdm/param/drcdm_param.py b/pcmdi_metrics/drcdm/param/drcdm_param.py new file mode 100644 index 000000000..3f4c113ee --- /dev/null +++ b/pcmdi_metrics/drcdm/param/drcdm_param.py @@ -0,0 +1,35 @@ +# Settings for extremes driver + +# These settings are required +vars = ["pr"] # Choices are 'pr','tasmax', 'tasmin' +test_data_set = ["MIROC6"] +realization = ["r1i1p1f1"] +test_data_path = "/p/css03/esgf_publish/CMIP6/CMIP/MIROC/MIROC6/1pctCO2/r1i1p1f1/day/%(variable)/gn/v20191016/" +filename_template = "%(variable)_day_MIROC6_1pctCO2_r1i1p1f1_gn_*.nc" + +metrics_output_path = "/home/ordonez4/pmp_results/drcdm/test/" + +# Note: You can use the following placeholders in file templates: +# %(variable) to substitute variable name from "vars" (except in sftlf filenames) +# %(model) to substitute model name from "test_data_set" +# %(realization) to substitute realization from "realization" + +# Optional settings +# See the README for more information about these settings +case_id = "test" +# sftlf_filename_template = '/p/css03/esgf_publish/CMIP6/CMIP/MIROC/MIROC6/piControl/r1i1p1f1/fx/sftlf/gn/v20190311/sftlf_fx_MIROC6_piControl_r1i1p1f1_gn.nc' + +ModUnitsAdjust = ( + True, + "multiply", + 86400.0, + "mm/day", +) # Convert model units from kg/m2/s to mm/day +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = True +regrid = False +plots = False +generate_sftlf = True +msyear = 3300 +meyear = 3349 From df5d077463802219efd0368ce79543665d620180 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 14:40:49 -0700 Subject: [PATCH 12/88] add README --- pcmdi_metrics/drcdm/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 pcmdi_metrics/drcdm/README.md diff --git a/pcmdi_metrics/drcdm/README.md b/pcmdi_metrics/drcdm/README.md new file mode 100644 index 000000000..1609ca8f7 --- /dev/null +++ b/pcmdi_metrics/drcdm/README.md @@ -0,0 +1 @@ +# Decision Relevant Climate Data Metrics \ No newline at end of file From 1ba973dd328e7907fea072b5642f5322505737cc Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 14:47:12 -0700 Subject: [PATCH 13/88] add example --- pcmdi_metrics/drcdm/drcdm_driver.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 388c5bfb3..f30c1eedc 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -268,8 +268,19 @@ # annual_tasmax_ge_100F, annual_tasmax_ge_105F, tasmax_q50, tasmax_q99p9 # tasmin: annualmean_tasmin, annual_tasmin_le_32F, annual_tnn # ETTCDI has some metrics for tas as well - # Putting a placeholder here for now: - print(model, run, varname) + + # In the extremes metrics, I go by variable and call compute_metrics.temperature_indices() or + # compute_metrics.precipitation_indices() and those functions handle all the data wrangling + # for getting all the indices for that variable. Not sure if that's the best way to do it here. + # For example: + Rx1day, Rx5day = compute_metrics.precipitation_indices( + ds, + sftlf, + units_adjust, + dec_mode, + drop_incomplete_djf, + annual_strict, + ) # ------------------------------- From 96e60661ad9145ff44410fb3b6bb699c64939379 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 16 May 2024 14:56:00 -0700 Subject: [PATCH 14/88] add test info --- pcmdi_metrics/drcdm/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/README.md b/pcmdi_metrics/drcdm/README.md index 1609ca8f7..694d91f46 100644 --- a/pcmdi_metrics/drcdm/README.md +++ b/pcmdi_metrics/drcdm/README.md @@ -1 +1,11 @@ -# Decision Relevant Climate Data Metrics \ No newline at end of file +# Decision Relevant Climate Data Metrics + +# How to test: +Create a conda environment with pcmdi_metrics and xclim +In the PMP root directory use: +`pip install .` + +Edit the metrics output path in pcmdi_metrics/drcdm/param/drcdm_param.py to be a location at which you have write permission. + +To launch a run with the demo parameter file use: +`drcdm_driver.py -p pcmdi_metrics/drcdm/param/drcdm_param.py` \ No newline at end of file From c1eabfb99589231938dca7cc268be3953f8b290d Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 7 Jun 2024 16:47:51 -0700 Subject: [PATCH 15/88] add temp ind --- pcmdi_metrics/drcdm/drcdm_driver.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index f30c1eedc..b6b000509 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -273,14 +273,7 @@ # compute_metrics.precipitation_indices() and those functions handle all the data wrangling # for getting all the indices for that variable. Not sure if that's the best way to do it here. # For example: - Rx1day, Rx5day = compute_metrics.precipitation_indices( - ds, - sftlf, - units_adjust, - dec_mode, - drop_incomplete_djf, - annual_strict, - ) + tasmax_metrics = compute_metrics.temperature_indices(ds) # ------------------------------- From b31c6a03af4ba1525c010ae59287e75ea1a3ee21 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 7 Jun 2024 16:48:08 -0700 Subject: [PATCH 16/88] swap for mean --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 163 +++++---------------- 1 file changed, 33 insertions(+), 130 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 0b640e37e..269b5e488 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -118,12 +118,20 @@ def annual_stats(self, stat, pentad=False): .groupby("time.year") .min(dim="time") ) + elif stat == "mean": + ds_ann = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .mean(dim="time") + ) else: # Group by date if stat == "max": ds_ann = ds.groupby("time.year").max(dim="time") elif stat == "min": ds_ann = ds.groupby("time.year").min(dim="time") + elif stat == "mean": + ds_ann = ds.groupby("time.year").mean(dim="time") # Need to fix time axis if groupby operation happened if "year" in ds_ann.coords: @@ -157,6 +165,8 @@ def seasonal_stats(self, season, stat, pentad=False): ds_stat = ds.resample(time="QS-DEC").max(dim="time") elif stat == "min": ds_stat = ds.resample(time="QS-DEC").min(dim="time") + elif stat == "mean": + ds_stat = ds.resample(time="QS-DEC").mean(dim="time") ds_stat = ds_stat.isel(time=ds_stat.time.dt.month.isin([12])) @@ -232,6 +242,12 @@ def seasonal_stats(self, season, stat, pentad=False): .groupby("time.year") .min(dim="time") ) + elif stat == "mean": + ds_stat = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .mean(dim="time") + ) else: # Other 3 seasons dates = { # Month/day tuples @@ -272,6 +288,12 @@ def seasonal_stats(self, season, stat, pentad=False): .groupby("time.year") .min(dim="time") ) + elif stat == "mean": + ds_stat = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .mean(dim="time") + ) # Need to fix time axis if groupby operation happened if "year" in ds_stat.coords: @@ -342,7 +364,7 @@ def convert_units(data, units_adjust): def temperature_indices( ds, varname, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict ): - # Returns annual max and min of provided temperature dataset + # Returns annual daily mean of provided temperature dataset # Temperature input can be "tasmax" or "tasmin". print("Generating temperature block extrema.") @@ -358,19 +380,17 @@ def temperature_indices( annual_strict=annual_strict, ) - Tmax = xr.Dataset() - Tmin = xr.Dataset() - Tmax["ANN"] = S.annual_stats("max") - Tmin["ANN"] = S.annual_stats("min") + Tmean = xr.Dataset() + Tmean["ANN"] = S.annual_stats("mean") for season in ["DJF", "MAM", "JJA", "SON"]: - Tmax[season] = S.seasonal_stats(season, "max") - Tmin[season] = S.seasonal_stats(season, "min") + Tmean[season] = S.seasonal_stats(season, "mean") + + Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) - Tmax = update_nc_attrs(Tmax, dec_mode, drop_incomplete_djf, annual_strict) - Tmin = update_nc_attrs(Tmin, dec_mode, drop_incomplete_djf, annual_strict) + # TODO: add quantile metrics - return Tmax, Tmin + return Tmean def precipitation_indices( @@ -395,7 +415,7 @@ def precipitation_indices( # Rx1day P1day = xr.Dataset() - P1day["ANN"] = S.annual_stats("max", pentad=False) + P1day["ANN"] = S.annual_stats("mean", pentad=False) # Can end up with very small negative values that should be 0 # Possibly related to this issue? https://github.com/pydata/bottleneck/issues/332 # (from https://github.com/pydata/xarray/issues/3855) @@ -403,7 +423,7 @@ def precipitation_indices( P1day["ANN"].where(P1day["ANN"] > 0, 0).where(~np.isnan(P1day["ANN"]), np.nan) ) for season in ["DJF", "MAM", "JJA", "SON"]: - P1day[season] = S.seasonal_stats(season, "max", pentad=False) + P1day[season] = S.seasonal_stats(season, "mean", pentad=False) P1day[season] = ( P1day[season] .where(P1day[season] > 0, 0) @@ -411,22 +431,7 @@ def precipitation_indices( ) P1day = update_nc_attrs(P1day, dec_mode, drop_incomplete_djf, annual_strict) - # Rx5day - P5day = xr.Dataset() - P5day["ANN"] = S.annual_stats("max", pentad=True) - P5day["ANN"] = ( - P5day["ANN"].where(P5day["ANN"] > 0, 0).where(~np.isnan(P5day["ANN"]), np.nan) - ) - for season in ["DJF", "MAM", "JJA", "SON"]: - P5day[season] = S.seasonal_stats(season, "max", pentad=True) - P5day[season] = ( - P5day[season] - .where(P5day[season] > 0, 0) - .where(~np.isnan(P5day[season]), np.nan) - ) - P5day = update_nc_attrs(P5day, dec_mode, drop_incomplete_djf, annual_strict) - - return P1day, P5day + return P1day # A couple of statistics that aren't being loaded from mean_climate @@ -595,105 +600,3 @@ def metrics_json(data_dict, obs_dict={}, region="land", regrid=True): met_dict[m][region]["std-obs_xy"][season] = std_obs_xy return met_dict - - -def metrics_json_return_value( - rv, blockex, obs, blockex_obs, stat, region="land", regrid=True -): - # Generate metrics for stationary return value comparing model and obs - # Arguments: - # rv: dataset - # blockex: dataset - # obs: dataset - # stat: string - # region: string - # regrid: bool - # Returns: - # met_dict: dictionary - met_dict = {stat: {}} - seasons_dict = {"ANN": "", "DJF": "", "MAM": "", "JJA": "", "SON": ""} - - # Looping over each type of extrema in data_dict - met_dict[stat] = { - region: {"mean": seasons_dict.copy(), "std_xy": seasons_dict.copy()} - } - # If obs available, add metrics comparing with obs - # If new statistics are added, be sure to update - # "statistic" entry in init_metrics_dict() - if obs is not None: - obs = obs.bounds.add_missing_bounds() - for k in [ - "std-obs_xy", - "pct_dif", - "bias_xy", - "cor_xy", - "mae_xy", - "rms_xy", - "rmsc_xy", - ]: - met_dict[stat][region][k] = seasons_dict.copy() - - rv_tmp = rv.copy(deep=True) - - for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: - # Global mean over land - rv_tmp[season] = remove_outliers(rv[season], blockex[season]) - met_dict[stat][region]["mean"][season] = mean_xy(rv_tmp, season) - std_xy = pmp_stats.std_xy(rv_tmp, season) - met_dict[stat][region]["std_xy"][season] = std_xy - - if obs is not None and not obs[season].equals(rv_tmp): - obs[season] = remove_outliers(obs[season], blockex_obs[season]) - # Regrid obs to model grid - if regrid: - target = xc.create_grid(rv_tmp.lat, rv_tmp.lon) - target = target.bounds.add_missing_bounds(["X", "Y"]) - obs_m = obs.regridder.horizontal(season, target, tool="regrid2") - else: - obs_m = obs - shp1 = (len(rv_tmp.lat), len(rv_tmp.lon)) - shp2 = (len(obs.lat), len(obs.lon)) - assert ( - shp1 == shp2 - ), "Model and Reference data dimensions 'lat' and 'lon' must match." - - # Get xy stats for temporal average - weights = rv_tmp.spatial.get_weights(axis=["X", "Y"]) - rms_xy = pmp_stats.rms_xy(rv_tmp, obs_m, var=season, weights=weights) - meanabs_xy = pmp_stats.meanabs_xy( - rv_tmp, obs_m, var=season, weights=weights - ) - bias_xy = pmp_stats.bias_xy(rv_tmp, obs_m, var=season, weights=weights) - cor_xy = pmp_stats.cor_xy(rv_tmp, obs_m, var=season, weights=weights) - rmsc_xy = pmp_stats.rmsc_xy(rv_tmp, obs_m, var=season, weights=weights) - std_obs_xy = pmp_stats.std_xy(rv_tmp, season) - pct_dif = percent_difference(obs_m, bias_xy, season, weights) - - met_dict[stat][region]["pct_dif"][season] = pct_dif - met_dict[stat][region]["rms_xy"][season] = rms_xy - met_dict[stat][region]["mae_xy"][season] = meanabs_xy - met_dict[stat][region]["bias_xy"][season] = bias_xy - met_dict[stat][region]["cor_xy"][season] = cor_xy - met_dict[stat][region]["rmsc_xy"][season] = rmsc_xy - met_dict[stat][region]["std-obs_xy"][season] = std_obs_xy - - return met_dict - - -def remove_outliers(rv, blockex): - # Remove outlier return values for metrics computation - # filtering by comparing to the block extreme values - # rv: data array - # blckex: data array - block_max = blockex.max("time", skipna=True).data - block_min = blockex.min("time", skipna=True).data - block_std = blockex.std("time", skipna=True).data - - # Remove values that are either: - # 8 standard deviations above the max value in block extema - # 8 standard deviations below the min value in block extrema - tol = 8 * block_std - plussig = block_max + tol - minsig = block_min - tol - rv_remove_outliers = rv.where((rv < plussig) & (rv > minsig)) - return rv_remove_outliers From a5646597d329c70900f47b20a0e799820cff78d2 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 18 Jun 2024 17:07:36 -0700 Subject: [PATCH 17/88] add tasmin --- pcmdi_metrics/drcdm/drcdm_driver.py | 42 +++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index b6b000509..77fb95ef3 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -269,12 +269,44 @@ # tasmin: annualmean_tasmin, annual_tasmin_le_32F, annual_tnn # ETTCDI has some metrics for tas as well - # In the extremes metrics, I go by variable and call compute_metrics.temperature_indices() or - # compute_metrics.precipitation_indices() and those functions handle all the data wrangling - # for getting all the indices for that variable. Not sure if that's the best way to do it here. - # For example: - tasmax_metrics = compute_metrics.temperature_indices(ds) + if varname == "tasmax": + # tasmax metrics + ( + Tmean, + Tmedian, + Tq99p9, + Tge95, + Tge100, + Tge105, + ) = compute_metrics.tasmax_indices( + ds, + sftlf, + units_adjust, + dec_mode, + drop_incomplete_djf, + annual_strict, + ) + # Printing the arrays just to check that they exist for development purposes. + print(Tmean) + print(Tmedian) + print(Tq99p9) + print(Tge95) + print(Tge100) + print(Tge105) + elif varname == "tasmin": + # tasmin metrics + Tmean, Tmin, Tle32 = compute_metrics.tasmin_indices( + ds, + sftlf, + units_adjust, + dec_mode, + drop_incomplete_djf, + annual_strict, + ) + print(Tmean) + print(Tmin) + print(Tle32) # ------------------------------- # Output JSON with metrics here From ec071416419a09d1d7d067551c37c2dddbf0f00d Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 18 Jun 2024 17:08:01 -0700 Subject: [PATCH 18/88] add diags --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 137 ++++++++++++++++++++- 1 file changed, 131 insertions(+), 6 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 269b5e488..c48d33ebd 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -124,6 +124,43 @@ def annual_stats(self, stat, pentad=False): .groupby("time.year") .mean(dim="time") ) + elif stat == "median": + ds_ann = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .median(dim="time") + ) + elif stat == "q99p9": + ds_ann = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .quantile(0.999, dim="time") + ) + elif stat.startswith("ge"): + num = int(stat.replace("ge", "")) + ds_ann = ( + ds.where(ds >= num) + .groupby("time.year") + .sel(time=date_range, method="nearest") + .count(dim="time") + / ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .count(dim="time") + * 100 + ) + elif stat.startswith("le"): + num = int(stat.replace("le", "")) + ds_ann = ( + ds.where(ds <= num) + .groupby("time.year") + .sel(time=date_range, method="nearest") + .count(dim="time") + / ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .count(dim="time") + * 100 + ) + else: # Group by date if stat == "max": @@ -132,6 +169,24 @@ def annual_stats(self, stat, pentad=False): ds_ann = ds.groupby("time.year").min(dim="time") elif stat == "mean": ds_ann = ds.groupby("time.year").mean(dim="time") + elif stat == "median": + ds_ann = ds.groupby("time.year").median(dim="time") + elif stat == "q99p9": + ds_ann = ds.groupby("time.year").quantile(0.999, dim="time") + elif stat.startswith("ge"): + num = int(stat.replace("ge", "")) + ds_ann = ( + ds.where(ds >= num).groupby("time.year").count(dim="time") + / ds.groupby("time.year").count(dim="time") + * 100 + ) + elif stat.startswith("le"): + num = int(stat.replace("le", "")) + ds_ann = ( + ds.where(ds <= num).groupby("time.year").count(dim="time") + / ds.groupby("time.year").count(dim="time") + * 100 + ) # Need to fix time axis if groupby operation happened if "year" in ds_ann.coords: @@ -167,6 +222,8 @@ def seasonal_stats(self, season, stat, pentad=False): ds_stat = ds.resample(time="QS-DEC").min(dim="time") elif stat == "mean": ds_stat = ds.resample(time="QS-DEC").mean(dim="time") + elif stat == "median": + ds_stat = ds.resample(time="QS-DEC").median(dim="time") ds_stat = ds_stat.isel(time=ds_stat.time.dt.month.isin([12])) @@ -248,6 +305,12 @@ def seasonal_stats(self, season, stat, pentad=False): .groupby("time.year") .mean(dim="time") ) + elif stat == "median": + ds_stat = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .median(dim="time") + ) else: # Other 3 seasons dates = { # Month/day tuples @@ -294,6 +357,12 @@ def seasonal_stats(self, season, stat, pentad=False): .groupby("time.year") .mean(dim="time") ) + elif stat == "median": + ds_stat = ( + ds.sel(time=date_range, method="nearest") + .groupby("time.year") + .median(dim="time") + ) # Need to fix time axis if groupby operation happened if "year" in ds_stat.coords: @@ -327,7 +396,8 @@ def update_nc_attrs(ds, dec_mode, drop_incomplete_djf, annual_strict): ds.lon_bnds.encoding["_FillValue"] = None ds.time_bnds.encoding["_FillValue"] = None for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: - ds[season].encoding["_FillValue"] = float(1e20) + if season in ds: + ds[season].encoding["_FillValue"] = float(1e20) # Drop type attribute that comes from land mask if "type" in ds: @@ -361,14 +431,14 @@ def convert_units(data, units_adjust): return data -def temperature_indices( - ds, varname, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict +def tasmax_indices( + ds, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict ): # Returns annual daily mean of provided temperature dataset # Temperature input can be "tasmax" or "tasmin". print("Generating temperature block extrema.") - + varname = "tasmax" ds[varname] = convert_units(ds[varname], units_adjust) TS = TimeSeriesData(ds, varname) @@ -381,16 +451,71 @@ def temperature_indices( ) Tmean = xr.Dataset() + Tmedian = xr.Dataset() + Tq99p9 = xr.Dataset() + Tge95 = xr.Dataset() + Tge100 = xr.Dataset() + Tge105 = xr.Dataset() Tmean["ANN"] = S.annual_stats("mean") + Tmedian["ANN"] = S.annual_stats("median") + Tq99p9["ANN"] = S.annual_stats("q99p9") + Tge95["ANN"] = S.annual_stats("ge95") + Tge100["ANN"] = S.annual_stats("ge100") + Tge105["ANN"] = S.annual_stats("ge105") for season in ["DJF", "MAM", "JJA", "SON"]: Tmean[season] = S.seasonal_stats(season, "mean") + Tge95.attrs["units"] = "%" + Tge100.attrs["units"] = "%" + Tge105.attrs["units"] = "%" + Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) + Tmedian = update_nc_attrs(Tmedian, dec_mode, drop_incomplete_djf, annual_strict) + Tq99p9 = update_nc_attrs(Tq99p9, dec_mode, drop_incomplete_djf, annual_strict) + Tge95 = update_nc_attrs(Tge95, dec_mode, drop_incomplete_djf, annual_strict) + Tge100 = update_nc_attrs(Tge100, dec_mode, drop_incomplete_djf, annual_strict) + Tge105 = update_nc_attrs(Tge105, dec_mode, drop_incomplete_djf, annual_strict) + + return Tmean, Tmedian, Tq99p9, Tge95, Tge100, Tge105 + - # TODO: add quantile metrics +def tasmin_indices( + ds, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict +): + # Returns annual daily mean of provided temperature dataset + # Temperature input can be "tasmax" or "tasmin". + + print("Generating temperature block extrema.") + varname = "tasmin" + ds[varname] = convert_units(ds[varname], units_adjust) + + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + Tmean = xr.Dataset() + Tmin = xr.Dataset() + Tle32 = xr.Dataset() + Tmean["ANN"] = S.annual_stats("mean") + Tmin["ANN"] = S.annual_stats("min") + Tle32["ANN"] = S.annual_stats("le32") + + for season in ["DJF", "MAM", "JJA", "SON"]: + Tmean[season] = S.seasonal_stats(season, "mean") + + Tle32.attrs["units"] = "%" + + Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) + Tmin = update_nc_attrs(Tmin, dec_mode, drop_incomplete_djf, annual_strict) + Tle32 = update_nc_attrs(Tle32, dec_mode, drop_incomplete_djf, annual_strict) - return Tmean + return Tmean, Tmin, Tle32 def precipitation_indices( From aefa41ce06a4a3271806608be0b337870655f217 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 18 Jun 2024 17:08:17 -0700 Subject: [PATCH 19/88] chunks --- pcmdi_metrics/drcdm/lib/utilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/utilities.py b/pcmdi_metrics/drcdm/lib/utilities.py index 701dad016..f4df82f83 100644 --- a/pcmdi_metrics/drcdm/lib/utilities.py +++ b/pcmdi_metrics/drcdm/lib/utilities.py @@ -19,9 +19,9 @@ def load_dataset(filepath): # Final item of sorted list would have most recent version date ds = xcdat_openxml.xcdat_openxml(filepath[-1]) elif len(filepath) > 1: - ds = xcdat.open_mfdataset(filepath, chunks=None) + ds = xcdat.open_mfdataset(filepath, chunks={"time": -1}) else: - ds = xcdat.open_dataset(filepath[0]) + ds = xcdat.open_dataset(filepath[0], chunks={"time": -1}) return ds From a06397176777a2f5c43c2be9335c93e44dbce6c5 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 20 Jun 2024 12:25:09 -0700 Subject: [PATCH 20/88] refactor --- pcmdi_metrics/drcdm/drcdm_driver.py | 646 +++++++++++++++------------- 1 file changed, 347 insertions(+), 299 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 77fb95ef3..8dda3c146 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import glob +import json import os import xcdat @@ -13,316 +14,363 @@ ) from pcmdi_metrics.utils import create_land_sea_mask -########## -# Set up -########## - -parser = create_drcdm_parser.create_extremes_parser() -parameter = parser.get_parameter(argparse_vals_only=False) - -# Parameters -# I/O settings -case_id = parameter.case_id -model_list = parameter.test_data_set -realization = parameter.realization -# Expected variables - pr, tasmax, tasmin, tas -variable_list = parameter.vars -filename_template = parameter.filename_template -sftlf_filename_template = parameter.sftlf_filename_template -test_data_path = parameter.test_data_path -reference_data_path = parameter.reference_data_path -reference_data_set = parameter.reference_data_set -reference_sftlf_template = parameter.reference_sftlf_template -metrics_output_path = parameter.metrics_output_path -# Do we require variables to use certain units? -ModUnitsAdjust = parameter.ModUnitsAdjust -ObsUnitsAdjust = parameter.ObsUnitsAdjust -plots = parameter.plots -# TODO: Some metrics require a baseline period. Do we use obs for that? Allow two model date ranges? -msyear = parameter.msyear -meyear = parameter.meyear -osyear = parameter.osyear -oeyear = parameter.oeyear -generate_sftlf = parameter.generate_sftlf -# Block extrema related settings -annual_strict = parameter.annual_strict -exclude_leap = parameter.exclude_leap -dec_mode = parameter.dec_mode -drop_incomplete_djf = parameter.drop_incomplete_djf -# Region masking -shp_path = parameter.shp_path -col = parameter.attribute -region_name = parameter.region_name -coords = parameter.coords - -# Check the region masking parameters, if provided -use_region_mask, region_name, coords = region_utilities.check_region_params( - shp_path, coords, region_name, col, "land" -) - -# Verifying output directory -metrics_output_path = utilities.verify_output_path(metrics_output_path, case_id) - -if isinstance(reference_data_set, list): - # Fix a command line issue - reference_data_set = reference_data_set[0] - -# Verify years -ok_mod = utilities.verify_years( - msyear, - meyear, - msg="Error: Model msyear and meyear must both be set or both be None (unset).", -) -ok_obs = utilities.verify_years( - osyear, - oeyear, - msg="Error: Obs osyear and oeyear must both be set or both be None (unset).", -) - -# Initialize output.json file -meta = metadata.MetadataFile(metrics_output_path) - -# Initialize other directories -# Not sure if needed so commented for now. -# nc_dir = os.path.join(metrics_output_path, "netcdf") -# os.makedirs(nc_dir, exist_ok=True) -# if plots: -# plot_dir_maps = os.path.join(metrics_output_path, "plots", "maps") -# os.makedirs(plot_dir_maps, exist_ok=True) - -# Setting up model realization list -find_all_realizations, realizations = utilities.set_up_realizations(realization) - -# Only include reference data in loop if it exists -if reference_data_path is not None: - model_loop_list = ["Reference"] + model_list -else: - model_loop_list = model_list - -# Initialize output JSON structures -# FYI: if the analysis output JSON is changed, remember to update this function! -metrics_dict = compute_metrics.init_metrics_dict( - model_loop_list, - variable_list, - dec_mode, - drop_incomplete_djf, - annual_strict, - region_name, -) - -############## -# Run Analysis -############## - -# Loop over models -for model in model_loop_list: - if model == "Reference": - list_of_runs = [str(reference_data_set)] - elif find_all_realizations: - tags = {"%(model)": model, "%(model_version)": model, "%(realization)": "*"} - test_data_full_path = os.path.join(test_data_path, filename_template) - test_data_full_path = utilities.replace_multi(test_data_full_path, tags) - ncfiles = glob.glob(test_data_full_path) - realizations = [] - for ncfile in ncfiles: - realizations.append(ncfile.split("/")[-1].split(".")[3]) - print("=================================") - print("model, runs:", model, realizations) - list_of_runs = realizations +if __name__ == "__main__": + ########## + # Set up + ########## + + parser = create_drcdm_parser.create_extremes_parser() + parameter = parser.get_parameter(argparse_vals_only=False) + + # Parameters + # I/O settings + case_id = parameter.case_id + model_list = parameter.test_data_set + realization = parameter.realization + # Expected variables - pr, tasmax, tasmin, tas + variable_list = parameter.vars + filename_template = parameter.filename_template + sftlf_filename_template = parameter.sftlf_filename_template + test_data_path = parameter.test_data_path + reference_data_path = parameter.reference_data_path + reference_data_set = parameter.reference_data_set + reference_sftlf_template = parameter.reference_sftlf_template + metrics_output_path = parameter.metrics_output_path + # Do we require variables to use certain units? + ModUnitsAdjust = parameter.ModUnitsAdjust + ObsUnitsAdjust = parameter.ObsUnitsAdjust + plots = parameter.plots + # TODO: Some metrics require a baseline period. Do we use obs for that? Allow two model date ranges? + msyear = parameter.msyear + meyear = parameter.meyear + osyear = parameter.osyear + oeyear = parameter.oeyear + generate_sftlf = parameter.generate_sftlf + # Block extrema related settings + annual_strict = parameter.annual_strict + exclude_leap = parameter.exclude_leap + dec_mode = parameter.dec_mode + drop_incomplete_djf = parameter.drop_incomplete_djf + # Region masking + shp_path = parameter.shp_path + col = parameter.attribute + region_name = parameter.region_name + coords = parameter.coords + + # Check the region masking parameters, if provided + use_region_mask, region_name, coords = region_utilities.check_region_params( + shp_path, coords, region_name, col, "land" + ) + + # Verifying output directory + metrics_output_path = utilities.verify_output_path(metrics_output_path, case_id) + + if isinstance(reference_data_set, list): + # Fix a command line issue + reference_data_set = reference_data_set[0] + + # Verify years + ok_mod = utilities.verify_years( + msyear, + meyear, + msg="Error: Model msyear and meyear must both be set or both be None (unset).", + ) + ok_obs = utilities.verify_years( + osyear, + oeyear, + msg="Error: Obs osyear and oeyear must both be set or both be None (unset).", + ) + + # Initialize output.json file + meta = metadata.MetadataFile(metrics_output_path) + + # Initialize other directories + # Not sure if needed so commented for now. + # nc_dir = os.path.join(metrics_output_path, "netcdf") + # os.makedirs(nc_dir, exist_ok=True) + # if plots: + # plot_dir_maps = os.path.join(metrics_output_path, "plots", "maps") + # os.makedirs(plot_dir_maps, exist_ok=True) + + # Setting up model realization list + find_all_realizations, realizations = utilities.set_up_realizations(realization) + + # Only include reference data in loop if it exists + if reference_data_path is not None: + model_loop_list = ["Reference"] + model_list else: - list_of_runs = realizations - - metrics_dict["RESULTS"][model] = {} - - # Loop over realizations - for run in list_of_runs: - # Finding land/sea mask - sftlf_exists = True - if run == reference_data_set: - if reference_sftlf_template is not None and os.path.exists( - reference_sftlf_template - ): - sftlf_filename = reference_sftlf_template - else: - print("No reference sftlf file template provided.") - if not generate_sftlf: - print("Skipping reference data") - else: - # Set flag to generate sftlf after loading data - sftlf_exists = False - else: - try: - tags = { - "%(model)": model, - "%(model_version)": model, - "%(realization)": run, - } - sftlf_filename_list = utilities.replace_multi( - sftlf_filename_template, tags - ) - sftlf_filename = glob.glob(sftlf_filename_list)[0] - except (AttributeError, IndexError): - print("No sftlf file found for", model, run) - if not generate_sftlf: - print("Skipping realization", run) - continue - else: - # Set flag to generate sftlf after loading data - sftlf_exists = False - if sftlf_exists: - sftlf = xcdat.open_dataset(sftlf_filename, decode_times=False) - if use_region_mask: - print("\nCreating region mask for land/sea mask.") - sftlf = region_utilities.mask_region( - sftlf, region_name, coords=coords, shp_path=shp_path, column=col - ) - - if run == reference_data_set: - units_adjust = ObsUnitsAdjust + model_loop_list = model_list + + # Initialize output JSON structures + # FYI: if the analysis output JSON is changed, remember to update this function! + metrics_dict = compute_metrics.init_metrics_dict( + model_loop_list, + variable_list, + dec_mode, + drop_incomplete_djf, + annual_strict, + region_name, + ) + + ############## + # Run Analysis + ############## + + # Loop over models + for model in model_loop_list: + if model == "Reference": + list_of_runs = [str(reference_data_set)] + elif find_all_realizations: + tags = {"%(model)": model, "%(model_version)": model, "%(realization)": "*"} + test_data_full_path = os.path.join(test_data_path, filename_template) + test_data_full_path = utilities.replace_multi(test_data_full_path, tags) + ncfiles = glob.glob(test_data_full_path) + realizations = [] + for ncfile in ncfiles: + realizations.append(ncfile.split("/")[-1].split(".")[3]) + print("=================================") + print("model, runs:", model, realizations) + list_of_runs = realizations else: - units_adjust = ModUnitsAdjust + list_of_runs = realizations - # Initialize results dictionary for this mode/run - # Presumably we will be populating this will metrics as we go - metrics_dict["RESULTS"][model][run] = {} + metrics_dict["RESULTS"][model] = {} - # In extremes we are looping over different variables. I am really - # not sure what will be the best way to approach this, if we should loop - # over variables, take in multiple variables at once, etc. - for varname in variable_list: - # Populate the filename templates to get actual data path + # Loop over realizations + for run in list_of_runs: + # Finding land/sea mask + sftlf_exists = True if run == reference_data_set: - test_data_full_path = reference_data_path - start_year = osyear - end_year = oeyear - else: - tags = { - "%(variable)": varname, - "%(model)": model, - "%(model_version)": model, - "%(realization)": run, - } - test_data_full_path = os.path.join(test_data_path, filename_template) - test_data_full_path = utilities.replace_multi(test_data_full_path, tags) - start_year = msyear - end_year = meyear - yrs = [str(start_year), str(end_year)] # for output file names - test_data_full_path = glob.glob(test_data_full_path) - test_data_full_path.sort() - if len(test_data_full_path) == 0: - print("") - print("-----------------------") - print("Not found: model, run, variable:", model, run, varname) - continue + if reference_sftlf_template is not None and os.path.exists( + reference_sftlf_template + ): + sftlf_filename = reference_sftlf_template + else: + print("No reference sftlf file template provided.") + if not generate_sftlf: + print("Skipping reference data") + else: + # Set flag to generate sftlf after loading data + sftlf_exists = False else: - print("") - print("-----------------------") - print("model, run, variable:", model, run, varname) - print("test_data (model in this case) full_path:") - for t in test_data_full_path: - print(" ", t) - - # Load and prep data - # TODO: substitute PMP version of loading - ds = utilities.load_dataset(test_data_full_path) - - if not sftlf_exists and generate_sftlf: - print("Generating land sea mask.") - sft = create_land_sea_mask(ds) - sftlf = ds.copy(data=None) - sftlf["sftlf"] = sft + try: + tags = { + "%(model)": model, + "%(model_version)": model, + "%(realization)": run, + } + sftlf_filename_list = utilities.replace_multi( + sftlf_filename_template, tags + ) + sftlf_filename = glob.glob(sftlf_filename_list)[0] + except (AttributeError, IndexError): + print("No sftlf file found for", model, run) + if not generate_sftlf: + print("Skipping realization", run) + continue + else: + # Set flag to generate sftlf after loading data + sftlf_exists = False + if sftlf_exists: + sftlf = xcdat.open_dataset(sftlf_filename, decode_times=False) if use_region_mask: print("\nCreating region mask for land/sea mask.") sftlf = region_utilities.mask_region( sftlf, region_name, coords=coords, shp_path=shp_path, column=col ) - # Mask out Antarctica - sftlf["sftlf"] = sftlf["sftlf"].where(sftlf.lat > -60) + if run == reference_data_set: + units_adjust = ObsUnitsAdjust + else: + units_adjust = ModUnitsAdjust + + # Initialize results dictionary for this mode/run + # Presumably we will be populating this will metrics as we go + metrics_dict["RESULTS"][model][run] = {} + + # In extremes we are looping over different variables. I am really + # not sure what will be the best way to approach this, if we should loop + # over variables, take in multiple variables at once, etc. + for varname in variable_list: + # Populate the filename templates to get actual data path + if run == reference_data_set: + test_data_full_path = reference_data_path + start_year = osyear + end_year = oeyear + else: + tags = { + "%(variable)": varname, + "%(model)": model, + "%(model_version)": model, + "%(realization)": run, + } + test_data_full_path = os.path.join( + test_data_path, filename_template + ) + test_data_full_path = utilities.replace_multi( + test_data_full_path, tags + ) + start_year = msyear + end_year = meyear + yrs = [str(start_year), str(end_year)] # for output file names + test_data_full_path = glob.glob(test_data_full_path) + test_data_full_path.sort() + if len(test_data_full_path) == 0: + print("") + print("-----------------------") + print("Not found: model, run, variable:", model, run, varname) + continue + else: + print("") + print("-----------------------") + print("model, run, variable:", model, run, varname) + print("test_data (model in this case) full_path:") + for t in test_data_full_path: + print(" ", t) + + # Load and prep data + # TODO: substitute PMP version of loading + ds = utilities.load_dataset(test_data_full_path) + + if not sftlf_exists and generate_sftlf: + print("Generating land sea mask.") + sft = create_land_sea_mask(ds) + sftlf = ds.copy(data=None) + sftlf["sftlf"] = sft + if use_region_mask: + print("\nCreating region mask for land/sea mask.") + sftlf = region_utilities.mask_region( + sftlf, + region_name, + coords=coords, + shp_path=shp_path, + column=col, + ) + + # Mask out Antarctica + sftlf["sftlf"] = sftlf["sftlf"].where(sftlf.lat > -60) - if use_region_mask: - print("Creating dataset mask.") - ds = region_utilities.mask_region( - ds, region_name, coords=coords, shp_path=shp_path, column=col - ) + if use_region_mask: + print("Creating dataset mask.") + ds = region_utilities.mask_region( + ds, region_name, coords=coords, shp_path=shp_path, column=col + ) - # Get time slice if year parameters exist - if start_year is not None: - ds = utilities.slice_dataset(ds, start_year, end_year) - else: - # Get labels for start/end years from dataset - yrs = [str(int(ds.time.dt.year[0])), str(int(ds.time.dt.year[-1]))] - - # If any of the metrics use daily data we'll want to keep - # something like this calendar handling - if ds.time.encoding["calendar"] != "noleap" and exclude_leap: - ds = ds.convert_calendar("noleap") - - # ------------------------------- - # Metrics go here - # ------------------------------- - # Maybe start with the metrics from this paper: https://climatemodeling.science.energy.gov/sites/default/files/2023-11/Validation%20of%20LOCA2%20and%20STAR-ESDM%20Statistically%20Downscaled%20Products%20v2.pdf - # - # pr: annualmean_pr, seasonalmean_pr, pr_q50, pr_q99p9, annual pxx - # tasmax: annualmean_tasmax, seasonalmean_tasmax, annual txx, annual_tasmax_ge_95F, - # annual_tasmax_ge_100F, annual_tasmax_ge_105F, tasmax_q50, tasmax_q99p9 - # tasmin: annualmean_tasmin, annual_tasmin_le_32F, annual_tnn - # ETTCDI has some metrics for tas as well - - if varname == "tasmax": - # tasmax metrics - ( - Tmean, - Tmedian, - Tq99p9, - Tge95, - Tge100, - Tge105, - ) = compute_metrics.tasmax_indices( - ds, - sftlf, - units_adjust, - dec_mode, - drop_incomplete_djf, - annual_strict, - ) - - # Printing the arrays just to check that they exist for development purposes. - print(Tmean) - print(Tmedian) - print(Tq99p9) - print(Tge95) - print(Tge100) - print(Tge105) - elif varname == "tasmin": - # tasmin metrics - Tmean, Tmin, Tle32 = compute_metrics.tasmin_indices( - ds, - sftlf, - units_adjust, - dec_mode, - drop_incomplete_djf, - annual_strict, - ) - print(Tmean) - print(Tmin) - print(Tle32) - -# ------------------------------- -# Output JSON with metrics here -# ------------------------------- - -# Update and write metadata file -# Will want something similar to below code I've left commented -# try: -# with open(fname, "r") as f: -# tmp = json.load(f) -# meta.update_provenance("environment", tmp["provenance"]) -# except Exception: -# # Skip provenance if there's an issue -# print("Error: Could not get provenance from extremes json for output.json.") - -meta.update_provenance("modeldata", test_data_path) -if reference_data_path is not None: - meta.update_provenance("obsdata", reference_data_path) -meta.write() + # Get time slice if year parameters exist + if start_year is not None: + ds = utilities.slice_dataset(ds, start_year, end_year) + else: + # Get labels for start/end years from dataset + yrs = [str(int(ds.time.dt.year[0])), str(int(ds.time.dt.year[-1]))] + + # If any of the metrics use daily data we'll want to keep + # something like this calendar handling + if ds.time.encoding["calendar"] != "noleap" and exclude_leap: + ds = ds.convert_calendar("noleap") + + ds[varname] = compute_metrics.convert_units(ds[varname], ModUnitsAdjust) + # ------------------------------- + # Metrics go here + # ------------------------------- + # Maybe start with the metrics from this paper: https://climatemodeling.science.energy.gov/sites/default/files/2023-11/Validation%20of%20LOCA2%20and%20STAR-ESDM%20Statistically%20Downscaled%20Products%20v2.pdf + # + # pr: annualmean_pr, seasonalmean_pr, pr_q50, pr_q99p9, annual pxx + # tasmax: annualmean_tasmax, seasonalmean_tasmax, annual txx, annual_tasmax_ge_95F, + # annual_tasmax_ge_100F, annual_tasmax_ge_105F, tasmax_q50, tasmax_q99p9 + # tasmin: annualmean_tasmin, annual_tasmin_le_32F, annual_tnn + # ETTCDI has some metrics for tas as well + + if varname == "tasmax": + # tasmax metrics + index = compute_metrics.get_annual_txx( + ds, + sftlf, + units_adjust, + dec_mode, + drop_incomplete_djf, + annual_strict, + ) + # This returns Tmean + + stats_dict = {"annual_txx": index} + + elif varname == "tasmin": + # tasmin metrics + index = compute_metrics.get_annual_tnn( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + ) + stats_dict = {"annual_tnn": index} + # Compute mean metric: this needs to get moved to some function in compute_metrics + + if run not in metrics_dict["DIMENSIONS"]["realization"]: + metrics_dict["DIMENSIONS"]["realization"].append(run) + + # TODO: This needs to get wrapped up in a functions somewhere so it can + # be run separately for each index. We cannot have many grids for many + # different indices accumulating, basically want to compute the metrics + # and delete the xr.Dataset before moving on to the next one. + if model != "Reference": + # Get stats and update metrics dictionary + print("Generating metrics.") + result_dict = compute_metrics.metrics_json( + stats_dict, obs_dict={}, region=region_name, regrid=False + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + if run not in metrics_dict["DIMENSIONS"]["realization"]: + metrics_dict["DIMENSIONS"]["realization"].append(run) + + del stats_dict + del index + + # Output JSON with metrics for single model + if model != "Reference": + # Pull out metrics for just this model + # and write to JSON + metrics_tmp = metrics_dict.copy() + metrics_tmp["DIMENSIONS"]["model"] = model + metrics_tmp["DIMENSIONS"]["realization"] = list_of_runs + metrics_tmp["RESULTS"] = {model: metrics_dict["RESULTS"][model]} + metrics_path = "{0}_metrics.json".format(model) + utilities.write_to_json(metrics_output_path, metrics_path, metrics_tmp) + + meta.update_metrics( + model, + os.path.join(metrics_output_path, metrics_path), + model + " results", + "Seasonal metrics for single dataset", + ) + del metrics_tmp + + # Output single file with all models + model_write_list = model_loop_list.copy() + if "Reference" in model_write_list: + model_write_list.remove("Reference") + metrics_dict["DIMENSIONS"]["model"] = model_write_list + utilities.write_to_json( + metrics_output_path, "decision_relevant_metrics.json", metrics_dict + ) + fname = os.path.join(metrics_output_path, "decision_relevant_metrics.json") + meta.update_metrics( + "All", + fname, + "All results", + "Decision relevant climate metrics for all datasets", + ) + + # Update and write metadata file + try: + with open(fname, "r") as f: + tmp = json.load(f) + meta.update_provenance("environment", tmp["provenance"]) + except Exception: + # Skip provenance if there's an issue + print("Error: Could not get provenance from extremes json for output.json.") + + meta.update_provenance("modeldata", test_data_path) + if reference_data_path is not None: + meta.update_provenance("obsdata", reference_data_path) + meta.write() From 1ca5563ad4a8c35175591bf624fa0f6f9ceb7a25 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 20 Jun 2024 12:25:28 -0700 Subject: [PATCH 21/88] refactor --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 252 ++++++++++++++------- 1 file changed, 169 insertions(+), 83 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index c48d33ebd..6a8dcaf03 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -431,16 +431,9 @@ def convert_units(data, units_adjust): return data -def tasmax_indices( - ds, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict -): - # Returns annual daily mean of provided temperature dataset - # Temperature input can be "tasmax" or "tasmin". - - print("Generating temperature block extrema.") +def get_annual_txx(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("annual_txx") varname = "tasmax" - ds[varname] = convert_units(ds[varname], units_adjust) - TS = TimeSeriesData(ds, varname) S = SeasonalAverager( TS, @@ -451,45 +444,107 @@ def tasmax_indices( ) Tmean = xr.Dataset() - Tmedian = xr.Dataset() - Tq99p9 = xr.Dataset() - Tge95 = xr.Dataset() - Tge100 = xr.Dataset() - Tge105 = xr.Dataset() Tmean["ANN"] = S.annual_stats("mean") - Tmedian["ANN"] = S.annual_stats("median") - Tq99p9["ANN"] = S.annual_stats("q99p9") - Tge95["ANN"] = S.annual_stats("ge95") - Tge100["ANN"] = S.annual_stats("ge100") - Tge105["ANN"] = S.annual_stats("ge105") - for season in ["DJF", "MAM", "JJA", "SON"]: Tmean[season] = S.seasonal_stats(season, "mean") + Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) + return Tmean - Tge95.attrs["units"] = "%" - Tge100.attrs["units"] = "%" - Tge105.attrs["units"] = "%" - Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) +def get_tasmax_q50(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("tasmax_q50") + varname = "tasmax" + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + Tmedian = xr.Dataset() + Tmedian["ANN"] = S.annual_stats("median") Tmedian = update_nc_attrs(Tmedian, dec_mode, drop_incomplete_djf, annual_strict) + return Tmedian + + +def get_tasmax_q99p9(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("tasmax_q99p99") + varname = "tasmax" + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + Tq99p9 = xr.Dataset() + Tq99p9["ANN"] = S.annual_stats("q99p9") Tq99p9 = update_nc_attrs(Tq99p9, dec_mode, drop_incomplete_djf, annual_strict) + return Tq99p9 + + +def get_annual_tasmax_ge_95F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("annual_tasmax_ge_95F") + varname = "tasmax" + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + Tge95 = xr.Dataset() + Tge95["ANN"] = S.annual_stats("ge95") + Tge95.attrs["units"] = "%" Tge95 = update_nc_attrs(Tge95, dec_mode, drop_incomplete_djf, annual_strict) + return Tge95 + + +def get_annual_tasmax_ge_100F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("annual_tasmax_ge_100F") + varname = "tasmax" + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + Tge100 = xr.Dataset() + Tge100["ANN"] = S.annual_stats("ge95") + Tge100.attrs["units"] = "%" Tge100 = update_nc_attrs(Tge100, dec_mode, drop_incomplete_djf, annual_strict) - Tge105 = update_nc_attrs(Tge105, dec_mode, drop_incomplete_djf, annual_strict) + return Tge100 - return Tmean, Tmedian, Tq99p9, Tge95, Tge100, Tge105 +def get_annual_tasmax_ge_105F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("annual_tasmax_ge_105F") + varname = "tasmax" + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + Tge105 = xr.Dataset() + Tge105["ANN"] = S.annual_stats("ge95") + Tge105.attrs["units"] = "%" + Tge105 = update_nc_attrs(Tge105, dec_mode, drop_incomplete_djf, annual_strict) + return Tge105 -def tasmin_indices( - ds, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict -): - # Returns annual daily mean of provided temperature dataset - # Temperature input can be "tasmax" or "tasmin". - print("Generating temperature block extrema.") +def get_annual_tnn(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("annual_tnn") varname = "tasmin" - ds[varname] = convert_units(ds[varname], units_adjust) - TS = TimeSeriesData(ds, varname) S = SeasonalAverager( TS, @@ -500,22 +555,49 @@ def tasmin_indices( ) Tmean = xr.Dataset() - Tmin = xr.Dataset() - Tle32 = xr.Dataset() Tmean["ANN"] = S.annual_stats("mean") - Tmin["ANN"] = S.annual_stats("min") - Tle32["ANN"] = S.annual_stats("le32") - for season in ["DJF", "MAM", "JJA", "SON"]: Tmean[season] = S.seasonal_stats(season, "mean") + Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) + return Tmean - Tle32.attrs["units"] = "%" - Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) +def get_annualmean_tasmin(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("annualmean_tasmin") + varname = "tasmin" + + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + Tmin = xr.Dataset() + Tmin["ANN"] = S.annual_stats("min") Tmin = update_nc_attrs(Tmin, dec_mode, drop_incomplete_djf, annual_strict) - Tle32 = update_nc_attrs(Tle32, dec_mode, drop_incomplete_djf, annual_strict) + return Tmin - return Tmean, Tmin, Tle32 + +def get_annual_tasmin_le_32F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): + print("annual_tasmin_le_32F") + varname = "tasmin" + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + Tle32 = xr.Dataset() + Tle32["ANN"] = S.annual_stats("le32") + Tle32.attrs["units"] = "%" + Tle32 = update_nc_attrs(Tle32, dec_mode, drop_incomplete_djf, annual_strict) + return Tle32 def precipitation_indices( @@ -682,46 +764,50 @@ def metrics_json(data_dict, obs_dict={}, region="land", regrid=True): ds_m = data_dict[m] for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: - # Global mean over land - met_dict[m][region]["mean"][season] = mean_xy(ds_m, season) - a = ds_m.temporal.average(season) - std_xy = pmp_stats.std_xy(a, season) - met_dict[m][region]["std_xy"][season] = std_xy - - if len(obs_dict) > 0 and not obs_dict[m].equals(ds_m): - # Regrid obs to model grid - if regrid: - target = xc.create_grid(ds_m.lat, ds_m.lon) - target = target.bounds.add_missing_bounds(["X", "Y"]) - obs_m = obs_dict[m].regridder.horizontal( - season, target, tool="regrid2" - ) - else: - obs_m = obs_dict[m] - shp1 = (len(ds_m[season].lat), len(ds_m[season].lon)) - shp2 = (len(obs_m[season].lat), len(obs_m[season].lon)) - assert ( - shp1 == shp2 - ), "Model and Reference data dimensions 'lat' and 'lon' must match." - - # Get xy stats for temporal average + if season in ds_m: + # Global mean over land + met_dict[m][region]["mean"][season] = mean_xy(ds_m, season) a = ds_m.temporal.average(season) - b = obs_m.temporal.average(season) - weights = ds_m.spatial.get_weights(axis=["X", "Y"]) - rms_xy = pmp_stats.rms_xy(a, b, var=season, weights=weights) - meanabs_xy = pmp_stats.meanabs_xy(a, b, var=season, weights=weights) - bias_xy = pmp_stats.bias_xy(a, b, var=season, weights=weights) - cor_xy = pmp_stats.cor_xy(a, b, var=season, weights=weights) - rmsc_xy = pmp_stats.rmsc_xy(a, b, var=season, weights=weights) - std_obs_xy = pmp_stats.std_xy(b, season) - pct_dif = percent_difference(b, bias_xy, season, weights) - - met_dict[m][region]["pct_dif"][season] = pct_dif - met_dict[m][region]["rms_xy"][season] = rms_xy - met_dict[m][region]["mae_xy"][season] = meanabs_xy - met_dict[m][region]["bias_xy"][season] = bias_xy - met_dict[m][region]["cor_xy"][season] = cor_xy - met_dict[m][region]["rmsc_xy"][season] = rmsc_xy - met_dict[m][region]["std-obs_xy"][season] = std_obs_xy + std_xy = pmp_stats.std_xy(a, season) + met_dict[m][region]["std_xy"][season] = std_xy + + if len(obs_dict) > 0 and not obs_dict[m].equals(ds_m): + # Regrid obs to model grid + if regrid: + target = xc.create_grid(ds_m.lat, ds_m.lon) + target = target.bounds.add_missing_bounds(["X", "Y"]) + obs_m = obs_dict[m].regridder.horizontal( + season, target, tool="regrid2" + ) + else: + obs_m = obs_dict[m] + shp1 = (len(ds_m[season].lat), len(ds_m[season].lon)) + shp2 = (len(obs_m[season].lat), len(obs_m[season].lon)) + assert ( + shp1 == shp2 + ), "Model and Reference data dimensions 'lat' and 'lon' must match." + + # Get xy stats for temporal average + a = ds_m.temporal.average(season) + b = obs_m.temporal.average(season) + weights = ds_m.spatial.get_weights(axis=["X", "Y"]) + rms_xy = pmp_stats.rms_xy(a, b, var=season, weights=weights) + meanabs_xy = pmp_stats.meanabs_xy(a, b, var=season, weights=weights) + bias_xy = pmp_stats.bias_xy(a, b, var=season, weights=weights) + cor_xy = pmp_stats.cor_xy(a, b, var=season, weights=weights) + rmsc_xy = pmp_stats.rmsc_xy(a, b, var=season, weights=weights) + std_obs_xy = pmp_stats.std_xy(b, season) + pct_dif = percent_difference(b, bias_xy, season, weights) + + met_dict[m][region]["pct_dif"][season] = pct_dif + met_dict[m][region]["rms_xy"][season] = rms_xy + met_dict[m][region]["mae_xy"][season] = meanabs_xy + met_dict[m][region]["bias_xy"][season] = bias_xy + met_dict[m][region]["cor_xy"][season] = cor_xy + met_dict[m][region]["rmsc_xy"][season] = rmsc_xy + met_dict[m][region]["std-obs_xy"][season] = std_obs_xy + else: + for item in met_dict[m][region]: + met_dict[m][region][item].pop(season) return met_dict From f0e9830de99b783f91acd4de22d3f00ec6341eda Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 20 Jun 2024 16:05:34 -0700 Subject: [PATCH 22/88] refactor p2 --- pcmdi_metrics/drcdm/drcdm_driver.py | 78 +++++++++++++++++------------ 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 8dda3c146..98410b772 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -56,6 +56,8 @@ col = parameter.attribute region_name = parameter.region_name coords = parameter.coords + nc_out = parameter.netcdf + plots = parameter.plot # Check the region masking parameters, if provided use_region_mask, region_name, coords = region_utilities.check_region_params( @@ -85,12 +87,12 @@ meta = metadata.MetadataFile(metrics_output_path) # Initialize other directories - # Not sure if needed so commented for now. - # nc_dir = os.path.join(metrics_output_path, "netcdf") - # os.makedirs(nc_dir, exist_ok=True) - # if plots: - # plot_dir_maps = os.path.join(metrics_output_path, "plots", "maps") - # os.makedirs(plot_dir_maps, exist_ok=True) + if nc_out: + nc_dir = os.path.join(metrics_output_path, "netcdf") + os.makedirs(nc_dir, exist_ok=True) + if plots: + fig_dir = os.path.join(metrics_output_path, "plots") + os.makedirs(fig_dir, exist_ok=True) # Setting up model realization list find_all_realizations, realizations = utilities.set_up_realizations(realization) @@ -229,7 +231,6 @@ print(" ", t) # Load and prep data - # TODO: substitute PMP version of loading ds = utilities.load_dataset(test_data_full_path) if not sftlf_exists and generate_sftlf: @@ -280,51 +281,64 @@ # tasmin: annualmean_tasmin, annual_tasmin_le_32F, annual_tnn # ETTCDI has some metrics for tas as well + if nc_out: + # $index will be replaced with index name in metrics function + nc_base = os.path.join(nc_dir, "_".join([model, run, "$index.nc"])) + else: + nc_base = None + if plots: + fig_base = os.path.join(nc_dir, "_".join([model, run, "$index.nc"])) + if varname == "tasmax": - # tasmax metrics - index = compute_metrics.get_annual_txx( + # Example using get_annual_txx + # Need to work all temperature metrics this way + result_dict = compute_metrics.get_mean_tasmax( ds, sftlf, - units_adjust, dec_mode, drop_incomplete_djf, annual_strict, + fig_base, + nc_base, ) - # This returns Tmean - - stats_dict = {"annual_txx": index} + metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_tasmax_q99p9( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) elif varname == "tasmin": # tasmin metrics - index = compute_metrics.get_annual_tnn( + result_dict = compute_metrics.get_annual_tnn( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, + fig_base, + nc_base, ) - stats_dict = {"annual_tnn": index} - # Compute mean metric: this needs to get moved to some function in compute_metrics - - if run not in metrics_dict["DIMENSIONS"]["realization"]: - metrics_dict["DIMENSIONS"]["realization"].append(run) + metrics_dict["RESULTS"][model][run].update(result_dict) - # TODO: This needs to get wrapped up in a functions somewhere so it can - # be run separately for each index. We cannot have many grids for many - # different indices accumulating, basically want to compute the metrics - # and delete the xr.Dataset before moving on to the next one. - if model != "Reference": - # Get stats and update metrics dictionary - print("Generating metrics.") - result_dict = compute_metrics.metrics_json( - stats_dict, obs_dict={}, region=region_name, regrid=False + result_dict = compute_metrics.get_annual_tasmin_le_32F( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - if run not in metrics_dict["DIMENSIONS"]["realization"]: - metrics_dict["DIMENSIONS"]["realization"].append(run) - del stats_dict - del index + if run not in metrics_dict["DIMENSIONS"]["realization"]: + metrics_dict["DIMENSIONS"]["realization"].append(run) # Output JSON with metrics for single model if model != "Reference": From 2d2ec80db6a2798e91dc8f0d5667266300c09782 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 20 Jun 2024 16:07:41 -0700 Subject: [PATCH 23/88] refactor p2 --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 334 +++++++++++++++++---- 1 file changed, 269 insertions(+), 65 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 6a8dcaf03..3e7283e5f 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -130,11 +130,12 @@ def annual_stats(self, stat, pentad=False): .groupby("time.year") .median(dim="time") ) - elif stat == "q99p9": + elif stat.startswith("q"): + num = float(stat.replace("q", "").replace("p", ".")) / 100.0 ds_ann = ( ds.sel(time=date_range, method="nearest") .groupby("time.year") - .quantile(0.999, dim="time") + .quantile(num, dim="time") ) elif stat.startswith("ge"): num = int(stat.replace("ge", "")) @@ -171,8 +172,9 @@ def annual_stats(self, stat, pentad=False): ds_ann = ds.groupby("time.year").mean(dim="time") elif stat == "median": ds_ann = ds.groupby("time.year").median(dim="time") - elif stat == "q99p9": - ds_ann = ds.groupby("time.year").quantile(0.999, dim="time") + elif stat.startswith("q"): + num = float(stat.replace("q", "").replace("p", ".")) / 100.0 + ds_ann = ds.groupby("time.year").quantile(num, dim="time") elif stat.startswith("ge"): num = int(stat.replace("ge", "")) ds_ann = ( @@ -412,7 +414,14 @@ def convert_units(data, units_adjust): # For example, (True, "multiply", 86400., "mm/day") # If flag is False, data is returned unaltered. if bool(units_adjust[0]): - op_dict = {"add": "+", "subtract": "-", "multiply": "*", "divide": "/"} + op_dict = { + "add": "+", + "subtract": "-", + "multiply": "*", + "divide": "/", + "CtoF": ")*(9/5)+32", + "KtoF": "-273)*(9/5)+32", + } if str(units_adjust[1]) not in op_dict: print( "Error in units conversion. Operation must be add, subtract, multiply, or divide." @@ -421,7 +430,11 @@ def convert_units(data, units_adjust): return data op = op_dict[str(units_adjust[1])] val = float(units_adjust[2]) - operation = "data {0} {1}".format(op, val) + if units_adjust[1] in ["KtoF", "CtoF"]: + # more complex equation for fahrenheit conversion + operation = "(data{0}".format(op) + else: + operation = "data {0} {1}".format(op, val) data = eval(operation) data.attrs["units"] = str(units_adjust[3]) else: @@ -431,8 +444,12 @@ def convert_units(data, units_adjust): return data -def get_annual_txx(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("annual_txx") +def get_mean_tasmax( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual and seasonal mean maximum daily temperature. + index = "mean_tasmax" + print("Metric:", index) varname = "tasmax" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -442,17 +459,65 @@ def get_annual_txx(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tmean = xr.Dataset() - Tmean["ANN"] = S.annual_stats("mean") + Tmean["ANN"] = S.annual_stats("max") for season in ["DJF", "MAM", "JJA", "SON"]: Tmean[season] = S.seasonal_stats(season, "mean") Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) - return Tmean + # Compute statistics + result_dict = metrics_json({index: Tmean}, obs_dict={}, region="land", regrid=False) + + if fig_file is not None: + # Figures: + # Map of time mean value of Tmean + # + + # can replace the $index substring with any other string + # for specific file name. fig_file already contains the + # rest of the file path and the model and run names. + fig_file1 = fig_file.replace("$index", index) + print(fig_file1) + + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tmean.to_netcdf(nc_file, "w") -def get_tasmax_q50(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("tasmax_q50") + del Tmean + return result_dict + + +def get_annual_txx( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual max of maximum daily temperature. + index = "annual_txx" + print(index) + varname = "tasmax" + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + Tmax = xr.Dataset() + Tmax["ANN"] = S.annual_stats("max") + Tmax = update_nc_attrs(Tmax, dec_mode, drop_incomplete_djf, annual_strict) + + # Compute statistics + result_dict = metrics_json({index: Tmax}, obs_dict={}, region="land", regrid=False) + + del Tmax + return result_dict + + +def get_tasmax_q50( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual median maximum daily temperature + index = "tasmax_q50" varname = "tasmax" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -465,11 +530,22 @@ def get_tasmax_q50(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): Tmedian = xr.Dataset() Tmedian["ANN"] = S.annual_stats("median") Tmedian = update_nc_attrs(Tmedian, dec_mode, drop_incomplete_djf, annual_strict) - return Tmedian + + # Compute statistics + result_dict = metrics_json( + {index: Tmedian}, obs_dict={}, region="land", regrid=False + ) + + del Tmedian + return result_dict -def get_tasmax_q99p9(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("tasmax_q99p99") +def get_tasmax_q99p9( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual 99.9 percentile maximum daily temperature. + index = "tasmax_q99p9" + print("Metric:", index) varname = "tasmax" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -482,11 +558,22 @@ def get_tasmax_q99p9(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): Tq99p9 = xr.Dataset() Tq99p9["ANN"] = S.annual_stats("q99p9") Tq99p9 = update_nc_attrs(Tq99p9, dec_mode, drop_incomplete_djf, annual_strict) - return Tq99p9 + # Compute statistics + result_dict = metrics_json( + {index: Tq99p9}, obs_dict={}, region="land", regrid=False + ) + + del Tq99p9 + return result_dict -def get_annual_tasmax_ge_95F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("annual_tasmax_ge_95F") + +def get_annual_tasmax_ge_95F( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual fraction of days with max temperature greater than or equal to 95F + index = "annual_tasmax_ge_95F" + print("Metric:", index) varname = "tasmax" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -496,16 +583,24 @@ def get_annual_tasmax_ge_95F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_st drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tge95 = xr.Dataset() Tge95["ANN"] = S.annual_stats("ge95") Tge95.attrs["units"] = "%" Tge95 = update_nc_attrs(Tge95, dec_mode, drop_incomplete_djf, annual_strict) - return Tge95 + # Compute statistics + result_dict = metrics_json({index: Tge95}, obs_dict={}, region="land", regrid=False) + + del Tge95 + return result_dict -def get_annual_tasmax_ge_100F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("annual_tasmax_ge_100F") + +def get_annual_tasmax_ge_100F( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual fraction of days with max temperature greater than or equal to 100F + index = "annual_tasmax_ge_100F" + print("Metric:", index) varname = "tasmax" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -515,16 +610,26 @@ def get_annual_tasmax_ge_100F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_s drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tge100 = xr.Dataset() Tge100["ANN"] = S.annual_stats("ge95") Tge100.attrs["units"] = "%" Tge100 = update_nc_attrs(Tge100, dec_mode, drop_incomplete_djf, annual_strict) - return Tge100 + + # Compute statistics + result_dict = metrics_json( + {index: Tge100}, obs_dict={}, region="land", regrid=False + ) + + del Tge100 + return result_dict -def get_annual_tasmax_ge_105F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("annual_tasmax_ge_105F") +def get_annual_tasmax_ge_105F( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual fraction of days with max temperature greater than or equal to 105F + index = "annual_tasmax_ge_105F" + print("Metric:", index) varname = "tasmax" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -534,16 +639,28 @@ def get_annual_tasmax_ge_105F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_s drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tge105 = xr.Dataset() Tge105["ANN"] = S.annual_stats("ge95") Tge105.attrs["units"] = "%" Tge105 = update_nc_attrs(Tge105, dec_mode, drop_incomplete_djf, annual_strict) - return Tge105 + + # Do plots or saving files here + + # Compute statistics + result_dict = metrics_json( + {index: Tge105}, obs_dict={}, region="land", regrid=False + ) + + del Tge105 + return result_dict -def get_annual_tnn(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("annual_tnn") +def get_annual_tnn( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual minimum daily minimum temperature. + index = "annual_tnn" + print("Metric:", index) varname = "tasmin" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -553,19 +670,24 @@ def get_annual_tnn(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tmean = xr.Dataset() Tmean["ANN"] = S.annual_stats("mean") - for season in ["DJF", "MAM", "JJA", "SON"]: - Tmean[season] = S.seasonal_stats(season, "mean") Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) - return Tmean + # Compute statistics + result_dict = metrics_json({index: Tmean}, obs_dict={}, region="land", regrid=False) + + del Tmean + return result_dict -def get_annualmean_tasmin(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("annualmean_tasmin") - varname = "tasmin" +def get_annualmean_tasmin( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual mean daily minimum temperature. + index = "annualmean_tasmin" + print("Metric:", index) + varname = "tasmin" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( TS, @@ -574,15 +696,23 @@ def get_annualmean_tasmin(ds, sftlf, dec_mode, drop_incomplete_djf, annual_stric drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tmin = xr.Dataset() Tmin["ANN"] = S.annual_stats("min") Tmin = update_nc_attrs(Tmin, dec_mode, drop_incomplete_djf, annual_strict) - return Tmin + # Compute statistics + result_dict = metrics_json({index: Tmin}, obs_dict={}, region="land", regrid=False) + + del Tmin + return result_dict -def get_annual_tasmin_le_32F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict): - print("annual_tasmin_le_32F") + +def get_annual_tasmin_le_32F( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual percentage of days with daily minimum temperature less than or + # equal to 32F. + index = "annual_tasmin_le_32F" varname = "tasmin" TS = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -592,20 +722,106 @@ def get_annual_tasmin_le_32F(ds, sftlf, dec_mode, drop_incomplete_djf, annual_st drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tle32 = xr.Dataset() Tle32["ANN"] = S.annual_stats("le32") Tle32.attrs["units"] = "%" Tle32 = update_nc_attrs(Tle32, dec_mode, drop_incomplete_djf, annual_strict) - return Tle32 + + # Compute statistics + result_dict = metrics_json({index: Tle32}, obs_dict={}, region="land", regrid=False) + + del Tle32 + return result_dict + + +# TODO: Fill out precipitation metrics +def get_annualmean_pr( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + varname = "pr" + PR = TimeSeriesData(ds, "pr") + S = SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + print(varname, S) + return + + +def get_seasonalmean_pr( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + varname = "pr" + PR = TimeSeriesData(ds, "pr") + S = SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + print(varname, S) + return + + +def get_pr_q50( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + varname = "pr" + PR = TimeSeriesData(ds, "pr") + S = SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + print(varname, S) + return + + +def get_pr_q99p9( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + varname = "pr" + PR = TimeSeriesData(ds, "pr") + S = SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + print(varname, S) + return + + +def get_annual_pxx( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + varname = "pr" + PR = TimeSeriesData(ds, "pr") + S = SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + print(varname, S) + return def precipitation_indices( ds, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict ): - # Returns annual Rx1day and Rx5day of provided precipitation dataset. - # Precipitation variable must be called "pr". - # Input data expected to have units of kg/m2/s + # TODO: the precipitation metrics need to be broken out the way the + # temperature metrics were. + + # annualmean_pr, seasonalmean_pr, pr_q50, pr_q99p9, annual pxx print("Generating precipitation block extrema.") @@ -704,26 +920,14 @@ def init_metrics_dict( # Only include the definitions for the indices in this particular analysis. for v in var_list: if v == "tasmax": - metrics["DIMENSIONS"]["index"].update( - {"TXx": "Maximum value of daily maximum temperature"} - ) - metrics["DIMENSIONS"]["index"].update( - {"TXn": "Minimum value of daily maximum temperature"} - ) + metrics["DIMENSIONS"]["index"].update({"": ""}) + metrics["DIMENSIONS"]["index"].update({"": ""}) if v == "tasmin": - metrics["DIMENSIONS"]["index"].update( - {"TNx": "Maximum value of daily minimum temperature"} - ) - metrics["DIMENSIONS"]["index"].update( - {"TNn": "Minimum value of daily minimum temperature"} - ) + metrics["DIMENSIONS"]["index"].update({"": ""}) + metrics["DIMENSIONS"]["index"].update({"": ""}) if v in ["pr", "PRECT", "precip"]: - metrics["DIMENSIONS"]["index"].update( - {"Rx5day": "Maximum consecutive 5-day mean precipitation, mm/day"} - ) - metrics["DIMENSIONS"]["index"].update( - {"Rx1day": "Maximum daily precipitation, mm/day"} - ) + metrics["DIMENSIONS"]["index"].update({"": ""}) + metrics["DIMENSIONS"]["index"].update({"": ""}) return metrics From 4ef21086289ab663aa0a716482099d86b110b2ca Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 20 Jun 2024 16:07:56 -0700 Subject: [PATCH 24/88] add params --- pcmdi_metrics/drcdm/lib/create_drcdm_parser.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py b/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py index 8992bb128..4ea851140 100644 --- a/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py +++ b/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py @@ -269,6 +269,8 @@ def create_extremes_parser(): help="For unit adjust for OBS dataset. For example:\n" "- (True, 'divide', 100.0, 'hPa') # Pa to hPa\n" "- (True, 'subtract', 273.15, 'C') # degK to degC\n" + "- (True, 'KtoF', '', 'F') # degK to degF, leave position 3 empty\n" + "- (True, 'CtoF', '', 'F') # degC to degF, leave position 3 empty\n" "- (False, 0, 0, None) # No adjustment (default)", ) parser.add_argument( @@ -278,6 +280,8 @@ def create_extremes_parser(): help="For unit adjust for model dataset. For example:\n" "- (True, 'divide', 100.0, 'hPa') # Pa to hPa\n" "- (True, 'subtract', 273.15, 'C') # degK to degC\n" + "- (True, 'KtoF', 0, 'F') # degK to degF, leave position 3 as 0\n" + "- (True, 'CtoF', 0, 'F') # degC to degF, leave position 3 as 0\n" "- (False, 0, 0, None) # No adjustment (default)", ) @@ -288,4 +292,18 @@ def create_extremes_parser(): help="Return period, in years, for obtaining return values.", ) + parser.add_argument( + "--netcdf", + action="store_true", + default=False, + help="Use to save netcdf intermediate results", + ) + + parser.add_argument( + "--plot", + type=bool, + default=False, + help="Option for generate individual plots for models: True (default) / False", + ) + return parser From 3c45c22bd2c08ea67333d738009ab4edd2593294 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 20 Jun 2024 16:53:23 -0700 Subject: [PATCH 25/88] fix annual tasmax mean --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 3e7283e5f..351f06dba 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -460,7 +460,7 @@ def get_mean_tasmax( annual_strict=annual_strict, ) Tmean = xr.Dataset() - Tmean["ANN"] = S.annual_stats("max") + Tmean["ANN"] = S.annual_stats("mean") for season in ["DJF", "MAM", "JJA", "SON"]: Tmean[season] = S.seasonal_stats(season, "mean") Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) From 01b8bb4263322bcaac6d99ae307cf1f4978a70ce Mon Sep 17 00:00:00 2001 From: James Seth Goodnight Date: Fri, 21 Jun 2024 16:05:56 -0700 Subject: [PATCH 26/88] added precipitation metrics --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 105 +++++++++++++++++---- 1 file changed, 85 insertions(+), 20 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 351f06dba..0ca503915 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -291,7 +291,9 @@ def seasonal_stats(self, season, stat, pentad=False): if stat == "max": ds_stat = ( - ds.sel(time=date_range, method="nearest") + ds.sel( + time=date_range, method="nearest" + ) # could also do ds.sel(time = slice(*(begin_cftime, end_cftime))) .groupby("time.year") .max(dim="time") ) @@ -738,8 +740,13 @@ def get_annual_tasmin_le_32F( def get_annualmean_pr( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): + index = "annualmean_pr" varname = "pr" - PR = TimeSeriesData(ds, "pr") + print("Metric:", index) + + # Get annual mean precipitation + + PR = TimeSeriesData(ds, varname) S = SeasonalAverager( PR, sftlf, @@ -747,15 +754,29 @@ def get_annualmean_pr( drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - print(varname, S) - return + PRmean = xr.Dataset() + PRmean["ANN"] = S.annual_stats("mean") + + PRmean = update_nc_attrs(PRmean, dec_mode, drop_incomplete_djf, annual_strict) + + # compute statistics + result_dict = metrics_json( + {index: PRmean}, obs_dict={}, region="land", regrid=False + ) + + del PRmean + return result_dict def get_seasonalmean_pr( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): + index = "seasonalmean_pr" varname = "pr" - PR = TimeSeriesData(ds, "pr") + print("Metric", index) + + # Get seasonal mean precipitation + PR = TimeSeriesData(ds, varname) S = SeasonalAverager( PR, sftlf, @@ -763,15 +784,28 @@ def get_seasonalmean_pr( drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - print(varname, S) - return + PRmean = xr.Dataset() + for season in ["DJF", "MAM", "JJA", "SON"]: + PRmean[season] = S.seasonal_stats(season, "mean") + PRmean = update_nc_attrs(PRmean, dec_mode, drop_incomplete_djf, annual_strict) + + result_dict = metrics_json( + {index: PRmean}, obs_dict={}, region="land", regrid=False + ) + + del PRmean + return result_dict def get_pr_q50( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): + index = "pr_q50" varname = "pr" - PR = TimeSeriesData(ds, "pr") + print("Metric:", index) + + # Get median (q50) precipitation + PR = TimeSeriesData(ds, varname) S = SeasonalAverager( PR, sftlf, @@ -779,15 +813,23 @@ def get_pr_q50( drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - print(varname, S) - return + PRq50 = xr.Dataset() + PRq50["ANN"] = S.annual_stats("median") -def get_pr_q99p9( - ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None -): + result_dict = metrics_json({index: PRq50}, obs_dict={}, region="land", regrid=False) + + del PRq50 + return result_dict + + +def get_pr_q99p9(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, nc_file=None): + index = "pr_q99p9" varname = "pr" - PR = TimeSeriesData(ds, "pr") + print("Metric:", index) + + # Get 99.9th percentile daily precipitation + PR = TimeSeriesData(ds, varname) S = SeasonalAverager( PR, sftlf, @@ -795,15 +837,32 @@ def get_pr_q99p9( drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - print(varname, S) - return + PRq99p9 = xr.Dataset() + PRq99p9["ANN"] = S.annual_stats("q99p9") + + result_dict = metrics_json( + {index: PRq99p9}, obs_dict={}, region="land", regrid=False + ) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + PRq99p9.to_netcdf(nc_file, "w") + + del PRq99p9 + return result_dict def get_annual_pxx( - ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, ): varname = "pr" - PR = TimeSeriesData(ds, "pr") + index = "annual_pxx" + + # Get annual max precipitation + PR = TimeSeriesData(ds, varname) S = SeasonalAverager( PR, sftlf, @@ -811,8 +870,14 @@ def get_annual_pxx( drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - print(varname, S) - return + + Pmax = xr.Dataset() + Pmax["ANN"] = S.annual_stats("max") + + result_dict = metrics_json({index: Pmax}, obs_dict={}, region="land", regrid=False) + + del Pmax + return result_dict def precipitation_indices( From a52a529c2e1e3395a2a30a267c7528c86ecc7564 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 28 Jun 2024 16:17:43 -0700 Subject: [PATCH 27/88] updates for runtime --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 0ca503915..77e4247e9 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -816,6 +816,7 @@ def get_pr_q50( PRq50 = xr.Dataset() PRq50["ANN"] = S.annual_stats("median") + PRq50 = update_nc_attrs(PRq50, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json({index: PRq50}, obs_dict={}, region="land", regrid=False) @@ -823,7 +824,9 @@ def get_pr_q50( return result_dict -def get_pr_q99p9(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, nc_file=None): +def get_pr_q99p9( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): index = "pr_q99p9" varname = "pr" print("Metric:", index) @@ -839,10 +842,11 @@ def get_pr_q99p9(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, nc_fil ) PRq99p9 = xr.Dataset() PRq99p9["ANN"] = S.annual_stats("q99p9") - + PRq99p9 = update_nc_attrs(PRq99p9, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: PRq99p9}, obs_dict={}, region="land", regrid=False ) + if nc_file is not None: nc_file = nc_file.replace("$index", index) PRq99p9.to_netcdf(nc_file, "w") @@ -852,11 +856,7 @@ def get_pr_q99p9(ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, nc_fil def get_annual_pxx( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): varname = "pr" index = "annual_pxx" @@ -873,6 +873,7 @@ def get_annual_pxx( Pmax = xr.Dataset() Pmax["ANN"] = S.annual_stats("max") + Pmax = update_nc_attrs(Pmax, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json({index: Pmax}, obs_dict={}, region="land", regrid=False) From 3c867692c39f5dc74fee82282dfd1fb62d29ffdd Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 28 Jun 2024 16:17:59 -0700 Subject: [PATCH 28/88] add all metrics calls --- pcmdi_metrics/drcdm/drcdm_driver.py | 120 +++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 98410b772..4844c5889 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -288,6 +288,8 @@ nc_base = None if plots: fig_base = os.path.join(nc_dir, "_".join([model, run, "$index.nc"])) + else: + fig_base = None if varname == "tasmax": # Example using get_annual_txx @@ -312,9 +314,58 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_tasmax_q50( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_annual_txx( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_annual_tasmax_ge_95F( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_annual_tasmax_ge_100F( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_annual_tasmax_ge_105F( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) elif varname == "tasmin": - # tasmin metrics result_dict = compute_metrics.get_annual_tnn( ds, sftlf, @@ -325,7 +376,6 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_annual_tasmin_le_32F( ds, sftlf, @@ -337,6 +387,72 @@ ) metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_annualmean_tasmin( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + elif varname == "pr": + # Annual mean precipitation + result_dict = compute_metrics.get_annualmean_pr( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + # Seasonal mean precipitation + result_dict = compute_metrics.get_seasonalmean_pr( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + # Median + result_dict = compute_metrics.get_pr_q50( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + # 99.9 percentile + result_dict = compute_metrics.get_pr_q99p9( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + # Max daily precip + result_dict = compute_metrics.get_annual_pxx( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) if run not in metrics_dict["DIMENSIONS"]["realization"]: metrics_dict["DIMENSIONS"]["realization"].append(run) From 18c455768cc87e7b72b07719ef5b6a287560a055 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 2 Jul 2024 11:27:17 -0700 Subject: [PATCH 29/88] get running --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 77e4247e9..778b27bb7 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -135,7 +135,7 @@ def annual_stats(self, stat, pentad=False): ds_ann = ( ds.sel(time=date_range, method="nearest") .groupby("time.year") - .quantile(num, dim="time") + .quantile(num, dim="time", skipna=True) ) elif stat.startswith("ge"): num = int(stat.replace("ge", "")) @@ -174,7 +174,7 @@ def annual_stats(self, stat, pentad=False): ds_ann = ds.groupby("time.year").median(dim="time") elif stat.startswith("q"): num = float(stat.replace("q", "").replace("p", ".")) / 100.0 - ds_ann = ds.groupby("time.year").quantile(num, dim="time") + ds_ann = ds.groupby("time.year").quantile(num, dim="time", skipna=True) elif stat.startswith("ge"): num = int(stat.replace("ge", "")) ds_ann = ( From ec83cb6c85b85b6b7178bc3c3eebafd14dd73559 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 12 Jul 2024 13:17:29 -0700 Subject: [PATCH 30/88] add to nc --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 94 ++++++++++++---------- 1 file changed, 52 insertions(+), 42 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 778b27bb7..c6bcc0db1 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -511,6 +511,10 @@ def get_annual_txx( # Compute statistics result_dict = metrics_json({index: Tmax}, obs_dict={}, region="land", regrid=False) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tmax.to_netcdf(nc_file, "w") + del Tmax return result_dict @@ -538,6 +542,10 @@ def get_tasmax_q50( {index: Tmedian}, obs_dict={}, region="land", regrid=False ) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tmedian.to_netcdf(nc_file, "w") + del Tmedian return result_dict @@ -566,6 +574,10 @@ def get_tasmax_q99p9( {index: Tq99p9}, obs_dict={}, region="land", regrid=False ) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tq99p9.to_netcdf(nc_file, "w") + del Tq99p9 return result_dict @@ -593,6 +605,10 @@ def get_annual_tasmax_ge_95F( # Compute statistics result_dict = metrics_json({index: Tge95}, obs_dict={}, region="land", regrid=False) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tge95.to_netcdf(nc_file, "w") + del Tge95 return result_dict @@ -622,6 +638,10 @@ def get_annual_tasmax_ge_100F( {index: Tge100}, obs_dict={}, region="land", regrid=False ) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tge100.to_netcdf(nc_file, "w") + del Tge100 return result_dict @@ -653,6 +673,10 @@ def get_annual_tasmax_ge_105F( {index: Tge105}, obs_dict={}, region="land", regrid=False ) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tge105.to_netcdf(nc_file, "w") + del Tge105 return result_dict @@ -679,6 +703,10 @@ def get_annual_tnn( # Compute statistics result_dict = metrics_json({index: Tmean}, obs_dict={}, region="land", regrid=False) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tmean.to_netcdf(nc_file, "w") + del Tmean return result_dict @@ -705,6 +733,10 @@ def get_annualmean_tasmin( # Compute statistics result_dict = metrics_json({index: Tmin}, obs_dict={}, region="land", regrid=False) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tmin.to_netcdf(nc_file, "w") + del Tmin return result_dict @@ -732,6 +764,10 @@ def get_annual_tasmin_le_32F( # Compute statistics result_dict = metrics_json({index: Tle32}, obs_dict={}, region="land", regrid=False) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tle32.to_netcdf(nc_file, "w") + del Tle32 return result_dict @@ -764,6 +800,10 @@ def get_annualmean_pr( {index: PRmean}, obs_dict={}, region="land", regrid=False ) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + PRmean.to_netcdf(nc_file, "w") + del PRmean return result_dict @@ -793,6 +833,10 @@ def get_seasonalmean_pr( {index: PRmean}, obs_dict={}, region="land", regrid=False ) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + PRmean.to_netcdf(nc_file, "w") + del PRmean return result_dict @@ -820,6 +864,10 @@ def get_pr_q50( result_dict = metrics_json({index: PRq50}, obs_dict={}, region="land", regrid=False) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + PRq50.to_netcdf(nc_file, "w") + del PRq50 return result_dict @@ -877,52 +925,14 @@ def get_annual_pxx( result_dict = metrics_json({index: Pmax}, obs_dict={}, region="land", regrid=False) + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Pmax.to_netcdf(nc_file, "w") + del Pmax return result_dict -def precipitation_indices( - ds, sftlf, units_adjust, dec_mode, drop_incomplete_djf, annual_strict -): - # TODO: the precipitation metrics need to be broken out the way the - # temperature metrics were. - - # annualmean_pr, seasonalmean_pr, pr_q50, pr_q99p9, annual pxx - - print("Generating precipitation block extrema.") - - ds["pr"] = convert_units(ds["pr"], units_adjust) - - PR = TimeSeriesData(ds, "pr") - S = SeasonalAverager( - PR, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) - - # Rx1day - P1day = xr.Dataset() - P1day["ANN"] = S.annual_stats("mean", pentad=False) - # Can end up with very small negative values that should be 0 - # Possibly related to this issue? https://github.com/pydata/bottleneck/issues/332 - # (from https://github.com/pydata/xarray/issues/3855) - P1day["ANN"] = ( - P1day["ANN"].where(P1day["ANN"] > 0, 0).where(~np.isnan(P1day["ANN"]), np.nan) - ) - for season in ["DJF", "MAM", "JJA", "SON"]: - P1day[season] = S.seasonal_stats(season, "mean", pentad=False) - P1day[season] = ( - P1day[season] - .where(P1day[season] > 0, 0) - .where(~np.isnan(P1day[season]), np.nan) - ) - P1day = update_nc_attrs(P1day, dec_mode, drop_incomplete_djf, annual_strict) - - return P1day - - # A couple of statistics that aren't being loaded from mean_climate def mean_xy(data, varname): # Spatial mean of single dataset From 5fa8660a31a2328c0161a53871727db83c0237df Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 12 Jul 2024 16:57:32 -0700 Subject: [PATCH 31/88] fix file name --- pcmdi_metrics/drcdm/drcdm_driver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 4844c5889..dd23b6fdb 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -287,7 +287,9 @@ else: nc_base = None if plots: - fig_base = os.path.join(nc_dir, "_".join([model, run, "$index.nc"])) + fig_base = os.path.join( + fig_dir, "_".join([model, run, "$index.png"]) + ) else: fig_base = None From 8245bd60d9d5dd6457202493c118af17b7bc5f3f Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 12 Jul 2024 17:00:20 -0700 Subject: [PATCH 32/88] add figures --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 157 +++++++++++++++++++-- 1 file changed, 146 insertions(+), 11 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index c6bcc0db1..7b299ebb7 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -2,12 +2,15 @@ import datetime import cftime +import matplotlib.pyplot as plt import numpy as np import xarray as xr import xcdat as xc from pcmdi_metrics.stats import compute_statistics_dataset as pmp_stats +bgclr = [0.45, 0.45, 0.45] + class TimeSeriesData: # Track years and calendar for time series grids @@ -471,15 +474,14 @@ def get_mean_tasmax( result_dict = metrics_json({index: Tmean}, obs_dict={}, region="land", regrid=False) if fig_file is not None: - # Figures: - # Map of time mean value of Tmean - # - - # can replace the $index substring with any other string - # for specific file name. fig_file already contains the - # rest of the file path and the model and run names. - fig_file1 = fig_file.replace("$index", index) - print(fig_file1) + for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: + Tmean[season].mean("time").plot(cmap="Oranges", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, season])) + plt.title("Mean " + season + " daily high temperature") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() if nc_file is not None: nc_file = nc_file.replace("$index", index) @@ -511,6 +513,15 @@ def get_annual_txx( # Compute statistics result_dict = metrics_json({index: Tmax}, obs_dict={}, region="land", regrid=False) + if fig_file is not None: + Tmax["ANN"].mean("time").plot(cmap="Oranges", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual max daily high temperature") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tmax.to_netcdf(nc_file, "w") @@ -542,6 +553,15 @@ def get_tasmax_q50( {index: Tmedian}, obs_dict={}, region="land", regrid=False ) + if fig_file is not None: + Tmedian["ANN"].mean("time").plot(cmap="Oranges", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual median daily high temperature") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tmedian.to_netcdf(nc_file, "w") @@ -574,6 +594,15 @@ def get_tasmax_q99p9( {index: Tq99p9}, obs_dict={}, region="land", regrid=False ) + if fig_file is not None: + Tq99p9["ANN"].mean("time").plot(cmap="Oranges", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual 99.9th percentile daily high temperature") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tq99p9.to_netcdf(nc_file, "w") @@ -605,6 +634,17 @@ def get_annual_tasmax_ge_95F( # Compute statistics result_dict = metrics_json({index: Tge95}, obs_dict={}, region="land", regrid=False) + if fig_file is not None: + Tge95["ANN"].mean("time").plot( + cmap="Oranges", vmin=0, vmax=100, cbar_kwargs={"label": "%"} + ) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Mean percentage of days per year\nwith high temperature >= 95F") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tge95.to_netcdf(nc_file, "w") @@ -638,6 +678,17 @@ def get_annual_tasmax_ge_100F( {index: Tge100}, obs_dict={}, region="land", regrid=False ) + if fig_file is not None: + Tge100["ANN"].mean("time").plot( + cmap="Oranges", vmin=0, vmax=100, cbar_kwargs={"label": "%"} + ) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Mean percentage of days per year\nwith high temperature >= 100F") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tge100.to_netcdf(nc_file, "w") @@ -666,13 +717,22 @@ def get_annual_tasmax_ge_105F( Tge105.attrs["units"] = "%" Tge105 = update_nc_attrs(Tge105, dec_mode, drop_incomplete_djf, annual_strict) - # Do plots or saving files here - # Compute statistics result_dict = metrics_json( {index: Tge105}, obs_dict={}, region="land", regrid=False ) + if fig_file is not None: + Tge105["ANN"].mean("time").plot( + cmap="Oranges", vmin=0, vmax=100, cbar_kwargs={"label": "%"} + ) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Mean percentage of days per year\nwith high temperature >= 105F") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tge105.to_netcdf(nc_file, "w") @@ -703,6 +763,15 @@ def get_annual_tnn( # Compute statistics result_dict = metrics_json({index: Tmean}, obs_dict={}, region="land", regrid=False) + if fig_file is not None: + Tmean["ANN"].mean("time").plot(cmap="YlGnBu_r", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual minimum daily low temperature") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tmean.to_netcdf(nc_file, "w") @@ -733,6 +802,15 @@ def get_annualmean_tasmin( # Compute statistics result_dict = metrics_json({index: Tmin}, obs_dict={}, region="land", regrid=False) + if fig_file is not None: + Tmin["ANN"].mean("time").plot(cmap="YlGnBu_r", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual mean daily low temperature") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tmin.to_netcdf(nc_file, "w") @@ -764,6 +842,17 @@ def get_annual_tasmin_le_32F( # Compute statistics result_dict = metrics_json({index: Tle32}, obs_dict={}, region="land", regrid=False) + if fig_file is not None: + Tle32["ANN"].mean("time").plot( + cmap="RdPu", vmin=0, vmax=100, cbar_kwargs={"label": "%"} + ) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Mean percentage of days per year\nwith low temperature < 32F") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Tle32.to_netcdf(nc_file, "w") @@ -800,6 +889,15 @@ def get_annualmean_pr( {index: PRmean}, obs_dict={}, region="land", regrid=False ) + if fig_file is not None: + PRmean["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Annual mean daily precipitation") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) PRmean.to_netcdf(nc_file, "w") @@ -833,6 +931,16 @@ def get_seasonalmean_pr( {index: PRmean}, obs_dict={}, region="land", regrid=False ) + if fig_file is not None: + for season in ["DJF", "MAM", "JJA", "SON"]: + PRmean[season].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, season])) + plt.title(season + " mean daily precipitation") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) PRmean.to_netcdf(nc_file, "w") @@ -864,6 +972,15 @@ def get_pr_q50( result_dict = metrics_json({index: PRq50}, obs_dict={}, region="land", regrid=False) + if fig_file is not None: + PRq50["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual median daily precipitation") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) PRq50.to_netcdf(nc_file, "w") @@ -895,6 +1012,15 @@ def get_pr_q99p9( {index: PRq99p9}, obs_dict={}, region="land", regrid=False ) + if fig_file is not None: + PRq99p9["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Mean of annual 99.9th percentile daily precipitation") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) PRq99p9.to_netcdf(nc_file, "w") @@ -925,6 +1051,15 @@ def get_annual_pxx( result_dict = metrics_json({index: Pmax}, obs_dict={}, region="land", regrid=False) + if fig_file is not None: + Pmax["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual maximum daily precipitation") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + if nc_file is not None: nc_file = nc_file.replace("$index", index) Pmax.to_netcdf(nc_file, "w") From 9b216f8bc568d6284c7c4fb6ad6ed0b0aca88990 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 18 Jul 2024 16:32:33 -0700 Subject: [PATCH 33/88] add more tasmin --- pcmdi_metrics/drcdm/drcdm_driver.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index dd23b6fdb..30a7b39a9 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -388,7 +388,26 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - + result_dict = compute_metrics.get_annual_tasmin_le_0F( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_annual_tasmin_ge_70F( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) result_dict = compute_metrics.get_annualmean_tasmin( ds, sftlf, From 0db6a95ad3ffcac37f22974212e30d232fadaba2 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 18 Jul 2024 16:32:55 -0700 Subject: [PATCH 34/88] add more tasmin --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 89 +++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 7b299ebb7..18facf782 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -826,6 +826,7 @@ def get_annual_tasmin_le_32F( # equal to 32F. index = "annual_tasmin_le_32F" varname = "tasmin" + print("Metric:", index) TS = TimeSeriesData(ds, varname) S = SeasonalAverager( TS, @@ -847,7 +848,7 @@ def get_annual_tasmin_le_32F( cmap="RdPu", vmin=0, vmax=100, cbar_kwargs={"label": "%"} ) fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean percentage of days per year\nwith low temperature < 32F") + plt.title("Mean percentage of days per year\nwith low temperature <= 32F") ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) @@ -861,6 +862,92 @@ def get_annual_tasmin_le_32F( return result_dict +def get_annual_tasmin_le_0F( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual percentage of days with daily minimum temperature less than or + # equal to 0F. + index = "annual_tasmin_le_0F" + varname = "tasmin" + print("Metric:", index) + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + Tle0 = xr.Dataset() + Tle0["ANN"] = S.annual_stats("le0") + Tle0.attrs["units"] = "%" + Tle0 = update_nc_attrs(Tle0, dec_mode, drop_incomplete_djf, annual_strict) + + # Compute statistics + result_dict = metrics_json({index: Tle0}, obs_dict={}, region="land", regrid=False) + + if fig_file is not None: + Tle0["ANN"].mean("time").plot( + cmap="RdPu", vmin=0, vmax=100, cbar_kwargs={"label": "%"} + ) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Mean percentage of days per year\nwith low temperature <= 0F") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tle0.to_netcdf(nc_file, "w") + + del Tle0 + return result_dict + + +def get_annual_tasmin_ge_70F( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + # Get annual percentage of days with daily minimum temperature + # greater than or equal to 70F (warm nights) + index = "annual_tasmin_ge_70F" + varname = "tasmin" + print("Metric:", index) + TS = TimeSeriesData(ds, varname) + S = SeasonalAverager( + TS, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + Tge70 = xr.Dataset() + Tge70["ANN"] = S.annual_stats("ge70") + Tge70.attrs["units"] = "%" + Tge70 = update_nc_attrs(Tge70, dec_mode, drop_incomplete_djf, annual_strict) + + # Compute statistics + result_dict = metrics_json({index: Tge70}, obs_dict={}, region="land", regrid=False) + + if fig_file is not None: + Tge70["ANN"].mean("time").plot( + cmap="RdPu", vmin=0, vmax=100, cbar_kwargs={"label": "%"} + ) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Mean percentage of days per year\nwith low temperature >= 70F") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Tge70.to_netcdf(nc_file, "w") + + del Tge70 + return result_dict + + # TODO: Fill out precipitation metrics def get_annualmean_pr( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None From b7dbd39db2610c260d12db16867e51a1d02d6e40 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 19 Jul 2024 10:38:47 -0700 Subject: [PATCH 35/88] add mm thresh --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 18facf782..9890a7c2e 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -1043,6 +1043,9 @@ def get_pr_q50( varname = "pr" print("Metric:", index) + # Need at least 1mm rain + ds[varname] = ds[varname].where(ds[varname] >= 1) + # Get median (q50) precipitation PR = TimeSeriesData(ds, varname) S = SeasonalAverager( @@ -1083,6 +1086,9 @@ def get_pr_q99p9( varname = "pr" print("Metric:", index) + # Need at least 1mm rain + ds[varname] = ds[varname].where(ds[varname] >= 1) + # Get 99.9th percentile daily precipitation PR = TimeSeriesData(ds, varname) S = SeasonalAverager( From ac714c7ee1fbfcb6183631beccb8074b3b674199 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 23 Jul 2024 14:50:21 -0700 Subject: [PATCH 36/88] update comments --- pcmdi_metrics/drcdm/drcdm_driver.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 30a7b39a9..b4f4bee7a 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -264,23 +264,12 @@ # Get labels for start/end years from dataset yrs = [str(int(ds.time.dt.year[0])), str(int(ds.time.dt.year[-1]))] - # If any of the metrics use daily data we'll want to keep - # something like this calendar handling if ds.time.encoding["calendar"] != "noleap" and exclude_leap: ds = ds.convert_calendar("noleap") ds[varname] = compute_metrics.convert_units(ds[varname], ModUnitsAdjust) - # ------------------------------- - # Metrics go here - # ------------------------------- - # Maybe start with the metrics from this paper: https://climatemodeling.science.energy.gov/sites/default/files/2023-11/Validation%20of%20LOCA2%20and%20STAR-ESDM%20Statistically%20Downscaled%20Products%20v2.pdf - # - # pr: annualmean_pr, seasonalmean_pr, pr_q50, pr_q99p9, annual pxx - # tasmax: annualmean_tasmax, seasonalmean_tasmax, annual txx, annual_tasmax_ge_95F, - # annual_tasmax_ge_100F, annual_tasmax_ge_105F, tasmax_q50, tasmax_q99p9 - # tasmin: annualmean_tasmin, annual_tasmin_le_32F, annual_tnn - # ETTCDI has some metrics for tas as well + # Set up output file names if nc_out: # $index will be replaced with index name in metrics function nc_base = os.path.join(nc_dir, "_".join([model, run, "$index.nc"])) @@ -293,9 +282,8 @@ else: fig_base = None + # Do metrics calculation if varname == "tasmax": - # Example using get_annual_txx - # Need to work all temperature metrics this way result_dict = compute_metrics.get_mean_tasmax( ds, sftlf, From 8a20657fdca3acd15806945634aab9a830769e3c Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 23 Jul 2024 15:05:36 -0700 Subject: [PATCH 37/88] add T units --- pcmdi_metrics/drcdm/param/drcdm_param.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pcmdi_metrics/drcdm/param/drcdm_param.py b/pcmdi_metrics/drcdm/param/drcdm_param.py index 3f4c113ee..01c2fcd33 100644 --- a/pcmdi_metrics/drcdm/param/drcdm_param.py +++ b/pcmdi_metrics/drcdm/param/drcdm_param.py @@ -16,7 +16,7 @@ # Optional settings # See the README for more information about these settings -case_id = "test" +case_id = "test_tasmin" # sftlf_filename_template = '/p/css03/esgf_publish/CMIP6/CMIP/MIROC/MIROC6/piControl/r1i1p1f1/fx/sftlf/gn/v20190311/sftlf_fx_MIROC6_piControl_r1i1p1f1_gn.nc' ModUnitsAdjust = ( @@ -25,11 +25,13 @@ 86400.0, "mm/day", ) # Convert model units from kg/m2/s to mm/day +# ModUnitsAdjust=(True, 'KtoF', 0, 'F') dec_mode = "DJF" annual_strict = False drop_incomplete_djf = True regrid = False -plots = False +plot = True +netcdf = True generate_sftlf = True msyear = 3300 meyear = 3349 From 5b555833d6cd8e012a9df9ebbd89b6738524942b Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 25 Jul 2024 10:57:39 -0700 Subject: [PATCH 38/88] refactor temp --- pcmdi_metrics/drcdm/drcdm_driver.py | 135 ++++++++++++---------------- 1 file changed, 56 insertions(+), 79 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index b4f4bee7a..b1d88ac04 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -294,15 +294,15 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_tasmax_q99p9( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, - fig_base, - nc_base, - ) + # result_dict = compute_metrics.get_tasmax_q99p9( + # ds, + # sftlf, + # dec_mode, + # drop_incomplete_djf, + # annual_strict, + # fig_base, + # nc_base, + # ) metrics_dict["RESULTS"][model][run].update(result_dict) result_dict = compute_metrics.get_tasmax_q50( ds, @@ -324,36 +324,19 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_annual_tasmax_ge_95F( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, - fig_base, - nc_base, - ) - metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_annual_tasmax_ge_100F( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, - fig_base, - nc_base, - ) - metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_annual_tasmax_ge_105F( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, - fig_base, - nc_base, - ) - metrics_dict["RESULTS"][model][run].update(result_dict) + for deg in [86, 90, 95, 100, 105, 110, 115]: + # TODO: figure out how to handle C units + result_dict = compute_metrics.get_annual_tasmax_ge_XF( + ds, + sftlf, + deg, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) elif varname == "tasmin": result_dict = compute_metrics.get_annual_tnn( @@ -366,36 +349,30 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_annual_tasmin_le_32F( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, - fig_base, - nc_base, - ) - metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_annual_tasmin_le_0F( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, - fig_base, - nc_base, - ) - metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_annual_tasmin_ge_70F( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, - fig_base, - nc_base, - ) - metrics_dict["RESULTS"][model][run].update(result_dict) + for deg in [0, 32]: + result_dict = compute_metrics.get_annual_tasmin_le_XF( + ds, + sftlf, + deg, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + for deg in [70, 75, 80, 85, 90]: + result_dict = compute_metrics.get_annual_tasmin_ge_XF( + ds, + sftlf, + deg, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) result_dict = compute_metrics.get_annualmean_tasmin( ds, sftlf, @@ -441,16 +418,16 @@ ) metrics_dict["RESULTS"][model][run].update(result_dict) # 99.9 percentile - result_dict = compute_metrics.get_pr_q99p9( - ds, - sftlf, - dec_mode, - drop_incomplete_djf, - annual_strict, - fig_base, - nc_base, - ) - metrics_dict["RESULTS"][model][run].update(result_dict) + # result_dict = compute_metrics.get_pr_q99p9( + # ds, + # sftlf, + # dec_mode, + # drop_incomplete_djf, + # annual_strict, + # fig_base, + # nc_base, + # ) + # metrics_dict["RESULTS"][model][run].update(result_dict) # Max daily precip result_dict = compute_metrics.get_annual_pxx( ds, From 3fecbdd2df56af5b425006d1888a6959713b3755 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 25 Jul 2024 10:58:36 -0700 Subject: [PATCH 39/88] refactor temp --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 196 ++++++++------------- 1 file changed, 69 insertions(+), 127 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 9890a7c2e..07fc5dcbc 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -611,53 +611,18 @@ def get_tasmax_q99p9( return result_dict -def get_annual_tasmax_ge_95F( - ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None -): - # Get annual fraction of days with max temperature greater than or equal to 95F - index = "annual_tasmax_ge_95F" - print("Metric:", index) - varname = "tasmax" - TS = TimeSeriesData(ds, varname) - S = SeasonalAverager( - TS, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) - Tge95 = xr.Dataset() - Tge95["ANN"] = S.annual_stats("ge95") - Tge95.attrs["units"] = "%" - Tge95 = update_nc_attrs(Tge95, dec_mode, drop_incomplete_djf, annual_strict) - - # Compute statistics - result_dict = metrics_json({index: Tge95}, obs_dict={}, region="land", regrid=False) - - if fig_file is not None: - Tge95["ANN"].mean("time").plot( - cmap="Oranges", vmin=0, vmax=100, cbar_kwargs={"label": "%"} - ) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean percentage of days per year\nwith high temperature >= 95F") - ax = plt.gca() - ax.set_facecolor(bgclr) - plt.savefig(fig_file1) - plt.close() - - if nc_file is not None: - nc_file = nc_file.replace("$index", index) - Tge95.to_netcdf(nc_file, "w") - - del Tge95 - return result_dict - - -def get_annual_tasmax_ge_100F( - ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +def get_annual_tasmax_ge_XF( + ds, + sftlf, + deg, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_file=None, + nc_file=None, ): - # Get annual fraction of days with max temperature greater than or equal to 100F - index = "annual_tasmax_ge_100F" + # Get annual fraction of days with max temperature greater than or equal to deg F + index = "annual_tasmax_ge_{0}F".format(deg) print("Metric:", index) varname = "tasmax" TS = TimeSeriesData(ds, varname) @@ -668,66 +633,25 @@ def get_annual_tasmax_ge_100F( drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tge100 = xr.Dataset() - Tge100["ANN"] = S.annual_stats("ge95") - Tge100.attrs["units"] = "%" - Tge100 = update_nc_attrs(Tge100, dec_mode, drop_incomplete_djf, annual_strict) + TgeX = xr.Dataset() + print("ge{0}".format(deg)) + TgeX["ANN"] = S.annual_stats("ge{0}".format(deg)) + TgeX.attrs["units"] = "%" + TgeX = update_nc_attrs(TgeX, dec_mode, drop_incomplete_djf, annual_strict) # Compute statistics - result_dict = metrics_json( - {index: Tge100}, obs_dict={}, region="land", regrid=False - ) + result_dict = metrics_json({index: TgeX}, obs_dict={}, region="land", regrid=False) if fig_file is not None: - Tge100["ANN"].mean("time").plot( + TgeX["ANN"].mean("time").plot( cmap="Oranges", vmin=0, vmax=100, cbar_kwargs={"label": "%"} ) fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean percentage of days per year\nwith high temperature >= 100F") - ax = plt.gca() - ax.set_facecolor(bgclr) - plt.savefig(fig_file1) - plt.close() - - if nc_file is not None: - nc_file = nc_file.replace("$index", index) - Tge100.to_netcdf(nc_file, "w") - - del Tge100 - return result_dict - - -def get_annual_tasmax_ge_105F( - ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None -): - # Get annual fraction of days with max temperature greater than or equal to 105F - index = "annual_tasmax_ge_105F" - print("Metric:", index) - varname = "tasmax" - TS = TimeSeriesData(ds, varname) - S = SeasonalAverager( - TS, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) - Tge105 = xr.Dataset() - Tge105["ANN"] = S.annual_stats("ge95") - Tge105.attrs["units"] = "%" - Tge105 = update_nc_attrs(Tge105, dec_mode, drop_incomplete_djf, annual_strict) - - # Compute statistics - result_dict = metrics_json( - {index: Tge105}, obs_dict={}, region="land", regrid=False - ) - - if fig_file is not None: - Tge105["ANN"].mean("time").plot( - cmap="Oranges", vmin=0, vmax=100, cbar_kwargs={"label": "%"} + plt.title( + "Mean percentage of days per year\nwith high temperature >= {0}F".format( + deg + ) ) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean percentage of days per year\nwith high temperature >= 105F") ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) @@ -735,9 +659,9 @@ def get_annual_tasmax_ge_105F( if nc_file is not None: nc_file = nc_file.replace("$index", index) - Tge105.to_netcdf(nc_file, "w") + TgeX.to_netcdf(nc_file, "w") - del Tge105 + del TgeX return result_dict @@ -819,12 +743,19 @@ def get_annualmean_tasmin( return result_dict -def get_annual_tasmin_le_32F( - ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +def get_annual_tasmin_le_XF( + ds, + sftlf, + deg, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_file=None, + nc_file=None, ): # Get annual percentage of days with daily minimum temperature less than or - # equal to 32F. - index = "annual_tasmin_le_32F" + # equal to deg F. + index = "annual_tasmin_le_{0}F".format(deg) varname = "tasmin" print("Metric:", index) TS = TimeSeriesData(ds, varname) @@ -835,20 +766,22 @@ def get_annual_tasmin_le_32F( drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tle32 = xr.Dataset() - Tle32["ANN"] = S.annual_stats("le32") - Tle32.attrs["units"] = "%" - Tle32 = update_nc_attrs(Tle32, dec_mode, drop_incomplete_djf, annual_strict) + TleX = xr.Dataset() + TleX["ANN"] = S.annual_stats("le{0}".format(deg)) + TleX.attrs["units"] = "%" + TleX = update_nc_attrs(TleX, dec_mode, drop_incomplete_djf, annual_strict) # Compute statistics - result_dict = metrics_json({index: Tle32}, obs_dict={}, region="land", regrid=False) + result_dict = metrics_json({index: TleX}, obs_dict={}, region="land", regrid=False) if fig_file is not None: - Tle32["ANN"].mean("time").plot( + TleX["ANN"].mean("time").plot( cmap="RdPu", vmin=0, vmax=100, cbar_kwargs={"label": "%"} ) fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean percentage of days per year\nwith low temperature <= 32F") + plt.title( + "Mean percentage of days per year\nwith low temperature <= {0}F".format(deg) + ) ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) @@ -856,18 +789,25 @@ def get_annual_tasmin_le_32F( if nc_file is not None: nc_file = nc_file.replace("$index", index) - Tle32.to_netcdf(nc_file, "w") + TleX.to_netcdf(nc_file, "w") - del Tle32 + del TleX return result_dict -def get_annual_tasmin_le_0F( - ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +def get_annual_tasmin_ge_XF( + ds, + sftlf, + deg, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_file=None, + nc_file=None, ): - # Get annual percentage of days with daily minimum temperature less than or - # equal to 0F. - index = "annual_tasmin_le_0F" + # Get annual percentage of days with daily minimum temperature + # greater than or equal to deg F (warm nights) + index = "annual_tasmin_ge_{0}F".format(deg) varname = "tasmin" print("Metric:", index) TS = TimeSeriesData(ds, varname) @@ -878,20 +818,22 @@ def get_annual_tasmin_le_0F( drop_incomplete_djf=drop_incomplete_djf, annual_strict=annual_strict, ) - Tle0 = xr.Dataset() - Tle0["ANN"] = S.annual_stats("le0") - Tle0.attrs["units"] = "%" - Tle0 = update_nc_attrs(Tle0, dec_mode, drop_incomplete_djf, annual_strict) + TgeX = xr.Dataset() + TgeX["ANN"] = S.annual_stats("ge{0}".format(deg)) + TgeX.attrs["units"] = "%" + TgeX = update_nc_attrs(TgeX, dec_mode, drop_incomplete_djf, annual_strict) # Compute statistics - result_dict = metrics_json({index: Tle0}, obs_dict={}, region="land", regrid=False) + result_dict = metrics_json({index: TgeX}, obs_dict={}, region="land", regrid=False) if fig_file is not None: - Tle0["ANN"].mean("time").plot( + TgeX["ANN"].mean("time").plot( cmap="RdPu", vmin=0, vmax=100, cbar_kwargs={"label": "%"} ) fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean percentage of days per year\nwith low temperature <= 0F") + plt.title( + "Mean percentage of days per year\nwith low temperature >= {0}F".format(deg) + ) ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) @@ -899,9 +841,9 @@ def get_annual_tasmin_le_0F( if nc_file is not None: nc_file = nc_file.replace("$index", index) - Tle0.to_netcdf(nc_file, "w") + TgeX.to_netcdf(nc_file, "w") - del Tle0 + del TgeX return result_dict From bf722c49d28e7777320dc9ea67afafc143b73a3d Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 1 Aug 2024 15:44:53 -0700 Subject: [PATCH 40/88] add cwd --- pcmdi_metrics/drcdm/drcdm_driver.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index b1d88ac04..a36fbd352 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -439,6 +439,16 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_annual_cwd( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) if run not in metrics_dict["DIMENSIONS"]["realization"]: metrics_dict["DIMENSIONS"]["realization"].append(run) From dc4249be9c0e604f41edc3b9d03d5bacd5e49507 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 1 Aug 2024 15:46:30 -0700 Subject: [PATCH 41/88] add cwd --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 07fc5dcbc..1f1193f2b 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -6,6 +6,7 @@ import numpy as np import xarray as xr import xcdat as xc +import xclim from pcmdi_metrics.stats import compute_statistics_dataset as pmp_stats @@ -1103,6 +1104,39 @@ def get_annual_pxx( return result_dict +def get_annual_cwd( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + index = "annual_cwd" + + # Get annual max precipitation + ds["pr"] = ds["pr"].where(sftlf.sftlf >= 0.5).where(sftlf.sftlf <= 1) + Pcwd = xr.Dataset() + Pcwd["ANN"] = xclim.indices.maximum_consecutive_wet_days( + ds.pr, thresh="1 mm/day", freq="YS", resample_before_rl=True + ) + Pcwd["time"]["encoding"]["calendar"] = ds.time.encoding["calendar"] + Pcwd = update_nc_attrs(Pcwd, dec_mode, drop_incomplete_djf, annual_strict) + + result_dict = metrics_json({index: Pcwd}, obs_dict={}, region="land", regrid=False) + + if fig_file is not None: + Pcwd["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "count"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual max count of consecutive wet days") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Pcwd.to_netcdf(nc_file, "w") + + del Pcwd + return result_dict + + # A couple of statistics that aren't being loaded from mean_climate def mean_xy(data, varname): # Spatial mean of single dataset From 5e64f155808cfd91a8dd7339a06b3412eb97cfb8 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 2 Aug 2024 13:13:31 -0700 Subject: [PATCH 42/88] fix cal --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 1f1193f2b..aa901ca1e 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -1115,7 +1115,7 @@ def get_annual_cwd( Pcwd["ANN"] = xclim.indices.maximum_consecutive_wet_days( ds.pr, thresh="1 mm/day", freq="YS", resample_before_rl=True ) - Pcwd["time"]["encoding"]["calendar"] = ds.time.encoding["calendar"] + Pcwd["time"].encoding["calendar"] = ds.time.encoding["calendar"] Pcwd = update_nc_attrs(Pcwd, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json({index: Pcwd}, obs_dict={}, region="land", regrid=False) From b29f8fa9e5f1e69cc29e2a85e5dfdd4950f810a5 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Mon, 5 Aug 2024 09:33:09 -0700 Subject: [PATCH 43/88] comment cwd --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index aa901ca1e..bc38330f7 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import datetime import cftime @@ -6,10 +5,12 @@ import numpy as np import xarray as xr import xcdat as xc -import xclim from pcmdi_metrics.stats import compute_statistics_dataset as pmp_stats +# import xclim + + bgclr = [0.45, 0.45, 0.45] @@ -1070,6 +1071,7 @@ def get_annual_pxx( ): varname = "pr" index = "annual_pxx" + print("Metric:", index) # Get annual max precipitation PR = TimeSeriesData(ds, varname) @@ -1104,10 +1106,12 @@ def get_annual_pxx( return result_dict +""" def get_annual_cwd( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): index = "annual_cwd" + print("Metric:", index) # Get annual max precipitation ds["pr"] = ds["pr"].where(sftlf.sftlf >= 0.5).where(sftlf.sftlf <= 1) @@ -1115,7 +1119,7 @@ def get_annual_cwd( Pcwd["ANN"] = xclim.indices.maximum_consecutive_wet_days( ds.pr, thresh="1 mm/day", freq="YS", resample_before_rl=True ) - Pcwd["time"].encoding["calendar"] = ds.time.encoding["calendar"] + Pcwd["time"].encoding["calendar"] = ds.time["calendar"] Pcwd = update_nc_attrs(Pcwd, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json({index: Pcwd}, obs_dict={}, region="land", regrid=False) @@ -1135,6 +1139,7 @@ def get_annual_cwd( del Pcwd return result_dict +""" # A couple of statistics that aren't being loaded from mean_climate From 61b65eb3663057f3badc2b179de5f93ee4864f8e Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Wed, 7 Aug 2024 16:50:53 -0700 Subject: [PATCH 44/88] fix cwd --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index bc38330f7..dd9fd84b7 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -5,12 +5,10 @@ import numpy as np import xarray as xr import xcdat as xc +import xclim from pcmdi_metrics.stats import compute_statistics_dataset as pmp_stats -# import xclim - - bgclr = [0.45, 0.45, 0.45] @@ -1106,7 +1104,6 @@ def get_annual_pxx( return result_dict -""" def get_annual_cwd( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): @@ -1119,7 +1116,7 @@ def get_annual_cwd( Pcwd["ANN"] = xclim.indices.maximum_consecutive_wet_days( ds.pr, thresh="1 mm/day", freq="YS", resample_before_rl=True ) - Pcwd["time"].encoding["calendar"] = ds.time["calendar"] + Pcwd.time.encoding["calendar"] = ds.time.encoding["calendar"] Pcwd = update_nc_attrs(Pcwd, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json({index: Pcwd}, obs_dict={}, region="land", regrid=False) @@ -1139,7 +1136,6 @@ def get_annual_cwd( del Pcwd return result_dict -""" # A couple of statistics that aren't being loaded from mean_climate From 7e97fe87b6ed3b2258f22928a94ee16c5f404f86 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 9 Aug 2024 15:09:13 -0700 Subject: [PATCH 45/88] add cdd --- pcmdi_metrics/drcdm/drcdm_driver.py | 47 +++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index a36fbd352..4478cf47f 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -406,8 +406,8 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - # Median - result_dict = compute_metrics.get_pr_q50( + # Max daily precip + result_dict = compute_metrics.get_annual_pxx( ds, sftlf, dec_mode, @@ -417,19 +417,8 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - # 99.9 percentile - # result_dict = compute_metrics.get_pr_q99p9( - # ds, - # sftlf, - # dec_mode, - # drop_incomplete_djf, - # annual_strict, - # fig_base, - # nc_base, - # ) - # metrics_dict["RESULTS"][model][run].update(result_dict) - # Max daily precip - result_dict = compute_metrics.get_annual_pxx( + # Consecutive wet days + result_dict = compute_metrics.get_annual_cwd( ds, sftlf, dec_mode, @@ -439,7 +428,33 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - result_dict = compute_metrics.get_annual_cwd( + # Consecutive dry days + result_dict = compute_metrics.get_annual_cdd( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + # Median + # This function must come after the consecutive dry days function + # Otherwise cdd generates all zeros. Unsure why. + result_dict = compute_metrics.get_pr_q50( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) + # 99.9 percentile + # Long running, can take 50 min per reals + result_dict = compute_metrics.get_pr_q99p9( ds, sftlf, dec_mode, From 42373589d9948d08f87f737f84db6be3cd59f68c Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 9 Aug 2024 15:09:36 -0700 Subject: [PATCH 46/88] add cdd --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 50 +++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index dd9fd84b7..0d9866d20 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -1110,14 +1110,17 @@ def get_annual_cwd( index = "annual_cwd" print("Metric:", index) - # Get annual max precipitation - ds["pr"] = ds["pr"].where(sftlf.sftlf >= 0.5).where(sftlf.sftlf <= 1) + # Get continuous wet days Pcwd = xr.Dataset() - Pcwd["ANN"] = xclim.indices.maximum_consecutive_wet_days( - ds.pr, thresh="1 mm/day", freq="YS", resample_before_rl=True + Pcwd["ANN"] = ( + xclim.indices.maximum_consecutive_wet_days( + ds.pr, thresh="1 mm/day", freq="YS", resample_before_rl=True + ) + .where(sftlf.sftlf >= 0.5) + .where(sftlf.sftlf <= 1) ) Pcwd.time.encoding["calendar"] = ds.time.encoding["calendar"] - Pcwd = update_nc_attrs(Pcwd, dec_mode, drop_incomplete_djf, annual_strict) + Pcwd = update_nc_attrs(Pcwd, dec_mode, drop_incomplete_djf, True) result_dict = metrics_json({index: Pcwd}, obs_dict={}, region="land", regrid=False) @@ -1138,6 +1141,43 @@ def get_annual_cwd( return result_dict +def get_annual_cdd( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + index = "annual_cdd" + print("Metric:", index) + + # Get continuous dry days + Pcdd = xr.Dataset() + Pcdd["ANN"] = ( + xclim.indices.maximum_consecutive_dry_days( + ds.pr, thresh="1 mm/day", freq="YS", resample_before_rl=True + ) + .where(sftlf.sftlf >= 0.5) + .where(sftlf.sftlf <= 1) + ) + Pcdd.time.encoding["calendar"] = ds.time.encoding["calendar"] + Pcdd = update_nc_attrs(Pcdd, dec_mode, drop_incomplete_djf, True) + + result_dict = metrics_json({index: Pcdd}, obs_dict={}, region="land", regrid=False) + + if fig_file is not None: + Pcdd["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "count"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Average annual max count of consecutive dry days") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + Pcdd.to_netcdf(nc_file, "w") + + del Pcdd + return result_dict + + # A couple of statistics that aren't being loaded from mean_climate def mean_xy(data, varname): # Spatial mean of single dataset From 03be5eaffd3b902cdf0d4ed9d078a5a6e8d3b659 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 13 Aug 2024 13:58:03 -0700 Subject: [PATCH 47/88] uncomment quant --- pcmdi_metrics/drcdm/drcdm_driver.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 4478cf47f..3e81c2c3a 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -294,15 +294,15 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) - # result_dict = compute_metrics.get_tasmax_q99p9( - # ds, - # sftlf, - # dec_mode, - # drop_incomplete_djf, - # annual_strict, - # fig_base, - # nc_base, - # ) + result_dict = compute_metrics.get_tasmax_q99p9( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) metrics_dict["RESULTS"][model][run].update(result_dict) result_dict = compute_metrics.get_tasmax_q50( ds, From cbac3323e6cdd202842fa43783ed6dddb19967cc Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 13 Aug 2024 13:58:20 -0700 Subject: [PATCH 48/88] fix return --- pcmdi_metrics/drcdm/lib/utilities.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pcmdi_metrics/drcdm/lib/utilities.py b/pcmdi_metrics/drcdm/lib/utilities.py index f4df82f83..ebeffe6b9 100644 --- a/pcmdi_metrics/drcdm/lib/utilities.py +++ b/pcmdi_metrics/drcdm/lib/utilities.py @@ -124,6 +124,7 @@ def set_up_realizations(realization): elif isinstance(realization, str): if realization.lower() in ["all", "*"]: find_all_realizations = True + realizations = None else: realizations = [realization] elif isinstance(realization, list): From 4b555a71300206e6b913729e7bfe21e1ecd8274b Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 22 Aug 2024 16:42:11 -0700 Subject: [PATCH 49/88] fix xy vars and tasmin --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 23 +++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 0d9866d20..64c25ac0d 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -7,6 +7,7 @@ import xcdat as xc import xclim +from pcmdi_metrics.io.xcdat_dataset_io import get_latitude_key, get_longitude_key from pcmdi_metrics.stats import compute_statistics_dataset as pmp_stats bgclr = [0.45, 0.45, 0.45] @@ -380,9 +381,13 @@ def seasonal_stats(self, season, stat, pentad=False): def update_nc_attrs(ds, dec_mode, drop_incomplete_djf, annual_strict): # Add bounds and record user settings in attributes # Use this function for any general dataset updates. - ds.lat.attrs["standard_name"] = "Y" - ds.lon.attrs["standard_name"] = "X" - bnds_dict = {"lat": "Y", "lon": "X", "time": "T"} + xvar = get_longitude_key(ds) + yvar = get_latitude_key(ds) + xvarbnds = xvar + "_bnds" + yvarbnds = yvar + "_bnds" + ds[yvar].attrs["standard_name"] = "Y" + ds[xvar].attrs["standard_name"] = "X" + bnds_dict = {yvar: "Y", xvar: "X", "time": "T"} for item in bnds_dict: if "bounds" in ds[item].attrs: bnds_var = ds[item].attrs["bounds"] @@ -396,11 +401,11 @@ def update_nc_attrs(ds, dec_mode, drop_incomplete_djf, annual_strict): ds.attrs["annual_strict"] = str(annual_strict) # Update fill value encoding - ds.lat.encoding["_FillValue"] = None - ds.lon.encoding["_FillValue"] = None + ds[yvar].encoding["_FillValue"] = None + ds[xvar].encoding["_FillValue"] = None ds.time.encoding["_FillValue"] = None - ds.lat_bnds.encoding["_FillValue"] = None - ds.lon_bnds.encoding["_FillValue"] = None + ds[yvarbnds].encoding["_FillValue"] = None + ds[xvarbnds].encoding["_FillValue"] = None ds.time_bnds.encoding["_FillValue"] = None for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: if season in ds: @@ -681,7 +686,7 @@ def get_annual_tnn( annual_strict=annual_strict, ) Tmean = xr.Dataset() - Tmean["ANN"] = S.annual_stats("mean") + Tmean["ANN"] = S.annual_stats("min") Tmean = update_nc_attrs(Tmean, dec_mode, drop_incomplete_djf, annual_strict) # Compute statistics @@ -720,7 +725,7 @@ def get_annualmean_tasmin( annual_strict=annual_strict, ) Tmin = xr.Dataset() - Tmin["ANN"] = S.annual_stats("min") + Tmin["ANN"] = S.annual_stats("mean") Tmin = update_nc_attrs(Tmin, dec_mode, drop_incomplete_djf, annual_strict) # Compute statistics From 25b4143512b9358aa26c34e6345445137725c7d2 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 22 Aug 2024 16:42:25 -0700 Subject: [PATCH 50/88] fix xy vars --- pcmdi_metrics/drcdm/drcdm_driver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 3e81c2c3a..a95f43e44 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -12,6 +12,7 @@ region_utilities, utilities, ) +from pcmdi_metrics.io.xcdat_dataset_io import get_latitude_key from pcmdi_metrics.utils import create_land_sea_mask if __name__ == "__main__": @@ -249,7 +250,8 @@ ) # Mask out Antarctica - sftlf["sftlf"] = sftlf["sftlf"].where(sftlf.lat > -60) + sflat = get_latitude_key(sftlf) + sftlf["sftlf"] = sftlf["sftlf"].where(sftlf[sflat] > -60) if use_region_mask: print("Creating dataset mask.") From 84cc350abd5e18d3e267a53303a2ceb8bfdc21cb Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 22 Aug 2024 16:42:36 -0700 Subject: [PATCH 51/88] fix files --- pcmdi_metrics/drcdm/lib/utilities.py | 32 +++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/utilities.py b/pcmdi_metrics/drcdm/lib/utilities.py index ebeffe6b9..c57d7cdb4 100644 --- a/pcmdi_metrics/drcdm/lib/utilities.py +++ b/pcmdi_metrics/drcdm/lib/utilities.py @@ -8,20 +8,37 @@ from pcmdi_metrics.io import xcdat_openxml from pcmdi_metrics.io.base import Base -from pcmdi_metrics.utils import create_land_sea_mask def load_dataset(filepath): # Load an xarray dataset from the given filepath. # If list of netcdf files, opens mfdataset. # If list of xmls, open last file in list. + def fix_calendar(ds): + cal = ds.time.calendar + # Add any calendar fixes here + cal = cal.replace("-", "_") + ds.time.attrs["calendar"] = cal + ds = xcdat.decode_time(ds) + return ds + if filepath[-1].endswith(".xml"): # Final item of sorted list would have most recent version date ds = xcdat_openxml.xcdat_openxml(filepath[-1]) elif len(filepath) > 1: - ds = xcdat.open_mfdataset(filepath, chunks={"time": -1}) + try: + ds = xcdat.open_mfdataset(filepath, chunks={"time": -1}) + except ValueError: + ds = xcdat.open_mfdataset(filepath, chunks={"time": -1}, decode_times=False) + ds = fix_calendar(ds) else: - ds = xcdat.open_dataset(filepath[0], chunks={"time": -1}) + try: + ds = xcdat.open_dataset(filepath[0], chunks={"time": -1}) + except ValueError: + ds = xcdat.open_dataset( + filepath[0], chunks={"time": -1}, decode_times=False + ) + ds = fix_calendar(ds) return ds @@ -131,12 +148,3 @@ def set_up_realizations(realization): realizations = realization return find_all_realizations, realizations - - -def generate_land_sea_mask(data, debug=False): - # generate sftlf if not provided. - sft = create_land_sea_mask(data) - sftlf = data.copy(data=None) - sftlf["sftlf"] = sft * 100 - - return sftlf From 064fe7214ac02a5a84b233557126fc5ad91e9d3e Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 20 Sep 2024 13:27:41 -0700 Subject: [PATCH 52/88] fix bug --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 64c25ac0d..cc752852b 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -145,8 +145,8 @@ def annual_stats(self, stat, pentad=False): num = int(stat.replace("ge", "")) ds_ann = ( ds.where(ds >= num) - .groupby("time.year") .sel(time=date_range, method="nearest") + .groupby("time.year") .count(dim="time") / ds.sel(time=date_range, method="nearest") .groupby("time.year") From 93d48e1b8b2efda66618271b932fc47401353625 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 20 Sep 2024 13:28:26 -0700 Subject: [PATCH 53/88] fix bug --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index cc752852b..b2231071c 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -430,7 +430,7 @@ def convert_units(data, units_adjust): "multiply": "*", "divide": "/", "CtoF": ")*(9/5)+32", - "KtoF": "-273)*(9/5)+32", + "KtoF": "-273.15)*(9/5)+32", } if str(units_adjust[1]) not in op_dict: print( From 965b4a31f11f0b5524dadb4870601541d357714b Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 24 Sep 2024 11:56:32 -0700 Subject: [PATCH 54/88] add pr 99p0 --- pcmdi_metrics/drcdm/drcdm_driver.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index a95f43e44..5f8c60ca1 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -466,6 +466,16 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) + result_dict = compute_metrics.get_pr_q99p0( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) if run not in metrics_dict["DIMENSIONS"]["realization"]: metrics_dict["DIMENSIONS"]["realization"].append(run) From 4833f38e944b3194ef14062b2c0fc019d8f7a681 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 24 Sep 2024 11:56:39 -0700 Subject: [PATCH 55/88] add pr 99p0 --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index b2231071c..14dbc972f 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -1026,6 +1026,49 @@ def get_pr_q50( return result_dict +def get_pr_q99p0( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + index = "pr_q99p0" + varname = "pr" + print("Metric:", index) + + # Need at least 1mm rain + ds[varname] = ds[varname].where(ds[varname] >= 1) + + # Get 99.9th percentile daily precipitation + PR = TimeSeriesData(ds, varname) + S = SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + PRq99p0 = xr.Dataset() + PRq99p0["ANN"] = S.annual_stats("q99p0") + PRq99p0 = update_nc_attrs(PRq99p0, dec_mode, drop_incomplete_djf, annual_strict) + result_dict = metrics_json( + {index: PRq99p0}, obs_dict={}, region="land", regrid=False + ) + + if fig_file is not None: + PRq99p0["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) + plt.title("Mean of annual 99.9th percentile daily precipitation") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + PRq99p0.to_netcdf(nc_file, "w") + + del PRq99p0 + return result_dict + + def get_pr_q99p9( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): From d646a3703355b9b522a46f77ac24ec010f05fed9 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 24 Sep 2024 11:58:08 -0700 Subject: [PATCH 56/88] cleanup --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 46 +--------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 14dbc972f..df6729418 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -852,50 +852,6 @@ def get_annual_tasmin_ge_XF( return result_dict -def get_annual_tasmin_ge_70F( - ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None -): - # Get annual percentage of days with daily minimum temperature - # greater than or equal to 70F (warm nights) - index = "annual_tasmin_ge_70F" - varname = "tasmin" - print("Metric:", index) - TS = TimeSeriesData(ds, varname) - S = SeasonalAverager( - TS, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) - Tge70 = xr.Dataset() - Tge70["ANN"] = S.annual_stats("ge70") - Tge70.attrs["units"] = "%" - Tge70 = update_nc_attrs(Tge70, dec_mode, drop_incomplete_djf, annual_strict) - - # Compute statistics - result_dict = metrics_json({index: Tge70}, obs_dict={}, region="land", regrid=False) - - if fig_file is not None: - Tge70["ANN"].mean("time").plot( - cmap="RdPu", vmin=0, vmax=100, cbar_kwargs={"label": "%"} - ) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean percentage of days per year\nwith low temperature >= 70F") - ax = plt.gca() - ax.set_facecolor(bgclr) - plt.savefig(fig_file1) - plt.close() - - if nc_file is not None: - nc_file = nc_file.replace("$index", index) - Tge70.to_netcdf(nc_file, "w") - - del Tge70 - return result_dict - - -# TODO: Fill out precipitation metrics def get_annualmean_pr( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): @@ -1036,7 +992,7 @@ def get_pr_q99p0( # Need at least 1mm rain ds[varname] = ds[varname].where(ds[varname] >= 1) - # Get 99.9th percentile daily precipitation + # Get 99th percentile daily precipitation PR = TimeSeriesData(ds, varname) S = SeasonalAverager( PR, From fd870cc012180edfc52ffb4dfbd37293556ec8f9 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 24 Sep 2024 12:00:50 -0700 Subject: [PATCH 57/88] add seasonal tasmin --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index df6729418..13bf14589 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -726,19 +726,22 @@ def get_annualmean_tasmin( ) Tmin = xr.Dataset() Tmin["ANN"] = S.annual_stats("mean") + for season in ["DJF", "MAM", "JJA", "SON"]: + Tmin[season] = S.seasonal_stats(season, "mean") Tmin = update_nc_attrs(Tmin, dec_mode, drop_incomplete_djf, annual_strict) # Compute statistics result_dict = metrics_json({index: Tmin}, obs_dict={}, region="land", regrid=False) if fig_file is not None: - Tmin["ANN"].mean("time").plot(cmap="YlGnBu_r", cbar_kwargs={"label": "F"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Average annual mean daily low temperature") - ax = plt.gca() - ax.set_facecolor(bgclr) - plt.savefig(fig_file1) - plt.close() + for season in ["ANN", "MAM", "JJA", "SON", "DJF"]: + Tmin[season].mean("time").plot(cmap="YlGnBu_r", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, season])) + plt.title("Average " + season + " mean daily low temperature") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() if nc_file is not None: nc_file = nc_file.replace("$index", index) From 24d153458c6f0b18ce3dd572ed528c0910e83d3a Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 1 Oct 2024 17:34:33 -0700 Subject: [PATCH 58/88] add wettest day 5 year --- pcmdi_metrics/drcdm/drcdm_driver.py | 11 ++++ pcmdi_metrics/drcdm/lib/compute_metrics.py | 70 ++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 5f8c60ca1..826232624 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -419,6 +419,17 @@ nc_base, ) metrics_dict["RESULTS"][model][run].update(result_dict) + # Wettest day in 5 year range + result_dict = compute_metrics.get_wettest_5yr( + ds, + sftlf, + dec_mode, + drop_incomplete_djf, + annual_strict, + fig_base, + nc_base, + ) + metrics_dict["RESULTS"][model][run].update(result_dict) # Consecutive wet days result_dict = compute_metrics.get_annual_cwd( ds, diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 13bf14589..38f103db6 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -1111,6 +1111,76 @@ def get_annual_pxx( return result_dict +def get_wettest_5yr( + ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None +): + varname = "pr" + index = "wettest_5yr" + print("Metric:", index) + + # Get annual max precipitation + PR = TimeSeriesData(ds, varname) + S = SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + + Pmax = xr.Dataset() + Pmax["ANN"] = S.annual_stats("max") + + # Get number of 5 year segments + nseg = int(np.floor(len(Pmax.time) / 5)) + lo = int(len(Pmax.time)) % 5 # leftover + + # Set up empty dataset + P5 = xr.zeros_like(Pmax.isel({"time": slice(0, nseg)})) + P5["lat"] = Pmax["lat"] + P5["lon"] = Pmax["lon"] + + # Will be updated time for 5 year quantity + timelist = [] + + # Fill dataset + for seg in range(0, nseg): + start = lo + 5 * seg + end = lo + 5 * (seg + 1) + myslice = ( + Pmax["ANN"].isel({"time": slice(start, end)}).max("time").compute().data + ) + mytime = Pmax["time"].isel({"time": start}).data.item() + timelist.append(mytime) + P5["ANN"].loc[{"time": Pmax["time"][seg]}] = myslice + + P5["time"] = timelist + P5.time.attrs["axis"] = "T" + P5["time"].encoding["calendar"] = Pmax.time.encoding["calendar"] + P5["time"].attrs["standard_name"] = "time" + P5.time.encoding["units"] = Pmax.time.encoding["units"] + P5 = update_nc_attrs(P5, dec_mode, drop_incomplete_djf, annual_strict) + P5 = P5.rename({"ANN": "ANN5"}) + + result_dict = metrics_json({index: P5}, obs_dict={}, region="land", regrid=False) + + if fig_file is not None: + P5["ANN5"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "ANN5"])) + plt.title("Average precipitation on wettest day in 5 years") + ax = plt.gca() + ax.set_facecolor(bgclr) + plt.savefig(fig_file1) + plt.close() + + if nc_file is not None: + nc_file = nc_file.replace("$index", index) + P5.to_netcdf(nc_file, "w") + + del Pmax, P5 + return result_dict + + def get_annual_cwd( ds, sftlf, dec_mode, drop_incomplete_djf, annual_strict, fig_file=None, nc_file=None ): From 2e74b3665d53c2c2bcdd3ff7d3c619657eb68352 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 8 Oct 2024 08:39:01 -0700 Subject: [PATCH 59/88] no land/sea mask --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 38f103db6..767f55493 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -58,7 +58,8 @@ def __init__( def masked_ds(self, ds): # Mask land where 0.5<=sftlf<=1 - return ds.where(self.sftlf >= 0.5).where(self.sftlf <= 1) + #return ds.where(self.sftlf >= 0.5).where(self.sftlf <= 1) + return ds def calc_5day_mean(self): # Get the 5-day mean dataset From 8e53d22e95dcc0d5a9c703a0aceb191dc6551f3c Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 8 Oct 2024 08:54:49 -0700 Subject: [PATCH 60/88] change quantiles --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 121 +++++++++------------ 1 file changed, 53 insertions(+), 68 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 767f55493..436ca31e1 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -542,16 +542,13 @@ def get_tasmax_q50( # Get annual median maximum daily temperature index = "tasmax_q50" varname = "tasmax" - TS = TimeSeriesData(ds, varname) - S = SeasonalAverager( - TS, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) - Tmedian = xr.Dataset() - Tmedian["ANN"] = S.annual_stats("median") + # Set up empty dataset + Tmedian = xr.zeros_like(ds) + Tmedian["lat"] = ds["lat"] + Tmedian["lon"] = ds["lon"] + Tmedian = Tmedian.drop_vars("time") + + Tmedian["median"] = ds[varname].median("time") Tmedian = update_nc_attrs(Tmedian, dec_mode, drop_incomplete_djf, annual_strict) # Compute statistics @@ -560,9 +557,9 @@ def get_tasmax_q50( ) if fig_file is not None: - Tmedian["ANN"].mean("time").plot(cmap="Oranges", cbar_kwargs={"label": "F"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Average annual median daily high temperature") + Tmedian["median"].plot(cmap="Oranges", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "median"])) + plt.title("Time median daily high temperature") ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) @@ -583,17 +580,18 @@ def get_tasmax_q99p9( index = "tasmax_q99p9" print("Metric:", index) varname = "tasmax" - TS = TimeSeriesData(ds, varname) - S = SeasonalAverager( - TS, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) - Tq99p9 = xr.Dataset() - Tq99p9["ANN"] = S.annual_stats("q99p9") + + # Set up empty dataset + Tq99p9 = xr.zeros_like(ds) + Tq99p9["lat"] = ds["lat"] + Tq99p9["lon"] = ds["lon"] + Tq99p9 = Tq99p9.drop_vars("time") + + Tq99p9["q99p9"] = ds[varname].quantile(0.999,dim="time") Tq99p9 = update_nc_attrs(Tq99p9, dec_mode, drop_incomplete_djf, annual_strict) + result_dict = metrics_json( + {index: Tq99p9}, obs_dict={}, region="land", regrid=False + ) # Compute statistics result_dict = metrics_json( @@ -601,9 +599,9 @@ def get_tasmax_q99p9( ) if fig_file is not None: - Tq99p9["ANN"].mean("time").plot(cmap="Oranges", cbar_kwargs={"label": "F"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Average annual 99.9th percentile daily high temperature") + Tq99p9["q99p9"].plot(cmap="Oranges", cbar_kwargs={"label": "F"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "q99p9"])) + plt.title("99.9th percentile daily high temperature") ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) @@ -953,26 +951,21 @@ def get_pr_q50( # Need at least 1mm rain ds[varname] = ds[varname].where(ds[varname] >= 1) - # Get median (q50) precipitation - PR = TimeSeriesData(ds, varname) - S = SeasonalAverager( - PR, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) + # Set up empty dataset + PRq50 = xr.zeros_like(ds) + PRq50["lat"] = ds["lat"] + PRq50["lon"] = ds["lon"] + PRq50 = PRq50.drop_vars("time") - PRq50 = xr.Dataset() - PRq50["ANN"] = S.annual_stats("median") + PRq50["median"] = ds[varname].median("time") PRq50 = update_nc_attrs(PRq50, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json({index: PRq50}, obs_dict={}, region="land", regrid=False) if fig_file is not None: - PRq50["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Average annual median daily precipitation") + PRq50["median"].plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "median"])) + plt.title("Time median daily precipitation") ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) @@ -996,26 +989,22 @@ def get_pr_q99p0( # Need at least 1mm rain ds[varname] = ds[varname].where(ds[varname] >= 1) - # Get 99th percentile daily precipitation - PR = TimeSeriesData(ds, varname) - S = SeasonalAverager( - PR, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) - PRq99p0 = xr.Dataset() - PRq99p0["ANN"] = S.annual_stats("q99p0") + # Set up empty dataset + PRq99p0 = xr.zeros_like(ds) + PRq99p0["lat"] = ds["lat"] + PRq99p0["lon"] = ds["lon"] + PRq99p0 = PRq99p0.drop_vars("time") + + PRq99p0["q99p0"] = ds[varname].quantile(0.990,dim="time") PRq99p0 = update_nc_attrs(PRq99p0, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: PRq99p0}, obs_dict={}, region="land", regrid=False ) if fig_file is not None: - PRq99p0["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean of annual 99.9th percentile daily precipitation") + PRq99p0["q99p0"].plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "q99p0"])) + plt.title("99.9th percentile daily precipitation") ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) @@ -1039,26 +1028,22 @@ def get_pr_q99p9( # Need at least 1mm rain ds[varname] = ds[varname].where(ds[varname] >= 1) - # Get 99.9th percentile daily precipitation - PR = TimeSeriesData(ds, varname) - S = SeasonalAverager( - PR, - sftlf, - dec_mode=dec_mode, - drop_incomplete_djf=drop_incomplete_djf, - annual_strict=annual_strict, - ) - PRq99p9 = xr.Dataset() - PRq99p9["ANN"] = S.annual_stats("q99p9") + # Set up empty dataset + PRq99p9 = xr.zeros_like(ds) + PRq99p9["lat"] = ds["lat"] + PRq99p9["lon"] = ds["lon"] + PRq99p9 = PRq99p9.drop_vars("time") + + PRq99p9["q99p9"] = ds[varname].quantile(0.999,dim="time") PRq99p9 = update_nc_attrs(PRq99p9, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: PRq99p9}, obs_dict={}, region="land", regrid=False ) if fig_file is not None: - PRq99p9["ANN"].mean("time").plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "ANN"])) - plt.title("Mean of annual 99.9th percentile daily precipitation") + PRq99p9["q99p9"].plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + fig_file1 = fig_file.replace("$index", "_".join([index, "q99p9"])) + plt.title("99.9th percentile daily precipitation") ax = plt.gca() ax.set_facecolor(bgclr) plt.savefig(fig_file1) From 29c5df7ef2a92f40a9ba7c02af71a39c144c4508 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 11 Oct 2024 13:44:25 -0700 Subject: [PATCH 61/88] redo quantiles --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 53 +++++++++++++++------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index 436ca31e1..db53cee05 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -1,6 +1,7 @@ import datetime import cftime +import dask import matplotlib.pyplot as plt import numpy as np import xarray as xr @@ -58,7 +59,7 @@ def __init__( def masked_ds(self, ds): # Mask land where 0.5<=sftlf<=1 - #return ds.where(self.sftlf >= 0.5).where(self.sftlf <= 1) + # return ds.where(self.sftlf >= 0.5).where(self.sftlf <= 1) return ds def calc_5day_mean(self): @@ -404,11 +405,22 @@ def update_nc_attrs(ds, dec_mode, drop_incomplete_djf, annual_strict): # Update fill value encoding ds[yvar].encoding["_FillValue"] = None ds[xvar].encoding["_FillValue"] = None - ds.time.encoding["_FillValue"] = None ds[yvarbnds].encoding["_FillValue"] = None ds[xvarbnds].encoding["_FillValue"] = None - ds.time_bnds.encoding["_FillValue"] = None - for season in ["ANN", "DJF", "MAM", "JJA", "SON"]: + if "time" in ds: + ds.time.encoding["_FillValue"] = None + ds.time_bnds.encoding["_FillValue"] = None + for season in [ + "ANN", + "DJF", + "MAM", + "JJA", + "SON", + "ANN5", + "median", + "q99p9", + "q99p0", + ]: if season in ds: ds[season].encoding["_FillValue"] = float(1e20) @@ -546,9 +558,9 @@ def get_tasmax_q50( Tmedian = xr.zeros_like(ds) Tmedian["lat"] = ds["lat"] Tmedian["lon"] = ds["lon"] - Tmedian = Tmedian.drop_vars("time") + Tmedian = Tmedian.drop_vars(["time", "time_bnds", varname], errors="ignore") - Tmedian["median"] = ds[varname].median("time") + Tmedian["q50"] = ds[varname].median("time") Tmedian = update_nc_attrs(Tmedian, dec_mode, drop_incomplete_djf, annual_strict) # Compute statistics @@ -557,7 +569,7 @@ def get_tasmax_q50( ) if fig_file is not None: - Tmedian["median"].plot(cmap="Oranges", cbar_kwargs={"label": "F"}) + Tmedian["q50"].plot(cmap="Oranges", cbar_kwargs={"label": "F"}) fig_file1 = fig_file.replace("$index", "_".join([index, "median"])) plt.title("Time median daily high temperature") ax = plt.gca() @@ -585,9 +597,12 @@ def get_tasmax_q99p9( Tq99p9 = xr.zeros_like(ds) Tq99p9["lat"] = ds["lat"] Tq99p9["lon"] = ds["lon"] - Tq99p9 = Tq99p9.drop_vars("time") + Tq99p9 = Tq99p9.drop_vars(["time", "time_bnds", varname], errors="ignore") - Tq99p9["q99p9"] = ds[varname].quantile(0.999,dim="time") + if isinstance(ds[varname], dask.array.core.Array): + Tq99p9["q99p9"] = ds[varname].chunk({"time": -1}).quantile(0.999, dim="time") + else: + Tq99p9["q99p9"] = ds[varname].quantile(0.999, dim="time") Tq99p9 = update_nc_attrs(Tq99p9, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: Tq99p9}, obs_dict={}, region="land", regrid=False @@ -955,15 +970,15 @@ def get_pr_q50( PRq50 = xr.zeros_like(ds) PRq50["lat"] = ds["lat"] PRq50["lon"] = ds["lon"] - PRq50 = PRq50.drop_vars("time") + PRq50 = PRq50.drop_vars(["time", "time_bnds", varname], errors="ignore") - PRq50["median"] = ds[varname].median("time") + PRq50["q50"] = ds[varname].median("time") PRq50 = update_nc_attrs(PRq50, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json({index: PRq50}, obs_dict={}, region="land", regrid=False) if fig_file is not None: - PRq50["median"].plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) + PRq50["q50"].plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) fig_file1 = fig_file.replace("$index", "_".join([index, "median"])) plt.title("Time median daily precipitation") ax = plt.gca() @@ -993,9 +1008,12 @@ def get_pr_q99p0( PRq99p0 = xr.zeros_like(ds) PRq99p0["lat"] = ds["lat"] PRq99p0["lon"] = ds["lon"] - PRq99p0 = PRq99p0.drop_vars("time") + PRq99p0 = PRq99p0.drop_vars(["time", "time_bnds", varname], errors="ignore") - PRq99p0["q99p0"] = ds[varname].quantile(0.990,dim="time") + if isinstance(ds[varname], dask.array.core.Array): + PRq99p0["q99p0"] = ds[varname].chunk({"time": -1}).quantile(0.990, dim="time") + else: + PRq99p0["q99p0"] = ds[varname].quantile(0.990, dim="time") PRq99p0 = update_nc_attrs(PRq99p0, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: PRq99p0}, obs_dict={}, region="land", regrid=False @@ -1032,9 +1050,12 @@ def get_pr_q99p9( PRq99p9 = xr.zeros_like(ds) PRq99p9["lat"] = ds["lat"] PRq99p9["lon"] = ds["lon"] - PRq99p9 = PRq99p9.drop_vars("time") + PRq99p9 = PRq99p9.drop_vars(["time", "time_bnds", varname], errors="ignore") - PRq99p9["q99p9"] = ds[varname].quantile(0.999,dim="time") + if isinstance(ds[varname], dask.array.core.Array): + PRq99p9["q99p9"] = ds[varname].chunk({"time": -1}).quantile(0.999, dim="time") + else: + PRq99p9["q99p9"] = ds[varname].quantile(0.999, dim="time") PRq99p9 = update_nc_attrs(PRq99p9, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: PRq99p9}, obs_dict={}, region="land", regrid=False From 6c65bd2ebc8f17e4b99a40f84e6a7470ac21f030 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Mon, 28 Oct 2024 13:05:33 -0700 Subject: [PATCH 62/88] update land/sea mask --- pcmdi_metrics/drcdm/lib/compute_metrics.py | 36 ++++++++++------------ 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/compute_metrics.py b/pcmdi_metrics/drcdm/lib/compute_metrics.py index db53cee05..e5e72fbf1 100644 --- a/pcmdi_metrics/drcdm/lib/compute_metrics.py +++ b/pcmdi_metrics/drcdm/lib/compute_metrics.py @@ -1,7 +1,6 @@ import datetime import cftime -import dask import matplotlib.pyplot as plt import numpy as np import xarray as xr @@ -59,7 +58,7 @@ def __init__( def masked_ds(self, ds): # Mask land where 0.5<=sftlf<=1 - # return ds.where(self.sftlf >= 0.5).where(self.sftlf <= 1) + return ds.where(self.sftlf >= 0.5).where(self.sftlf <= 1) return ds def calc_5day_mean(self): @@ -389,7 +388,10 @@ def update_nc_attrs(ds, dec_mode, drop_incomplete_djf, annual_strict): yvarbnds = yvar + "_bnds" ds[yvar].attrs["standard_name"] = "Y" ds[xvar].attrs["standard_name"] = "X" - bnds_dict = {yvar: "Y", xvar: "X", "time": "T"} + if "time" in ds: + bnds_dict = {yvar: "Y", xvar: "X", "time": "T"} + else: + bnds_dict = {yvar: "Y", xvar: "X"} for item in bnds_dict: if "bounds" in ds[item].attrs: bnds_var = ds[item].attrs["bounds"] @@ -570,7 +572,7 @@ def get_tasmax_q50( if fig_file is not None: Tmedian["q50"].plot(cmap="Oranges", cbar_kwargs={"label": "F"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "median"])) + fig_file1 = fig_file.replace("$index", index) plt.title("Time median daily high temperature") ax = plt.gca() ax.set_facecolor(bgclr) @@ -599,10 +601,8 @@ def get_tasmax_q99p9( Tq99p9["lon"] = ds["lon"] Tq99p9 = Tq99p9.drop_vars(["time", "time_bnds", varname], errors="ignore") - if isinstance(ds[varname], dask.array.core.Array): - Tq99p9["q99p9"] = ds[varname].chunk({"time": -1}).quantile(0.999, dim="time") - else: - Tq99p9["q99p9"] = ds[varname].quantile(0.999, dim="time") + # PRISM threw errors if chunk not specified + Tq99p9["q99p9"] = ds[varname].chunk({"time": -1}).quantile(0.999, dim="time") Tq99p9 = update_nc_attrs(Tq99p9, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: Tq99p9}, obs_dict={}, region="land", regrid=False @@ -615,7 +615,7 @@ def get_tasmax_q99p9( if fig_file is not None: Tq99p9["q99p9"].plot(cmap="Oranges", cbar_kwargs={"label": "F"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "q99p9"])) + fig_file1 = fig_file.replace("$index", index) plt.title("99.9th percentile daily high temperature") ax = plt.gca() ax.set_facecolor(bgclr) @@ -979,7 +979,7 @@ def get_pr_q50( if fig_file is not None: PRq50["q50"].plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "median"])) + fig_file1 = fig_file.replace("$index", index) plt.title("Time median daily precipitation") ax = plt.gca() ax.set_facecolor(bgclr) @@ -1010,10 +1010,8 @@ def get_pr_q99p0( PRq99p0["lon"] = ds["lon"] PRq99p0 = PRq99p0.drop_vars(["time", "time_bnds", varname], errors="ignore") - if isinstance(ds[varname], dask.array.core.Array): - PRq99p0["q99p0"] = ds[varname].chunk({"time": -1}).quantile(0.990, dim="time") - else: - PRq99p0["q99p0"] = ds[varname].quantile(0.990, dim="time") + # PRISM threw errors if chunk not specified + PRq99p0["q99p0"] = ds[varname].chunk({"time": -1}).quantile(0.990, dim="time") PRq99p0 = update_nc_attrs(PRq99p0, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: PRq99p0}, obs_dict={}, region="land", regrid=False @@ -1021,7 +1019,7 @@ def get_pr_q99p0( if fig_file is not None: PRq99p0["q99p0"].plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "q99p0"])) + fig_file1 = fig_file.replace("$index", index) plt.title("99.9th percentile daily precipitation") ax = plt.gca() ax.set_facecolor(bgclr) @@ -1052,10 +1050,8 @@ def get_pr_q99p9( PRq99p9["lon"] = ds["lon"] PRq99p9 = PRq99p9.drop_vars(["time", "time_bnds", varname], errors="ignore") - if isinstance(ds[varname], dask.array.core.Array): - PRq99p9["q99p9"] = ds[varname].chunk({"time": -1}).quantile(0.999, dim="time") - else: - PRq99p9["q99p9"] = ds[varname].quantile(0.999, dim="time") + # PRISM threw errors if chunk not specified + PRq99p9["q99p9"] = ds[varname].chunk({"time": -1}).quantile(0.999, dim="time") PRq99p9 = update_nc_attrs(PRq99p9, dec_mode, drop_incomplete_djf, annual_strict) result_dict = metrics_json( {index: PRq99p9}, obs_dict={}, region="land", regrid=False @@ -1063,7 +1059,7 @@ def get_pr_q99p9( if fig_file is not None: PRq99p9["q99p9"].plot(cmap="BuPu", cbar_kwargs={"label": "mm"}) - fig_file1 = fig_file.replace("$index", "_".join([index, "q99p9"])) + fig_file1 = fig_file.replace("$index", index) plt.title("99.9th percentile daily precipitation") ax = plt.gca() ax.set_facecolor(bgclr) From 610e693365cd21d28131afe06104b85f706318ff Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Mon, 28 Oct 2024 13:05:47 -0700 Subject: [PATCH 63/88] land/sea option --- pcmdi_metrics/drcdm/drcdm_driver.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 826232624..0ba8cb010 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -3,6 +3,7 @@ import json import os +import xarray import xcdat from pcmdi_metrics.drcdm.lib import ( @@ -143,6 +144,7 @@ for run in list_of_runs: # Finding land/sea mask sftlf_exists = True + skip_sftlf = False if run == reference_data_set: if reference_sftlf_template is not None and os.path.exists( reference_sftlf_template @@ -151,7 +153,9 @@ else: print("No reference sftlf file template provided.") if not generate_sftlf: - print("Skipping reference data") + print("No land/sea mask applied") + skip_sftlf = True + sftlf_exists = False else: # Set flag to generate sftlf after loading data sftlf_exists = False @@ -169,8 +173,9 @@ except (AttributeError, IndexError): print("No sftlf file found for", model, run) if not generate_sftlf: - print("Skipping realization", run) - continue + print("No land/sea mask applied") + skip_sftlf = True + sftlf_exists = False else: # Set flag to generate sftlf after loading data sftlf_exists = False @@ -248,6 +253,19 @@ shp_path=shp_path, column=col, ) + elif skip_sftlf: + # Make mask with all ones + sftlf = ds.copy(data=None) + sftlf["sftlf"] = xarray.ones_like(ds[varname].isel({"time": 0})) + if use_region_mask: + print("\nCreating region mask for land/sea mask.") + sftlf = region_utilities.mask_region( + sftlf, + region_name, + coords=coords, + shp_path=shp_path, + column=col, + ) # Mask out Antarctica sflat = get_latitude_key(sftlf) @@ -466,7 +484,6 @@ ) metrics_dict["RESULTS"][model][run].update(result_dict) # 99.9 percentile - # Long running, can take 50 min per reals result_dict = compute_metrics.get_pr_q99p9( ds, sftlf, From d66a0f90aee4c4a46b94653f43567a38498be22b Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 17 Dec 2024 12:47:41 -0800 Subject: [PATCH 64/88] update readme --- pcmdi_metrics/drcdm/README.md | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/pcmdi_metrics/drcdm/README.md b/pcmdi_metrics/drcdm/README.md index 694d91f46..f70f3a4b3 100644 --- a/pcmdi_metrics/drcdm/README.md +++ b/pcmdi_metrics/drcdm/README.md @@ -1,5 +1,53 @@ # Decision Relevant Climate Data Metrics +# How to run: +Install the PCMDI Metrics Package. + +Set up a parameter file with your model information. An example parameter file can be found at param/drcdm_param.py. See the Parameters section below for more information. + +Run the decision relevant metrics driver using the following command: +``` +drcdm_driver.py -p your_parameter_file.py +``` + +# Parameters: +| Parameter | Definition | +--------------|------------- +| case_id | (str) Will be appended to the metrics_output_path if present. | +| model_list | (list) List of model names. | +| realization | (list) List of realizations. | +| vars | (list) List of variables: "pr", "tasmax", and/or "tasmin". | +| filename_template | (str) The template for the model file name. May contain placeholders %(variable), %(model), %(model_version), or %(realization) | +| test_data_path | (str) The template for the directory containing the model file. May contain placeholders %(variable), %(model), %(model_version), or %(realization) | +| sftlf_filename_template | (str) The template for the model land/sea mask file. May contain placeholders %(model), %(model_version), or %(realization). Takes precedence over --generate_sftlf | +| generate_sftlf | (bool) If true, generate a land/sea mask on the fly when the model or reference land/sea mask is not found. If false, no land/sea mask is applied. | +| metrics_output_path | (str) The directory to write output files to. | +| plots | (bool) True to save world map figures of mean metrics. | +| nc_out | (bool) True to save netcdf files (required for postprocessing). | +| msyear | (int) Start year for model data set. | +| meyear | (int) End year for model data set. | +| ModUnitsAdjust | (tuple) Provide information for units conversion. Uses format (flag (bool), operation (str), value (float), new units (str)). Operation can be "add", "subtract", "multiply", or "divide". For example, use (True, 'multiply', 86400, 'mm/day') to convert kg/m2/s to mm/day.| +| dec_mode | (str) Toggle how season containing December, January, and February is defined. "DJF" or "JFD". Default "DJF". | +| annual_strict | (bool) This only matters for rolling 5-day metrics. If True, only use data from within a given year in the 5-day means. If False, the rolling mean will include the last 4 days of the prior year. Default False. | +| drop_incomplete_djf | (bool) If True, don't include data from the first January/February and last December in the analysis. Default False. | +| shp_path | (str) path to shapefile. | +| attribute | (str) Attribute used to identify region (eg, column of attribute table). For example, "COUNTRY" in a shapefile of countries. | +| region_name | (str) Unique feature value of the region that occurs in the attribute given by "--attribute". Must match only one geometry in the shapefile. An example is "NORTH_AMERICA" under the attribute "CONTINENTS". | + +# Key information + +The temperature data must be provided in Fahrenheit. The ModUnitsAdjust parameter can be used to convert either Kelvin or Celsius units to Fahrenheit on-the-fly. See this example: + +``` +# Kelvin to Fahrenheit +ModUnitsAdjust = (True, 'KtoF', 0, 'F') + +# Celsius to Fahrenheit +ModUnitsAdjust = (True, 'CtoF', 0, 'F') +``` + +The most efficient way to get postprocessed metrics for multiple regions is to run the drcdm driver without any region subsetting (leave shp_path, attribute, and region_name unset). The regions can be applied during postprocessing. + # How to test: Create a conda environment with pcmdi_metrics and xclim In the PMP root directory use: From 34192c61a19d78d8c0f4974f9c35c8137c38c4e0 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 17 Dec 2024 12:52:52 -0800 Subject: [PATCH 65/88] Update README.md --- pcmdi_metrics/drcdm/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pcmdi_metrics/drcdm/README.md b/pcmdi_metrics/drcdm/README.md index f70f3a4b3..5368d66f6 100644 --- a/pcmdi_metrics/drcdm/README.md +++ b/pcmdi_metrics/drcdm/README.md @@ -10,7 +10,7 @@ Run the decision relevant metrics driver using the following command: drcdm_driver.py -p your_parameter_file.py ``` -# Parameters: +## Parameters: | Parameter | Definition | --------------|------------- | case_id | (str) Will be appended to the metrics_output_path if present. | @@ -34,8 +34,9 @@ drcdm_driver.py -p your_parameter_file.py | attribute | (str) Attribute used to identify region (eg, column of attribute table). For example, "COUNTRY" in a shapefile of countries. | | region_name | (str) Unique feature value of the region that occurs in the attribute given by "--attribute". Must match only one geometry in the shapefile. An example is "NORTH_AMERICA" under the attribute "CONTINENTS". | -# Key information +## Key information +### Units The temperature data must be provided in Fahrenheit. The ModUnitsAdjust parameter can be used to convert either Kelvin or Celsius units to Fahrenheit on-the-fly. See this example: ``` @@ -45,7 +46,9 @@ ModUnitsAdjust = (True, 'KtoF', 0, 'F') # Celsius to Fahrenheit ModUnitsAdjust = (True, 'CtoF', 0, 'F') ``` +Precipitation units must be provided in mm. ModUnitsAdjust can also be used as documented in the Parameters section to convert units such as kg/m2/s to mm. +### Regions The most efficient way to get postprocessed metrics for multiple regions is to run the drcdm driver without any region subsetting (leave shp_path, attribute, and region_name unset). The regions can be applied during postprocessing. # How to test: @@ -56,4 +59,4 @@ In the PMP root directory use: Edit the metrics output path in pcmdi_metrics/drcdm/param/drcdm_param.py to be a location at which you have write permission. To launch a run with the demo parameter file use: -`drcdm_driver.py -p pcmdi_metrics/drcdm/param/drcdm_param.py` \ No newline at end of file +`drcdm_driver.py -p pcmdi_metrics/drcdm/param/drcdm_param.py` From 0ac598b20a3d0a2b38e63ae846c383cc58bdd4b6 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 17 Dec 2024 12:55:59 -0800 Subject: [PATCH 66/88] Update README.md --- pcmdi_metrics/drcdm/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pcmdi_metrics/drcdm/README.md b/pcmdi_metrics/drcdm/README.md index 5368d66f6..15208eab9 100644 --- a/pcmdi_metrics/drcdm/README.md +++ b/pcmdi_metrics/drcdm/README.md @@ -10,6 +10,14 @@ Run the decision relevant metrics driver using the following command: drcdm_driver.py -p your_parameter_file.py ``` +## Inputs +The Decision Relevant Metrics Driver works on daily gridded climate data. This package expects input netcdf files to be cf-compliant and on regular latitude/longitude grids. X and Y dimensions must be named "lon" and "lat", and the time dimension must be named "time". The input variables must be called "tasmax", "tasmin", or "pr". Input files must contain lat, lon, and time bounds. + +## Land/Sea mask +Metrics should only be calculated over land, so users have the option to provide a land/sea mask if the ocean area are not already masked in the input data. If the land/sea mask contains fractional values, land will defined as grid cells where the land area percentage is between 50 and 100. Areas south of 50S will be masked out. + +If available, users should provide the land/sea mask that accompanies their datasets. The mask variable in the land/sea mask file must be called "sftlf". If land/sea masks are not provided, there is an option to generate them on-the-fly using pcmdi_utils. If no mask is provided and --generate-sftlf is set to False, no masking will be done by the PMP. + ## Parameters: | Parameter | Definition | --------------|------------- From 9739687eab74db7020f56d94f48a8972a81fb07b Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 23 Jan 2025 12:03:44 -0800 Subject: [PATCH 67/88] add iqr script --- pcmdi_metrics/drcdm/scripts/iqr.py | 508 +++++++++++++++++++++++++++++ 1 file changed, 508 insertions(+) create mode 100644 pcmdi_metrics/drcdm/scripts/iqr.py diff --git a/pcmdi_metrics/drcdm/scripts/iqr.py b/pcmdi_metrics/drcdm/scripts/iqr.py new file mode 100644 index 000000000..cefe6f311 --- /dev/null +++ b/pcmdi_metrics/drcdm/scripts/iqr.py @@ -0,0 +1,508 @@ +import argparse +import csv +import glob +import json +import os + +import numpy as np +import xarray as xr +import xcdat as xc +from xclim import ensembles + +from pcmdi_metrics.io.region_from_file import region_from_file + + +def relative_diff(ds, obs): + diff = (ds - obs) / obs * 100 + diff = diff.where(~np.isnan(ds)).where(~np.isnan(obs)) + diff = xr.where((diff < 1e200), diff, np.nan) # remove inf + diff25 = diff.quantile(0.25, skipna=True).data.item() + diff50 = diff.quantile(0.5, skipna=True).data.item() + diff75 = diff.quantile(0.75, skipna=True).data.item() + diffs = {"0.25": diff25, "0.5": diff50, "0.75": diff75} + return diffs + + +def abs_diff(ds, obs): + diff = ds - obs + diff = diff.where(~np.isnan(ds)).where(~np.isnan(obs)) + diff = xr.where((diff < 1e200), diff, np.nan) # remove inf + diff25 = diff.quantile(0.25, skipna=True).data.item() + diff50 = diff.quantile(0.5, skipna=True).data.item() + diff75 = diff.quantile(0.75, skipna=True).data.item() + diffs = {"0.25": diff25, "0.5": diff50, "0.75": diff75} + return diffs + + +def filterLatLon(ds_mod, ds_obs, mod_lat, mod_lon, obs_lat, obs_lon): + # Given two datasets with "close" but not exact lat/lon coordinates, return the "matched" lat/lon coordinates + + if len(ds_mod[mod_lat].data) >= len(ds_obs[obs_lat].data): + arr1 = ds_mod[mod_lat].data + arr2 = ds_obs[obs_lat].data + else: + arr1 = ds_obs[obs_lat].data + arr2 = ds_mod[mod_lat].data + + lat = [i for i in arr1 if any((abs(i - arr2) < 0.01))] # tolerance + + if len(ds_mod[mod_lon].data) >= len(ds_obs[obs_lon].data): + arr1 = ds_mod[mod_lon].data + arr2 = ds_obs[obs_lon].data + else: + arr1 = ds_obs[obs_lon].data + arr2 = ds_mod[mod_lon].data + + lon = [i for i in arr1 if any((abs(i - arr2) < 0.01))] # tolerance + + return lat, lon + + +def clean_data(ds_mod, ds_obs, var1): + """Fixes issue where grid coordinates might not match exactly.""" + + try: + if ds_mod.lon.data[0] < 0: # convert to a common lon coordinate (0, 360) + ds_mod["lon"] = ds_mod["lon"] + 360 + mod_lon = "lon" + mod_lat = "lat" + except Exception: + if ds_mod.longitude.data[0] < 0: + ds_mod["longitude"] = ds_mod["longitude"] + 360 + mod_lon = "longitude" + mod_lat = "latitude" + try: + if ds_obs.lon.data[0] < 0: + ds_obs["lon"] = ds_obs["lon"] + 360 + obs_lat = "lat" + obs_lon = "lon" + except Exception: + if ds_obs.longitude.data[0] < 0: + ds_obs["longitude"] = ds_obs["longitude"] + 360 + obs_lat = "latitude" + obs_lon = "longitude" + + ds_obs[var1] = ds_obs[var1].astype(float) + ds_mod[var1] = ds_mod[var1].astype(float) + + ds_obs = ds_obs.where( + ds_obs[var1] is not None, np.nan + ) # NaNs get stored as None in JSON files, switch back. + ds_mod = ds_mod.where(ds_mod[var1] is not None, np.nan) + + lat, lon = filterLatLon( + ds_mod, ds_obs, mod_lat, mod_lon, obs_lat, obs_lon + ) # the a lat/lon coordinate that is common to both datasets + + try: + ds_mod = ds_mod.rename( + {"latitude": "lat", "longitude": "lon"} + ) # common coordinate names + except Exception: + pass + + try: + ds_obs = ds_obs.rename({"latitude": "lat", "longitude": "lon"}) + except Exception: + pass + + """ + In some cases, the model/observation lat/lons would be slightly off. For example, a location in the loca2 dataset may have been (25.00N, 270.00N) and the PRISM dataset the same + grid box would've been (25.0001N, 270.0001N). The code below corrects for those inconsistencies. If we didn't do this xarray would exclude any mismatching coords. + """ + + mod_lat_attrs = ds_mod.lat.attrs # preserve attrs for later + mod_lon_attrs = ds_mod.lon.attrs + + ds_mod = ds_mod.sel(lat=lat, lon=lon, method="nearest") + ds_mod = ds_mod.assign_coords(lat=lat, lon=lon) + ds_mod = ds_mod.reindex(lat=lat, lon=lon, tolerance=1e-5) # + + ds_mod.lat.attrs = mod_lat_attrs # add them back + ds_mod.lon.attrs = mod_lon_attrs + + obs_lat_attrs = ds_obs.lat.attrs # preserve attrs for later + obs_lon_attrs = ds_obs.lon.attrs + + ds_obs = ds_obs.sel(lat=lat, lon=lon, method="nearest") + ds_obs = ds_obs.assign_coords(lat=lat, lon=lon) + ds_obs = ds_obs.reindex( + lat=lat, lon=lon, tolerance=1e-5 + ) # ensures the lat/lon coords are indentical + + ds_obs.lat.attrs = obs_lat_attrs + ds_obs.lon.attrs = obs_lon_attrs + + if len(list(obs_lon_attrs.keys())) == 0: + ds_obs.lon.attrs = mod_lon_attrs # add in the model attributes if the obs didn't originally have any attributes + + return ds_mod, ds_obs + + +def stats_dif(ds, obs, shp): + tmpstats = {} + region_list = [ + "Northeast", + "Northwest", + "Southeast", + "Southwest", + "Northern Great Plains", + "Southern Great Plains", + "Midwest", + ] + + for region in region_list: + tmpstats[region] = {} + ds1 = region_from_file(ds, shp, "NAME", region).compute() + obs1 = region_from_file(obs, shp, "NAME", region).compute() + for varname in ["ANN", "DJF", "MAM", "JJA", "SON", "q50", "q99p9", "q99p0"]: + if varname in obs1.keys(): + print(varname) + ds2, obs2 = clean_data(ds1, obs1, varname) + rg_dict = {} + rg_dict["diff"] = dict( + {"absolute": abs_diff(ds2[varname], obs2[varname])} + ) + rg_dict["diff"]["relative"] = relative_diff(ds2[varname], obs2[varname]) + tmpstats[region][varname] = rg_dict + return tmpstats + + +def fix_ds(ds): + # Fixes some issues in the quantile fields + # that aren't saved with a time axis + # and the NaN encoding seems to have issues + for var in ["time_bnds", "lat_bnds", "lon_bnds"]: + if var in ds: + ds = ds.drop_vars(var) + ds = ds.expand_dims(dim="time") + ds = ds.bounds.add_missing_bounds() + for var in ["q50", "q99p0", "q99p9"]: + if var in ds: + ds = ds.where(ds[var] < 1000) + return ds + + +def get_varnames(ds): + varlist = [] + for item in ["ANN", "DJF", "MAM", "JJA", "SON", "q50", "q99p0", "q99p9"]: + if item in ds: + varlist.append(item) + return varlist + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="iqr.py", description="Create interquartile range table" + ) + parser.add_argument( + "--filename_template", + dest="filename_template", + type=str, + help="Filename template of model data", + ) + parser.add_argument( + "--reference_template", + dest="reference_template", + type=str, + help="Filename template of observation data", + ) + parser.add_argument( + "--shapefile_path", + dest="shapefile_path", + type=str, + help="Path to regions shapefile", + ) + parser.add_argument( + "--output_path", + dest="output_path", + default=".", + type=str, + help="The directory at which to write figure file", + ) + parser.add_argument( + "--obs_name", + dest="obs_name", + type=str, + help="The name of the observation dataset", + ) + parser.add_argument( + "--model_name", + dest="model_name", + type=str, + help="The name of the model dataset", + ) + args = parser.parse_args() + + filename_template = args.filename_template + reference_template = args.reference_template + shapefile_path = args.shapefile_path + output_path = args.output_path + obs_name = args.obs_name + model_name = args.model_name + + flist = { + "pr": [ + "_annual_pxx.nc", + "_annualmean_pr.nc", + "_pr_q50.nc", + "_pr_q99p0.nc", + "_pr_q99p9.nc", + "_seasonalmean_pr.nc", + "_annual_cdd.nc", + "_annual_cwd.nc", + "_wettest_5yr.nc", + ], + "tasmax": [ + "_annual_txx.nc", + "_mean_tasmax.nc", + "_tasmax_q50.nc", + "_tasmax_q99p9.nc", + "_annual_tasmax_ge_86F.nc", + "_annual_tasmax_ge_90F.nc", + "_annual_tasmax_ge_95F.nc", + "_annual_tasmax_ge_100F.nc", + "_annual_tasmax_ge_105F.nc", + "_annual_tasmax_ge_110F.nc", + "_annual_tasmax_ge_115F.nc", + ], + "tasmin": [ + "_annual_tnn.nc", + "_annualmean_tasmin.nc", + "_annual_tasmin_ge_70F.nc", + "_annual_tasmin_ge_75F.nc", + "_annual_tasmin_ge_80F.nc", + "_annual_tasmin_ge_85F.nc", + "_annual_tasmin_ge_90F.nc", + "_annual_tasmin_le_0F.nc", + "_annual_tasmin_le_32F.nc", + ], + } + + # First get all the model-obs differences for each run + # and write to file. + print("\nGenerating regional difference JSONs\n") + for variable in ["pr", "tasmin", "tasmax"]: + for suffix in flist[variable]: + try: + # Open obs + reference_path_tmp = reference_template.replace("%(variable)", variable) + reference_path_tmp = os.path.join( + reference_path_tmp, "netcdf/Reference_None" + suffix + ) + obs = xc.open_dataset(reference_path_tmp) + if "time" not in obs: + obs = fix_ds(obs) + # Open model ensemble + filename_template_tmp = filename_template.replace( + "%(variable)", variable + ) + filelist = os.path.join( + filename_template_tmp, "netcdf/*{0}".format(suffix) + ) + filelist = glob.glob(filelist) + ens = ensembles.create_ensemble(filelist).load() + # Get the model-obs differences + varnames = get_varnames(ens) + print(variable, obs_name, suffix) + for item in varnames: + filename_out = os.path.join( + output_path, + "{0}{1}_{2}_{3}_{4}.json".format( + variable, suffix.split(".")[0], item, model_name, obs_name + ), + ) + if "time" in ens: + ens_mean = ens.mean(["realization", "time"]) + else: + ens_mean = ens.mean(["realization"]) + obs2 = obs.regridder.horizontal( + item, ens_mean, tool="xesmf", method="nearest_s2d" + ) + if "time" in obs2: + obs2 = obs2.mean("time") + if suffix in [ + "_annual_txx.nc", # convert units F to C + "_mean_tasmax.nc", + "_tasmax_q50.nc", + "_tasmax_q99p9.nc", + "_annual_tnn.nc", + "_annualmean_tasmin.nc", + ]: + ens_mean[item] = (ens_mean[item] - 32) * 5 / 9 + obs2[item] = (obs2[item] - 32) * 5 / 9 + tmp = stats_dif(ens_mean, obs2, shapefile_path) + with open(filename_out, "w") as json_out: + json.dump(tmp, json_out, indent=4) + except Exception as e: + print("Could not get stats") + print(e) + + # Once all the differences are written to file, compile into the IQR table + print("\nGenerating IQR CSV\n") + metrics_dict = { + "annualmean_pr (%)": ( + "pr_annualmean_pr_ANN_{0}_{1}.json".format(model_name, obs_name), + "ANN", + ), + "seasonalmean_pr (DJF)(%)": ( + "pr_seasonalmean_pr_DJF_{0}_{1}.json".format(model_name, obs_name), + "DJF", + ), + "seasonalmean_pr (MAM)(%)": ( + "pr_seasonalmean_pr_MAM_{0}_{1}.json".format(model_name, obs_name), + "MAM", + ), + "seasonalmean_pr (JJA)(%)": ( + "pr_seasonalmean_pr_JJA_{0}_{1}.json".format(model_name, obs_name), + "JJA", + ), + "seasonalmean_pr (SON)(%)": ( + "pr_seasonalmean_pr_SON_{0}_{1}.json".format(model_name, obs_name), + "SON", + ), + "pr_q50 (%)": ( + "pr_pr_q50_q50_{0}_{1}.json".format(model_name, obs_name), + "q50", + ), + "pr_q99p9 (%)": ( + "pr_pr_q99p9_q99p9_{0}_{1}.json".format(model_name, obs_name), + "q99p9", + ), + "annual_pxx (%)": ( + "pr_annual_pxx_ANN_{0}_{1}.json".format(model_name, obs_name), + "ANN", + ), + "annualmean_tasmax (C)": ( + "tasmax_mean_tasmax_ANN_{0}_{1}.json".format(model_name, obs_name), + "ANN", + ), + "seasonalmean_tasmax (DJF)(C)": ( + "tasmax_mean_tasmax_DJF_{0}_{1}.json".format(model_name, obs_name), + "DJF", + ), + "seasonalmean_tasmax (MAM)(C)": ( + "tasmax_mean_tasmax_MAM_{0}_{1}.json".format(model_name, obs_name), + "MAM", + ), + "seasonalmean_tasmax (JJA)(C)": ( + "tasmax_mean_tasmax_JJA_{0}_{1}.json".format(model_name, obs_name), + "JJA", + ), + "seasonalmean_tasmax (SON)(C)": ( + "tasmax_mean_tasmax_SON_{0}_{1}.json".format(model_name, obs_name), + "SON", + ), + "tasmax_q50 (C)": ( + "tasmax_tasmax_q50_q50_{0}_{1}.json".format(model_name, obs_name), + "q50", + ), + "tasmax_q99p9 (C)": ( + "tasmax_tasmax_q99p9_q99p9_{0}_{1}.json".format(model_name, obs_name), + "q99p9", + ), + "annual_tasmax_ge_95F (%)": ( + "tasmax_annual_tasmax_ge_95F_ANN_{0}_{1}.json".format(model_name, obs_name), + "ANN", + ), + "annual_tasmax_ge_100F (%)": ( + "tasmax_annual_tasmax_ge_100F_ANN_{0}_{1}.json".format( + model_name, obs_name + ), + "ANN", + ), + "annual_tasmax_ge_105F (%)": ( + "tasmax_annual_tasmax_ge_105F_ANN_{0}_{1}.json".format( + model_name, obs_name + ), + "ANN", + ), + "annual_txx (C)": ( + "tasmax_annual_txx_ANN_{0}_{1}.json".format(model_name, obs_name), + "ANN", + ), + "annualmean_tasmin (C)": ( + "tasmin_annualmean_tasmin_ANN_{0}_{1}.json".format(model_name, obs_name), + "ANN", + ), + "annual_tasmin_le_32F (C)": ( + "tasmin_annual_tasmin_le_32F_ANN_{0}_{1}.json".format(model_name, obs_name), + "ANN", + ), + "annual_tnn (C)": ( + "tasmin_annual_tnn_ANN_{0}_{1}.json".format(model_name, obs_name), + "ANN", + ), + } + regions_list = [ + "Midwest", + "Northeast", + "Northern Great Plains", + "Northwest", + "Southeast", + "Southern Great Plains", + "Southwest", + ] + regions_dict = {} + for item in regions_list: + regions_dict[item + " 25th"] = "" + regions_dict[item + " 50th"] = "" + regions_dict[item + " 75th"] = "" + header_list = [x for x in regions_dict] + table_dict = {} + for item in metrics_dict: + table_dict[item] = regions_dict.copy() + for item in metrics_dict: + filepath = os.path.join(output_path, metrics_dict[item][0]) + with open(filepath, "r") as data_in: + data = json.load(data_in) + for region in regions_list: + if item.endswith("(%)"): + table_dict[item][region + " 25th"] = round( + float( + data[region][metrics_dict[item][1]]["diff"]["relative"]["0.25"] + ), + 2, + ) + table_dict[item][region + " 50th"] = round( + float( + data[region][metrics_dict[item][1]]["diff"]["relative"]["0.5"] + ), + 2, + ) + table_dict[item][region + " 75th"] = round( + float( + data[region][metrics_dict[item][1]]["diff"]["relative"]["0.75"] + ), + 2, + ) + elif item.endswith("(C)"): + table_dict[item][region + " 25th"] = round( + float( + data[region][metrics_dict[item][1]]["diff"]["absolute"]["0.25"] + ), + 2, + ) + table_dict[item][region + " 50th"] = round( + float( + data[region][metrics_dict[item][1]]["diff"]["absolute"]["0.5"] + ), + 2, + ) + table_dict[item][region + " 75th"] = round( + float( + data[region][metrics_dict[item][1]]["diff"]["absolute"]["0.75"] + ), + 2, + ) + csv_file = os.path.join( + output_path, "iqr_table_{0}_{1}.csv".format(model_name, obs_name) + ) + print("Writing", csv_file) + with open(csv_file, "w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=["Metric"] + header_list) + writer.writeheader() + for name, details in table_dict.items(): + row = {"Metric": name} + row.update(details) + writer.writerow(row) From 93c896779f93453a63faeab33d6763aab078de5c Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 23 Jan 2025 14:21:17 -0800 Subject: [PATCH 68/88] update README --- pcmdi_metrics/drcdm/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pcmdi_metrics/drcdm/README.md b/pcmdi_metrics/drcdm/README.md index 15208eab9..c2837045b 100644 --- a/pcmdi_metrics/drcdm/README.md +++ b/pcmdi_metrics/drcdm/README.md @@ -10,6 +10,23 @@ Run the decision relevant metrics driver using the following command: drcdm_driver.py -p your_parameter_file.py ``` +### Interquartile range script + +After running the drcdm_driver over an ensemble for the variables pr, tasmax, and tasmin, users have the optional to produced an interquartile range table using the script scripts/iqr.py. This script is hard-coded to use the NCA5 CONUS regions. + +| Parameter | Definition | +--------------|------------- +| filename_template | (str) The template for the model DRCDM results. May contain placeholder %(variable). A wildcard must be used in place of model name or realization name. | +| reference_template | (str) The template for the reference DRCDM results. May contain placeholder %(variable). A wildcard must be used in place of model name or realization name. | +| output_path | (str) Output directory for iqr.py results (optional). | +| shapefile_path | (str) Full path to the NCA5 regions shapefile. | +| obs_name | (str) Name of the reference data, for example, "PRISM". | +| model_name | (str) Name of the model data, for example, "LOCA2". | + +For example, a user's PMP DRCDM model results are stored at /home/userid/LOCA2. They organized by variable, model, and realization, so the results for GFDL-CM4 r1i1p1f1 precipitation are found at /home/userid/LOCA2/pr/GFDL-CM4/r1i1p1f1. The user's `filename_template` variable would then be formatted as "/home/userid/LOCA2/%(variable)/*/*/". + +This script will write JSON files for ensemble mean statistics (relative and absolute differences between ensemble mean and reference dataset) and a csv file containing the values for the interquartile range table. + ## Inputs The Decision Relevant Metrics Driver works on daily gridded climate data. This package expects input netcdf files to be cf-compliant and on regular latitude/longitude grids. X and Y dimensions must be named "lon" and "lat", and the time dimension must be named "time". The input variables must be called "tasmax", "tasmin", or "pr". Input files must contain lat, lon, and time bounds. From a53c29208b518db5ff50511b796f489e763a641c Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 23 Jan 2025 14:21:50 -0800 Subject: [PATCH 69/88] add header --- pcmdi_metrics/drcdm/scripts/iqr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pcmdi_metrics/drcdm/scripts/iqr.py b/pcmdi_metrics/drcdm/scripts/iqr.py index cefe6f311..aaaace269 100644 --- a/pcmdi_metrics/drcdm/scripts/iqr.py +++ b/pcmdi_metrics/drcdm/scripts/iqr.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import argparse import csv import glob From 092ab319ed83d906df6f6ab895ef92ae9b953f66 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 23 Jan 2025 14:24:22 -0800 Subject: [PATCH 70/88] Update README.md --- pcmdi_metrics/drcdm/README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pcmdi_metrics/drcdm/README.md b/pcmdi_metrics/drcdm/README.md index c2837045b..bc1f38139 100644 --- a/pcmdi_metrics/drcdm/README.md +++ b/pcmdi_metrics/drcdm/README.md @@ -10,23 +10,6 @@ Run the decision relevant metrics driver using the following command: drcdm_driver.py -p your_parameter_file.py ``` -### Interquartile range script - -After running the drcdm_driver over an ensemble for the variables pr, tasmax, and tasmin, users have the optional to produced an interquartile range table using the script scripts/iqr.py. This script is hard-coded to use the NCA5 CONUS regions. - -| Parameter | Definition | ---------------|------------- -| filename_template | (str) The template for the model DRCDM results. May contain placeholder %(variable). A wildcard must be used in place of model name or realization name. | -| reference_template | (str) The template for the reference DRCDM results. May contain placeholder %(variable). A wildcard must be used in place of model name or realization name. | -| output_path | (str) Output directory for iqr.py results (optional). | -| shapefile_path | (str) Full path to the NCA5 regions shapefile. | -| obs_name | (str) Name of the reference data, for example, "PRISM". | -| model_name | (str) Name of the model data, for example, "LOCA2". | - -For example, a user's PMP DRCDM model results are stored at /home/userid/LOCA2. They organized by variable, model, and realization, so the results for GFDL-CM4 r1i1p1f1 precipitation are found at /home/userid/LOCA2/pr/GFDL-CM4/r1i1p1f1. The user's `filename_template` variable would then be formatted as "/home/userid/LOCA2/%(variable)/*/*/". - -This script will write JSON files for ensemble mean statistics (relative and absolute differences between ensemble mean and reference dataset) and a csv file containing the values for the interquartile range table. - ## Inputs The Decision Relevant Metrics Driver works on daily gridded climate data. This package expects input netcdf files to be cf-compliant and on regular latitude/longitude grids. X and Y dimensions must be named "lon" and "lat", and the time dimension must be named "time". The input variables must be called "tasmax", "tasmin", or "pr". Input files must contain lat, lon, and time bounds. @@ -76,6 +59,23 @@ Precipitation units must be provided in mm. ModUnitsAdjust can also be used as d ### Regions The most efficient way to get postprocessed metrics for multiple regions is to run the drcdm driver without any region subsetting (leave shp_path, attribute, and region_name unset). The regions can be applied during postprocessing. +## Interquartile range script + +After running the drcdm_driver over an ensemble for the variables pr, tasmax, and tasmin, users have the optional to produced an interquartile range table using the script scripts/iqr.py. This script is hard-coded to use the NCA5 CONUS regions. + +| Parameter | Definition | +--------------|------------- +| filename_template | (str) The template for the model DRCDM results. May contain placeholder %(variable). A wildcard must be used in place of model name or realization name. | +| reference_template | (str) The template for the reference DRCDM results. May contain placeholder %(variable). A wildcard must be used in place of model name or realization name. | +| output_path | (str) Output directory for iqr.py results (optional). | +| shapefile_path | (str) Full path to the NCA5 regions shapefile. | +| obs_name | (str) Name of the reference data, for example, "PRISM". | +| model_name | (str) Name of the model data, for example, "LOCA2". | + +For example, a user's PMP DRCDM model results are stored at /home/userid/LOCA2. They organized by variable, model, and realization, so the results for GFDL-CM4 r1i1p1f1 precipitation are found at /home/userid/LOCA2/pr/GFDL-CM4/r1i1p1f1. The user's `filename_template` variable would then be formatted as "/home/userid/LOCA2/%(variable)/\*/\*/". + +This script will write JSON files for ensemble mean statistics (relative and absolute differences between ensemble mean and reference dataset) and a csv file containing the values for the interquartile range table. + # How to test: Create a conda environment with pcmdi_metrics and xclim In the PMP root directory use: From 54b29a645708bf372fd2affedacb042898732739 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 4 Feb 2025 09:16:47 -0800 Subject: [PATCH 71/88] update shp --- pcmdi_metrics/drcdm/scripts/iqr.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/scripts/iqr.py b/pcmdi_metrics/drcdm/scripts/iqr.py index aaaace269..a50be5caa 100644 --- a/pcmdi_metrics/drcdm/scripts/iqr.py +++ b/pcmdi_metrics/drcdm/scripts/iqr.py @@ -10,6 +10,7 @@ import xcdat as xc from xclim import ensembles +from pcmdi_metrics import resources from pcmdi_metrics.io.region_from_file import region_from_file @@ -140,7 +141,11 @@ def clean_data(ds_mod, ds_obs, var1): return ds_mod, ds_obs -def stats_dif(ds, obs, shp): +def stats_dif(ds, obs, shp=None): + if shp is None: + egg_pth = resources.resource_path() + shp = os.path.join(egg_pth, "nca5_regions.shp") + tmpstats = {} region_list = [ "Northeast", @@ -213,6 +218,7 @@ def get_varnames(ds): dest="shapefile_path", type=str, help="Path to regions shapefile", + default=None, ) parser.add_argument( "--output_path", From 7886be75b86976a3ea27f786f7492545a58760e7 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 11 Feb 2025 09:34:22 -0800 Subject: [PATCH 72/88] update shapefile --- pcmdi_metrics/drcdm/scripts/iqr.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pcmdi_metrics/drcdm/scripts/iqr.py b/pcmdi_metrics/drcdm/scripts/iqr.py index a50be5caa..435787f71 100644 --- a/pcmdi_metrics/drcdm/scripts/iqr.py +++ b/pcmdi_metrics/drcdm/scripts/iqr.py @@ -141,10 +141,15 @@ def clean_data(ds_mod, ds_obs, var1): return ds_mod, ds_obs -def stats_dif(ds, obs, shp=None): - if shp is None: - egg_pth = resources.resource_path() - shp = os.path.join(egg_pth, "nca5_regions.shp") +def stats_dif(ds, obs): + # To use a different shapefile, users should update variables + # 'shp', 'region_list', and 'name' to match their shapefile + # Search for other instances of 'region_list' to edit in this script. + + # Load the NCA regions shapefile packaged with the PMP + # shp should be the full path to the shapefile + egg_pth = resources.resource_path() + shp = os.path.join(egg_pth, "cb_2023_us_state_500k_ncaregions_wgs84.shp") tmpstats = {} region_list = [ @@ -159,8 +164,9 @@ def stats_dif(ds, obs, shp=None): for region in region_list: tmpstats[region] = {} - ds1 = region_from_file(ds, shp, "NAME", region).compute() - obs1 = region_from_file(obs, shp, "NAME", region).compute() + name = "NCARegion" # attribute name for regions in shp + ds1 = region_from_file(ds, shp, name, region).compute() + obs1 = region_from_file(obs, shp, name, region).compute() for varname in ["ANN", "DJF", "MAM", "JJA", "SON", "q50", "q99p9", "q99p0"]: if varname in obs1.keys(): print(varname) @@ -213,13 +219,6 @@ def get_varnames(ds): type=str, help="Filename template of observation data", ) - parser.add_argument( - "--shapefile_path", - dest="shapefile_path", - type=str, - help="Path to regions shapefile", - default=None, - ) parser.add_argument( "--output_path", dest="output_path", @@ -243,7 +242,6 @@ def get_varnames(ds): filename_template = args.filename_template reference_template = args.reference_template - shapefile_path = args.shapefile_path output_path = args.output_path obs_name = args.obs_name model_name = args.model_name @@ -338,7 +336,8 @@ def get_varnames(ds): ]: ens_mean[item] = (ens_mean[item] - 32) * 5 / 9 obs2[item] = (obs2[item] - 32) * 5 / 9 - tmp = stats_dif(ens_mean, obs2, shapefile_path) + # Regions can be edited in the stats_dif function called below + tmp = stats_dif(ens_mean, obs2) with open(filename_out, "w") as json_out: json.dump(tmp, json_out, indent=4) except Exception as e: @@ -441,6 +440,7 @@ def get_varnames(ds): "ANN", ), } + # If using a different shapefile, edit these regions regions_list = [ "Midwest", "Northeast", From b85fc1387d745a9d3a360370c8a979b621fb6dfc Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 14 Feb 2025 15:26:01 -0800 Subject: [PATCH 73/88] add tests --- tests/test_drcdm.py | 282 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 280 insertions(+), 2 deletions(-) diff --git a/tests/test_drcdm.py b/tests/test_drcdm.py index a8a8219c2..7b5a06c76 100644 --- a/tests/test_drcdm.py +++ b/tests/test_drcdm.py @@ -1,2 +1,280 @@ -def test_pass(): - pass +import numpy as np +import xarray as xr +import xcdat # noqa: F401 + +from pcmdi_metrics.drcdm.lib import compute_metrics + + +def create_random_precip(years, max_val=None, min_val=None): + # Returns array of precip along with sftlf + times = xr.cftime_range( + start="{0}-01-01".format(years[0]), + end="{0}-12-31".format(years[1]), + freq="D", + calendar="noleap", + name="time", + ) + latd = 2 + lond = 2 + + values = np.ones((len(times), latd, lond)) * 20 + np.random.randint( + -10, high=10, size=(len(times), latd, lond) + ) + values = values / 86400 # convert to kg m-2 s-1 + lat = np.arange(0, latd) + lon = np.arange(0, lond) + fake_ds = xr.Dataset( + { + "pr": xr.DataArray( + data=values, # enter data here + dims=["time", "lat", "lon"], + coords={"time": times, "lat": lat, "lon": lon}, + attrs={"_FillValue": -999.9, "units": "kg m-2 s-1"}, + ) + } + ) + + fake_ds["time"].encoding["calendar"] = "noleap" + fake_ds["time"].encoding["units"] = "days since 0000-01-01" + fake_ds = fake_ds.bounds.add_missing_bounds() + + if max_val is not None: + fake_ds["pr"] = fake_ds.pr.where(fake_ds.pr <= max_val, max_val) + if min_val is not None: + fake_ds["pr"] = fake_ds.pr.where(fake_ds.pr >= min_val, min_val) + + sftlf_arr = np.ones((latd, lond)) + sftlf_arr[0, 0] = 0 + sftlf = xr.Dataset( + { + "sftlf": xr.DataArray( + data=sftlf_arr, + dims=["lat", "lon"], + coords={"lat": lat, "lon": lon}, + attrs={"_FillValue": -999.9}, + ) + } + ) + sftlf = sftlf.bounds.add_missing_bounds(["X", "Y"]) + + return fake_ds, sftlf + + +def create_seasonal_precip(season): + # Returns array of precip along with covariate and sftlf + sd = {"DJF": [1, 2, 12], "MAM": [3, 4, 5], "JJA": [6, 7, 8], "SON": [9, 10, 11]} + mos = sd[season] + + years = [1980, 1981] + times = xr.cftime_range( + start="{0}-01-01".format(years[0]), + end="{0}-12-31".format(years[1]), + freq="D", + calendar="noleap", + name="time", + ) + latd = 2 + lond = 2 + + values = np.ones((len(times), latd, lond)) + lat = np.arange(0, latd) + lon = np.arange(0, lond) + fake_ds = xr.Dataset( + { + "pr": xr.DataArray( + data=values, # enter data here + dims=["time", "lat", "lon"], + coords={"time": times, "lat": lat, "lon": lon}, + attrs={"_FillValue": -999.9, "units": "mm"}, + ) + } + ) + fake_ds = fake_ds.where( + ( + (fake_ds["time.month"] == mos[0]) + | (fake_ds["time.month"] == mos[1]) + | (fake_ds["time.month"] == mos[2]) + ), + 0.0, + ) + fake_ds["time"].encoding["calendar"] = "noleap" + fake_ds["time"].encoding["units"] = "days since 0000-01-01" + fake_ds = fake_ds.bounds.add_missing_bounds() + + sftlf_arr = np.ones((latd, lond)) + sftlf_arr[0, 0] = 0 + sftlf = xr.Dataset( + { + "sftlf": xr.DataArray( + data=sftlf_arr, + dims=["lat", "lon"], + coords={"lat": lat, "lon": lon}, + attrs={"_FillValue": -999.9}, + ) + } + ) + sftlf = sftlf.bounds.add_missing_bounds(["X", "Y"]) + + return fake_ds, sftlf + + +def test_seasonal_averager_settings(): + # Testing that the defaults and mask are set + ds, sftlf = create_random_precip([1980, 1981]) + PR = compute_metrics.TimeSeriesData(ds, "pr") + S = compute_metrics.SeasonalAverager(PR, sftlf) + + assert S.dec_mode == "DJF" + assert S.drop_incomplete_djf + assert S.annual_strict + assert S.sftlf.equals(sftlf["sftlf"]) + + +def test_seasonal_averager_ann_max(): + drop_incomplete_djf = True + dec_mode = "DJF" + annual_strict = True + ds, sftlf = create_random_precip([1980, 1981]) + PR = compute_metrics.TimeSeriesData(ds, "pr") + S = compute_metrics.SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + ann_max = S.annual_stats("max", pentad=False) + + assert np.mean(ann_max) == np.mean(ds.groupby("time.year").max(dim="time")) + + +def test_seasonal_averager_ann_min(): + drop_incomplete_djf = True + dec_mode = "DJF" + annual_strict = True + ds, sftlf = create_random_precip([1980, 1981]) + PR = compute_metrics.TimeSeriesData(ds, "pr") + S = compute_metrics.SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + ann_min = S.annual_stats("min", pentad=False) + + assert np.mean(ann_min) == np.mean(ds.groupby("time.year").min(dim="time")) + + +def test_seasonal_averager_ann_mean(): + drop_incomplete_djf = True + dec_mode = "DJF" + annual_strict = True + ds, sftlf = create_random_precip([1980, 1981]) + PR = compute_metrics.TimeSeriesData(ds, "pr") + S = compute_metrics.SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + ann_max = S.annual_stats("mean", pentad=False) + + assert np.mean(ann_max) == np.mean(ds.temporal.group_average("pr", "year")) + + +# Test that drop_incomplete_djf puts nans in correct places +# Test that rolling averages for say a month is matching manual version + + +def test_seasonal_averager_ann_djf(): + drop_incomplete_djf = True + dec_mode = "DJF" + annual_strict = True + ds, sftlf = create_seasonal_precip("DJF") + PR = compute_metrics.TimeSeriesData(ds, "pr") + S = compute_metrics.SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + djf = S.seasonal_stats("DJF", "max", pentad=False) + + assert djf.max().data == 1.0 + assert djf.mean().data == 1.0 + + +def test_seasonal_averager_rolling_mam(): + ds, sftlf = create_seasonal_precip("MAM") + PR = compute_metrics.TimeSeriesData(ds, "pr") + S = compute_metrics.SeasonalAverager(PR, sftlf) + S.calc_5day_mean() + + # Get the MAM mean value of the rolling mean calculated by the seasonal averager + rolling_mean = float( + S.pentad.where(((ds["time.month"] >= 3) & (ds["time.month"] <= 5))).mean() + ) + + # This is what the mean value of the 5-day rolling means should be, if + # MAM are 1 and all other times are 0 + true_mean = ((1 / 5) + (2 / 5) + (3 / 5) + (4 / 5) + 1 * (31 - 4) + 30 + 31) / ( + 31 + 30 + 31 + ) + + assert rolling_mean == true_mean + + +def test_seasonal_averager_rolling_djf(): + drop_incomplete_djf = False + dec_mode = "DJF" + annual_strict = True + ds, sftlf = create_seasonal_precip("DJF") + PR = compute_metrics.TimeSeriesData(ds, "pr") + S = compute_metrics.SeasonalAverager( + PR, + sftlf, + dec_mode=dec_mode, + drop_incomplete_djf=drop_incomplete_djf, + annual_strict=annual_strict, + ) + S.calc_5day_mean() + + # Get the DJF mean value of the rolling mean calculated by the seasonal averager + rolling_mean = float( + S.pentad.where( + ( + (ds["time.month"] == 1) + | (ds["time.month"] == 2) + | (ds["time.month"] == 12) + ) + ).mean() + ) + + # This is what the mean value of the 5-day rolling means should be, if + # DJF are 1 and all other times are 0. Have to slice off 4 days from the first January + # because that is where the time series starts + D = 31 + J = 31 + F = 28 + total_days = D + J + F + true_mean = ( + (J - 4) + + F + + (1 / 5) + + (2 / 5) + + (3 / 5) + + (4 / 5) + + (D - 4) + + J + + F + + (1 / 5) + + (2 / 5) + + (3 / 5) + + (4 / 5) + + (D - 4) + ) / (2 * total_days - 4) + + assert rolling_mean == true_mean From 474037b76c26c2ae8b90501c75a5a90aa312c4bd Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 14 Feb 2025 16:04:49 -0800 Subject: [PATCH 74/88] add xclim --- conda-env/dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda-env/dev.yml b/conda-env/dev.yml index 466fdf7a2..759dc625d 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -31,6 +31,7 @@ dependencies: - nc-time-axis - colorcet - cmocean + - xclim # ================== # Testing # ================== From 87ef19272890c3a289fd6020dc2e4e2d9e899f3a Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Fri, 14 Feb 2025 16:15:50 -0800 Subject: [PATCH 75/88] add xclim --- conda-env/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda-env/ci.yml b/conda-env/ci.yml index 5eede91cc..1aac76a01 100644 --- a/conda-env/ci.yml +++ b/conda-env/ci.yml @@ -31,6 +31,7 @@ dependencies: - nc-time-axis - colorcet - cmocean + - xclim # ================== # Testing # ================== From d0d10e35012bbcb6366b654a3b1b924264d8d3d8 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Mon, 24 Feb 2025 11:15:24 -0800 Subject: [PATCH 76/88] fix date slices --- pcmdi_metrics/drcdm/lib/utilities.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pcmdi_metrics/drcdm/lib/utilities.py b/pcmdi_metrics/drcdm/lib/utilities.py index c57d7cdb4..d2c43e051 100644 --- a/pcmdi_metrics/drcdm/lib/utilities.py +++ b/pcmdi_metrics/drcdm/lib/utilities.py @@ -44,12 +44,12 @@ def fix_calendar(ds): def slice_dataset(ds, start_year, end_year): cal = ds.time.encoding["calendar"] - start_time = cftime.datetime(start_year, 1, 1, calendar=cal) - datetime.timedelta( - days=0 - ) - end_time = cftime.datetime(end_year + 1, 1, 1, calendar=cal) - datetime.timedelta( - days=1 - ) + start_time = cftime.datetime( + start_year, 1, 1, 0, 0, calendar=cal + ) - datetime.timedelta(days=0) + end_time = cftime.datetime( + end_year + 1, 1, 1, 23, 59, 59, calendar=cal + ) - datetime.timedelta(days=1) ds = ds.sel(time=slice(start_time, end_time)) return ds From cae2e0820df0c089e1538a27e1cf48bfe62882ab Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Tue, 25 Feb 2025 09:25:38 -0800 Subject: [PATCH 77/88] fix exclude leap --- pcmdi_metrics/drcdm/drcdm_driver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pcmdi_metrics/drcdm/drcdm_driver.py b/pcmdi_metrics/drcdm/drcdm_driver.py index 0ba8cb010..3cd75b6a7 100644 --- a/pcmdi_metrics/drcdm/drcdm_driver.py +++ b/pcmdi_metrics/drcdm/drcdm_driver.py @@ -285,7 +285,10 @@ yrs = [str(int(ds.time.dt.year[0])), str(int(ds.time.dt.year[-1]))] if ds.time.encoding["calendar"] != "noleap" and exclude_leap: + units = ds.time.encoding["units"] ds = ds.convert_calendar("noleap") + ds.time.encoding["calendar"] = "noleap" + ds.time.encoding["units"] = units ds[varname] = compute_metrics.convert_units(ds[varname], ModUnitsAdjust) From 8609235c850498adacadc8761e1e83d94278db12 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 27 Feb 2025 10:56:38 -0800 Subject: [PATCH 78/88] add dev note --- pcmdi_metrics/drcdm/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pcmdi_metrics/drcdm/README.md b/pcmdi_metrics/drcdm/README.md index bc1f38139..6bf140f55 100644 --- a/pcmdi_metrics/drcdm/README.md +++ b/pcmdi_metrics/drcdm/README.md @@ -85,3 +85,6 @@ Edit the metrics output path in pcmdi_metrics/drcdm/param/drcdm_param.py to be a To launch a run with the demo parameter file use: `drcdm_driver.py -p pcmdi_metrics/drcdm/param/drcdm_param.py` + +# For developers +Some testing and issues have been documented in the original DRCDM PR: https://github.com/PCMDI/pcmdi_metrics/pull/1131 From 43b0139eca036175c70f08b2bd4e6adb0eb397b5 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 27 Feb 2025 11:15:20 -0800 Subject: [PATCH 79/88] add LOCA2 run scripts --- .../param/LOCA2/drcdm_LOCA2_template_pr.py | 24 +++++++++ .../LOCA2/drcdm_LOCA2_template_tasmax.py | 18 +++++++ .../LOCA2/drcdm_LOCA2_template_tasmin.py | 18 +++++++ .../drcdm/param/LOCA2/launch_all_for_model.py | 50 +++++++++++++++++++ .../drcdm/param/LOCA2/pr_template.txt | 13 +++++ .../drcdm/param/LOCA2/tasmax_template.txt | 13 +++++ .../drcdm/param/LOCA2/tasmin_template.txt | 13 +++++ 7 files changed, 149 insertions(+) create mode 100644 pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_pr.py create mode 100644 pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmax.py create mode 100644 pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmin.py create mode 100644 pcmdi_metrics/drcdm/param/LOCA2/launch_all_for_model.py create mode 100644 pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt create mode 100644 pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt create mode 100644 pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt diff --git a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_pr.py b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_pr.py new file mode 100644 index 000000000..547251c8e --- /dev/null +++ b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_pr.py @@ -0,0 +1,24 @@ +# Settings for drcdm driver + +# Note: make sure you know whether the precip version you are using is LOCA2-1 +# or LOCA2-0. I believe 20220519 is LOCA2-1 but confirm before using. +vars = ["pr"] # Choices are 'pr','tasmax', 'tasmin' +test_data_path = "/global/cfs/projectdirs/m3522/cmip6/LOCA2/%(model)/0p0625deg/%(realization)/historical/pr/" +filename_template = "pr.%(model).historical.*.LOCA_16thdeg_v20220519.nc" + + +ModUnitsAdjust = ( + True, + "multiply", + 86400.0, + "mm/day", +) +dec_mode = "JFD" # Use JFD to match Tempest Extremes results +annual_strict = False # This only matters for 5-day values +drop_incomplete_djf = False # False to match Tempest Extremes +regrid = False +plot = True +netcdf = True +generate_sftlf = False +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmax.py b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmax.py new file mode 100644 index 000000000..5a60d3d12 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmax.py @@ -0,0 +1,18 @@ +# Settings for extremes driver + +# These settings are required +vars = ["tasmax"] # Choices are 'pr','tasmax', 'tasmin' +test_data_path = "/global/cfs/projectdirs/m3522/cmip6/LOCA2/%(model)/0p0625deg/%(realization)/historical/tasmax/" +filename_template = "tasmax.%(model).historical.*.LOCA_16thdeg_*.nc" + + +ModUnitsAdjust=(True, 'KtoF', 0, 'F') +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = False +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmin.py b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmin.py new file mode 100644 index 000000000..6566acf49 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmin.py @@ -0,0 +1,18 @@ +# Settings for extremes driver + +# These settings are required +vars = ["tasmin"] # Choices are 'pr','tasmax', 'tasmin' +test_data_path = "/global/cfs/projectdirs/m3522/cmip6/LOCA2/%(model)/0p0625deg/%(realization)/historical/tasmin/" +filename_template = "tasmin.%(model).historical.*.LOCA_16thdeg_*.nc" + + +ModUnitsAdjust=(True, 'KtoF', 0, 'F') +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = False +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/LOCA2/launch_all_for_model.py b/pcmdi_metrics/drcdm/param/LOCA2/launch_all_for_model.py new file mode 100644 index 000000000..1605aa9a4 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/LOCA2/launch_all_for_model.py @@ -0,0 +1,50 @@ +import glob +import os +from string import Template + +# Before running for the first time, create the following subdirectories: +# pr/, tasmax/, and tasmin/ +# Edit model name in L11 and run script to generate job submission scripts for that model. +# Uncomment the os.system() calls to submit jobs. +# To change the job script contents, edit the template files in this directory +# (pr_template.txt, tasmax_template.txt, and tasmin_template.txt) +model = "TaiESM1" +dpath = "/global/cfs/projectdirs/m3522/cmip6/LOCA2/{0}/0p0625deg/".format(model) +reals = glob.glob(os.path.join(dpath,"*")) + +for rpath in reals: + realization = os.path.basename(rpath) + params = { + "mymodel": '"{0}"'.format(model), + "myreal": '"{0}"'.format(realization), + "NR": "{0}".format(realization.split("i")[0].replace("r","")), + "modabbr": model[0:3], + "msyear": 1985 + } + + # PR + with open("/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/pr_template.txt","r") as f: + tmpl = Template(f.read()) + pf = tmpl.substitute(params) + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/pr/run_LOCA2_pr_{0}_{1}.sh".format(model, realization) + with open(outpath,"w") as fout: + fout.write(pf) + #os.system("sbatch {0}".format(outpath)) #uncomment to submit job + + # TASMIN + with open("/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmin_template.txt","r") as f: + tmpl = Template(f.read()) + pf = tmpl.substitute(params) + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmin/run_LOCA2_tasmin_{0}_{1}.sh".format(model, realization) + with open(outpath,"w") as fout: + fout.write(pf) + #os.system("sbatch {0}".format(outpath)) + + # TASMAX + with open("/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmax_template.txt","r") as f: + tmpl = Template(f.read()) + pf = tmpl.substitute(params) + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmax/run_LOCA2_tasmax_{0}_{1}.sh".format(model, realization) + with open(outpath,"w") as fout: + fout.write(pf) + #os.system("sbatch {0}".format(outpath)) \ No newline at end of file diff --git a/pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt b/pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt new file mode 100644 index 000000000..5d476da4e --- /dev/null +++ b/pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=${modabbr}pr${NR} +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_pr.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/pr/${mymodel}/${myreal}/ \ No newline at end of file diff --git a/pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt b/pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt new file mode 100644 index 000000000..50647d1b0 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=${modabbr}tmax${NR} +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_tasmax.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/tasmax/${mymodel}/${myreal}/ \ No newline at end of file diff --git a/pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt b/pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt new file mode 100644 index 000000000..a442b8f0e --- /dev/null +++ b/pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=${modabbr}tmin${NR} +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_tasmin.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/tasmin/${mymodel}/${myreal}/ \ No newline at end of file From e454a8bd98f510795553561c0c633b043d8f2920 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 27 Feb 2025 11:19:50 -0800 Subject: [PATCH 80/88] add STAR run scripts --- .../STAR-ESDM/drcdm_STAR-ESDM_template_pr.py | 18 +++++++ .../drcdm_STAR-ESDM_template_tasmax.py | 18 +++++++ .../drcdm_STAR-ESDM_template_tasmin.py | 18 +++++++ .../param/STAR-ESDM/launch_all_for_model.py | 49 +++++++++++++++++++ .../drcdm/param/STAR-ESDM/pr_template.txt | 13 +++++ .../drcdm/param/STAR-ESDM/tasmax_template.txt | 13 +++++ .../drcdm/param/STAR-ESDM/tasmin_template.txt | 13 +++++ 7 files changed, 142 insertions(+) create mode 100644 pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py create mode 100644 pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py create mode 100644 pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py create mode 100644 pcmdi_metrics/drcdm/param/STAR-ESDM/launch_all_for_model.py create mode 100644 pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt create mode 100644 pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt create mode 100644 pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py new file mode 100644 index 000000000..54e47eca4 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py @@ -0,0 +1,18 @@ +# Settings for extremes driver + +# These settings are required +vars = ["pr"] # Choices are 'pr','tasmax', 'tasmin' +test_data_path = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/%(model)/CMIP/historical/%(realization)/day/%(variable)/v20241130/" +filename_template = "pr_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" + + +# Units are actually mm in data so no conversion +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = False +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py new file mode 100644 index 000000000..a8b25cb96 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py @@ -0,0 +1,18 @@ +# Settings for extremes driver + +# These settings are required +vars = ["tasmax"] # Choices are 'pr','tasmax', 'tasmin' +test_data_path = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/%(model)/CMIP/historical/%(realization)/day/%(variable)/v20241130/" +filename_template = "tasmax_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" + + +ModUnitsAdjust=(True, 'KtoF', 0, 'F') +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = False +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py new file mode 100644 index 000000000..bd411c118 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py @@ -0,0 +1,18 @@ +# Settings for extremes driver + +# These settings are required +vars = ["tasmin"] # Choices are 'pr','tasmax', 'tasmin' +test_data_path = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/%(model)/CMIP/historical/%(realization)/day/%(variable)/v20241130/" +filename_template = "tasmin_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" + + +ModUnitsAdjust=(True, 'KtoF', 0, 'F') +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = False +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/launch_all_for_model.py b/pcmdi_metrics/drcdm/param/STAR-ESDM/launch_all_for_model.py new file mode 100644 index 000000000..0a96b9cc4 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/launch_all_for_model.py @@ -0,0 +1,49 @@ +import glob +import os +from string import Template + +# Before running for the first time, create the following subdirectories: +# pr/, tasmax/, and tasmin/ +# Edit model name in L11 and run script to generate job submission scripts for that model. +# Uncomment the os.system() calls to submit jobs. +# To change the job script contents, edit the template files in this directory +# (pr_template.txt, tasmax_template.txt, and tasmin_template.txt) +model = "ACCESS-CM2" +dpath = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/{0}/CMIP/historical/".format(model) +reals = glob.glob(os.path.join(dpath,"*")) + +for rpath in reals: + realization = os.path.basename(rpath) + params = { + "mymodel": '"{0}"'.format(model), + "myreal": '"{0}"'.format(realization), + "NR": "{0}".format(realization.split("i")[0].replace("r","")), + "modabbr": model[0:3], + } + + # PR + with open("/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/pr_template.txt","r") as f: + tmpl = Template(f.read()) + pf = tmpl.substitute(params) + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/pr/run_STAR-ESDM_pr_{0}_{1}.sh".format(model, realization) + with open(outpath,"w") as fout: + fout.write(pf) + os.system("sbatch {0}".format(outpath)) + + # TASMIN + with open("/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmin_template.txt","r") as f: + tmpl = Template(f.read()) + pf = tmpl.substitute(params) + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmin/run_STAR-ESDM_tasmin_{0}_{1}.sh".format(model, realization) + with open(outpath,"w") as fout: + fout.write(pf) + os.system("sbatch {0}".format(outpath)) + + # TASMAX + with open("/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmax_template.txt","r") as f: + tmpl = Template(f.read()) + pf = tmpl.substitute(params) + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmax/run_STAR-ESDM_tasmax_{0}_{1}.sh".format(model, realization) + with open(outpath,"w") as fout: + fout.write(pf) + os.system("sbatch {0}".format(outpath)) \ No newline at end of file diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt b/pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt new file mode 100644 index 000000000..237ee9e04 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=${modabbr}pr${NR} +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/pr/${mymodel}/${myreal}/ \ No newline at end of file diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt b/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt new file mode 100644 index 000000000..6ef265328 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=${modabbr}tmax${NR} +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/tasmax/${mymodel}/${myreal}/ \ No newline at end of file diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt b/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt new file mode 100644 index 000000000..d41d055b2 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=${modabbr}tmin${NR} +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/tasmin/${mymodel}/${myreal}/ \ No newline at end of file From c7a3045eade7260738d4ecda444e8e7513c9aa80 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 27 Feb 2025 11:22:03 -0800 Subject: [PATCH 81/88] add PRISM run scripts --- .../drcdm/param/PRISM/drcdm_PRISM_pr.py | 20 +++++++++++++++++ .../drcdm/param/PRISM/drcdm_PRISM_tasmax.py | 22 +++++++++++++++++++ .../drcdm/param/PRISM/drcdm_PRISM_tasmin.py | 22 +++++++++++++++++++ .../drcdm/param/PRISM/run_CONUS_obs_pr.sh | 14 ++++++++++++ .../drcdm/param/PRISM/run_CONUS_obs_tasmax.sh | 14 ++++++++++++ .../drcdm/param/PRISM/run_CONUS_obs_tasmin.sh | 14 ++++++++++++ 6 files changed, 106 insertions(+) create mode 100644 pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_pr.py create mode 100644 pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmax.py create mode 100644 pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmin.py create mode 100644 pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh create mode 100644 pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh create mode 100644 pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh diff --git a/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_pr.py b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_pr.py new file mode 100644 index 000000000..b1a9a1080 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_pr.py @@ -0,0 +1,20 @@ +# Settings for extremes driver + +# These settings are required +vars = ["pr"] # Choices are 'pr','tasmax', 'tasmin' +test_data_set = ["Reference"] +realization = ["PRISM-M3"] +test_data_path = "/pscratch/sd/l/lee1043/obs4MIPs/OSU/PRISM-M3/day/pr/gn/latest/" +filename_template = "pr_day_PRISM-M3_PCMDI_gn_*.nc" + +metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/pr/PRISM/" + +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = True +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmax.py b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmax.py new file mode 100644 index 000000000..cdfe68dcb --- /dev/null +++ b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmax.py @@ -0,0 +1,22 @@ +# Settings for extremes driver + +# These settings are required +vars = ["tasmax"] # Choices are 'pr','tasmax', 'tasmin' +test_data_set = ["Reference"] +realization = ["PRISM-M3"] +test_data_path = "/pscratch/sd/l/lee1043/obs4MIPs/OSU/PRISM-M3/day/tasmax/gn/latest/" +filename_template = "tasmax_day_PRISM-M3_PCMDI_gn_*.nc" + +metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmax/PRISM_2025/" + + +ModUnitsAdjust=(True, 'CtoF', 0, 'F') +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = True +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmin.py b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmin.py new file mode 100644 index 000000000..ab20ebddb --- /dev/null +++ b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmin.py @@ -0,0 +1,22 @@ +# Settings for extremes driver + +# These settings are required +vars = ["tasmin"] # Choices are 'pr','tasmax', 'tasmin' +test_data_set = ["Reference"] +realization = ["PRISM-M3"] +test_data_path = "/pscratch/sd/l/lee1043/obs4MIPs/OSU/PRISM-M3/day/tasmin/gn/latest/" +filename_template = "tasmin_day_PRISM-M3_PCMDI_gn_*.nc" + +metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmin/PRISM_2025/" + + +ModUnitsAdjust=(True, 'CtoF', 0, 'F') +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = True +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh new file mode 100644 index 000000000..1b72c5cc5 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=obspr +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/PRISM/drcdm_PRISM_pr.py + diff --git a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh new file mode 100644 index 000000000..7d7e2fa5f --- /dev/null +++ b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=obstas +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/PRISM//drcdm_PRISM_tasmax.py + diff --git a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh new file mode 100644 index 000000000..4288bc8be --- /dev/null +++ b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=obstas +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/PRISM/drcdm_PRISM_tasmin.py + From 26d3176f5ce28736beaebbf250d6c236a4260cb5 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 27 Feb 2025 11:23:21 -0800 Subject: [PATCH 82/88] add nclim run scripts --- .../nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py | 26 +++++++++++++++++++ .../drcdm_nClimGrid_tasmax.py | 22 ++++++++++++++++ .../drcdm_nClimGrid_tasmin.py | 22 ++++++++++++++++ .../nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh | 14 ++++++++++ .../run_CONUS_obs_tasmax.sh | 14 ++++++++++ .../run_CONUS_obs_tasmin.sh | 14 ++++++++++ 6 files changed, 112 insertions(+) create mode 100644 pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py create mode 100644 pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py create mode 100644 pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py create mode 100644 pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh create mode 100644 pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh create mode 100644 pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py new file mode 100644 index 000000000..3bc814b3d --- /dev/null +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py @@ -0,0 +1,26 @@ +# Settings for extremes driver + +# These settings are required +vars = ["pr"] # Choices are 'pr','tasmax', 'tasmin' +test_data_set = ["Reference"] +realization = ["nClimGrid-Daily-1-0"] +test_data_path = "/pscratch/sd/a/aordonez/nclim/pr/" +filename_template = "pr_day_nClimGrid-Daily-1-0_PCMDI_5km_*.nc" + +metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/pr/nClimGrid-Daily-1-0/" + +ModUnitsAdjust = ( + True, + "multiply", + 86400.0, + "mm/day", +) +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = True +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py new file mode 100644 index 000000000..9fce8f6ce --- /dev/null +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py @@ -0,0 +1,22 @@ +# Settings for extremes driver + +# These settings are required +vars = ["tasmax"] # Choices are 'pr','tasmax', 'tasmin' +test_data_set = ["Reference"] +realization = ["nClimGrid-Daily-1-0"] +test_data_path = "/pscratch/sd/a/aordonez/nclim/tasmax/" +filename_template = "tasmax_day_nClimGrid-Daily-1-0_PCMDI_5km_*.nc" + +metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmax/nClimGrid-Daily-1-0/" + + +ModUnitsAdjust=(True, 'KtoF', 0, 'F') +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = True +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py new file mode 100644 index 000000000..45f231c86 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py @@ -0,0 +1,22 @@ +# Settings for extremes driver + +# These settings are required +vars = ["tasmin"] # Choices are 'pr','tasmax', 'tasmin' +test_data_set = ["Reference"] +realization = ["nClimGrid-Daily-1-0"] +test_data_path = "/pscratch/sd/a/aordonez/nclim/tasmin/" +filename_template = "tasmin_day_nClimGrid-Daily-1-0_PCMDI_5km_*.nc" + +metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmin/nClimGrid-Daily-1-0/" + + +ModUnitsAdjust=(True, 'KtoF', 0, 'F') +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = True +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh new file mode 100644 index 000000000..e3c66dd4d --- /dev/null +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=Nclimpr +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py + diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh new file mode 100644 index 000000000..870225ad9 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=Nclimtas +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py + diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh new file mode 100644 index 000000000..78e568c22 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=NCtmin +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py + From 624056a624128910ab769e5b1aeedebbe409fb25 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 27 Feb 2025 11:24:29 -0800 Subject: [PATCH 83/88] add livneh scripts --- .../livneh-unsplit/drcdm_livneh-unsplit_pr.py | 26 +++++++++++++++++++ .../param/livneh-unsplit/run_CONUS_obs_pr.sh | 14 ++++++++++ 2 files changed, 40 insertions(+) create mode 100644 pcmdi_metrics/drcdm/param/livneh-unsplit/drcdm_livneh-unsplit_pr.py create mode 100644 pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh diff --git a/pcmdi_metrics/drcdm/param/livneh-unsplit/drcdm_livneh-unsplit_pr.py b/pcmdi_metrics/drcdm/param/livneh-unsplit/drcdm_livneh-unsplit_pr.py new file mode 100644 index 000000000..5f3519fe9 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/livneh-unsplit/drcdm_livneh-unsplit_pr.py @@ -0,0 +1,26 @@ +# Settings for extremes driver + +# These settings are required +vars = ["pr"] # Choices are 'pr','tasmax', 'tasmin' +test_data_set = ["Reference"] +realization = ["livneh-unsplit"] +test_data_path = "/pscratch/sd/a/aordonez/livneh/" +filename_template = "pr_day_livneh-unsplit-1-0_PCMDI_5km_*.nc" + +metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/pr/livneh-unsplit/" + +ModUnitsAdjust = ( + True, + "multiply", + 86400.0, + "mm/day", +) +dec_mode = "DJF" +annual_strict = False +drop_incomplete_djf = False +regrid = False +plot = True +netcdf = True +generate_sftlf = True +msyear = 1985 +meyear = 2014 diff --git a/pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh b/pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh new file mode 100644 index 000000000..e4472473e --- /dev/null +++ b/pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH -A m2637 +#SBATCH --job-name=livnehpr +#SBATCH --nodes=1 +#SBATCH --time=3:00:00 +#SBATCH --qos=regular +#SBATCH --constraint=cpu + + +source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh +conda activate pmp_drcdm + +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/livneh-unsplit/drcdm_livneh-unsplit_pr.py + From 47ce24b2d71dc0d1878d87431a2d8a10de053d64 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 27 Feb 2025 11:26:04 -0800 Subject: [PATCH 84/88] add readme --- pcmdi_metrics/drcdm/param/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 pcmdi_metrics/drcdm/param/README.md diff --git a/pcmdi_metrics/drcdm/param/README.md b/pcmdi_metrics/drcdm/param/README.md new file mode 100644 index 000000000..35846ff24 --- /dev/null +++ b/pcmdi_metrics/drcdm/param/README.md @@ -0,0 +1 @@ +# DRCDM Parameter files \ No newline at end of file From b899de9cf6d9d5e49e19684218e9c16c4e081fa7 Mon Sep 17 00:00:00 2001 From: Ana Ordonez Date: Thu, 27 Feb 2025 11:37:34 -0800 Subject: [PATCH 85/88] Update README.md --- pcmdi_metrics/drcdm/param/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/drcdm/param/README.md b/pcmdi_metrics/drcdm/param/README.md index 35846ff24..818a95340 100644 --- a/pcmdi_metrics/drcdm/param/README.md +++ b/pcmdi_metrics/drcdm/param/README.md @@ -1 +1,11 @@ -# DRCDM Parameter files \ No newline at end of file +# DRCDM Parameter files + +## Basic parameter file + +Users can look at drcdm_param.py as an example of a basic parameter file for the decision relevant climate data metrics. Take care to consult the documentation for these settings as user choices can change the metrics results. + +## Dataset specific files + +The other directories under param/ contain parameter files and scripts to run the DRCDM driver for specific datasets on NERSC. These scripts are provided as-is with no warantee. These scripts will require edits before they can be run by a new user. Users should verify and change account numbers, conda environment names, and file paths as needed before attempting to run these scripts. They should also review the settings in the .py parameter files and update those as needed. + +LOCA2/ and STAR-ESDM/ contain a script called "launch_all_for_model.py" which can be used to generate job scripts for a single model using the parameter file and batch job templates provided. For the observational datasets, users should manually edit and run the *.sh scripts provided. Many of these scripts contain informational comments and script code should be reviewed before running. From 8393e69ac0c8f8e5f93c92ba15c6a854e2dd8e33 Mon Sep 17 00:00:00 2001 From: Jiwoo Lee Date: Thu, 27 Feb 2025 14:00:19 -0800 Subject: [PATCH 86/88] pre-commit fix --- .../param/LOCA2/drcdm_LOCA2_template_pr.py | 6 +-- .../LOCA2/drcdm_LOCA2_template_tasmax.py | 2 +- .../LOCA2/drcdm_LOCA2_template_tasmin.py | 2 +- .../drcdm/param/LOCA2/launch_all_for_model.py | 54 +++++++++++-------- .../drcdm/param/PRISM/drcdm_PRISM_tasmax.py | 2 +- .../drcdm/param/PRISM/drcdm_PRISM_tasmin.py | 2 +- .../STAR-ESDM/drcdm_STAR-ESDM_template_pr.py | 4 +- .../drcdm_STAR-ESDM_template_tasmax.py | 6 ++- .../drcdm_STAR-ESDM_template_tasmin.py | 6 ++- .../param/STAR-ESDM/launch_all_for_model.py | 52 +++++++++++------- .../nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py | 4 +- .../drcdm_nClimGrid_tasmax.py | 6 ++- .../drcdm_nClimGrid_tasmin.py | 6 ++- 13 files changed, 95 insertions(+), 57 deletions(-) diff --git a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_pr.py b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_pr.py index 547251c8e..af6549055 100644 --- a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_pr.py +++ b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_pr.py @@ -13,9 +13,9 @@ 86400.0, "mm/day", ) -dec_mode = "JFD" # Use JFD to match Tempest Extremes results -annual_strict = False # This only matters for 5-day values -drop_incomplete_djf = False # False to match Tempest Extremes +dec_mode = "JFD" # Use JFD to match Tempest Extremes results +annual_strict = False # This only matters for 5-day values +drop_incomplete_djf = False # False to match Tempest Extremes regrid = False plot = True netcdf = True diff --git a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmax.py b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmax.py index 5a60d3d12..ecc65945f 100644 --- a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmax.py +++ b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmax.py @@ -6,7 +6,7 @@ filename_template = "tasmax.%(model).historical.*.LOCA_16thdeg_*.nc" -ModUnitsAdjust=(True, 'KtoF', 0, 'F') +ModUnitsAdjust = (True, "KtoF", 0, "F") dec_mode = "DJF" annual_strict = False drop_incomplete_djf = False diff --git a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmin.py b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmin.py index 6566acf49..733965e5f 100644 --- a/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmin.py +++ b/pcmdi_metrics/drcdm/param/LOCA2/drcdm_LOCA2_template_tasmin.py @@ -6,7 +6,7 @@ filename_template = "tasmin.%(model).historical.*.LOCA_16thdeg_*.nc" -ModUnitsAdjust=(True, 'KtoF', 0, 'F') +ModUnitsAdjust = (True, "KtoF", 0, "F") dec_mode = "DJF" annual_strict = False drop_incomplete_djf = False diff --git a/pcmdi_metrics/drcdm/param/LOCA2/launch_all_for_model.py b/pcmdi_metrics/drcdm/param/LOCA2/launch_all_for_model.py index 1605aa9a4..997e41201 100644 --- a/pcmdi_metrics/drcdm/param/LOCA2/launch_all_for_model.py +++ b/pcmdi_metrics/drcdm/param/LOCA2/launch_all_for_model.py @@ -4,47 +4,59 @@ # Before running for the first time, create the following subdirectories: # pr/, tasmax/, and tasmin/ -# Edit model name in L11 and run script to generate job submission scripts for that model. +# Edit model name in L11 and run script to generate job submission scripts for that model. # Uncomment the os.system() calls to submit jobs. # To change the job script contents, edit the template files in this directory # (pr_template.txt, tasmax_template.txt, and tasmin_template.txt) model = "TaiESM1" dpath = "/global/cfs/projectdirs/m3522/cmip6/LOCA2/{0}/0p0625deg/".format(model) -reals = glob.glob(os.path.join(dpath,"*")) - +reals = glob.glob(os.path.join(dpath, "*")) + for rpath in reals: realization = os.path.basename(rpath) params = { "mymodel": '"{0}"'.format(model), "myreal": '"{0}"'.format(realization), - "NR": "{0}".format(realization.split("i")[0].replace("r","")), + "NR": "{0}".format(realization.split("i")[0].replace("r", "")), "modabbr": model[0:3], - "msyear": 1985 - } - + "msyear": 1985, + } + # PR - with open("/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/pr_template.txt","r") as f: + with open( + "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/pr_template.txt", "r" + ) as f: tmpl = Template(f.read()) pf = tmpl.substitute(params) - outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/pr/run_LOCA2_pr_{0}_{1}.sh".format(model, realization) - with open(outpath,"w") as fout: + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/pr/run_LOCA2_pr_{0}_{1}.sh".format( + model, realization + ) + with open(outpath, "w") as fout: fout.write(pf) - #os.system("sbatch {0}".format(outpath)) #uncomment to submit job - + # os.system("sbatch {0}".format(outpath)) #uncomment to submit job + # TASMIN - with open("/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmin_template.txt","r") as f: + with open( + "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmin_template.txt", "r" + ) as f: tmpl = Template(f.read()) pf = tmpl.substitute(params) - outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmin/run_LOCA2_tasmin_{0}_{1}.sh".format(model, realization) - with open(outpath,"w") as fout: + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmin/run_LOCA2_tasmin_{0}_{1}.sh".format( + model, realization + ) + with open(outpath, "w") as fout: fout.write(pf) - #os.system("sbatch {0}".format(outpath)) - + # os.system("sbatch {0}".format(outpath)) + # TASMAX - with open("/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmax_template.txt","r") as f: + with open( + "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmax_template.txt", "r" + ) as f: tmpl = Template(f.read()) pf = tmpl.substitute(params) - outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmax/run_LOCA2_tasmax_{0}_{1}.sh".format(model, realization) - with open(outpath,"w") as fout: + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/LOCA2/tasmax/run_LOCA2_tasmax_{0}_{1}.sh".format( + model, realization + ) + with open(outpath, "w") as fout: fout.write(pf) - #os.system("sbatch {0}".format(outpath)) \ No newline at end of file + # os.system("sbatch {0}".format(outpath)) diff --git a/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmax.py b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmax.py index cdfe68dcb..f4232749b 100644 --- a/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmax.py +++ b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmax.py @@ -10,7 +10,7 @@ metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmax/PRISM_2025/" -ModUnitsAdjust=(True, 'CtoF', 0, 'F') +ModUnitsAdjust = (True, "CtoF", 0, "F") dec_mode = "DJF" annual_strict = False drop_incomplete_djf = False diff --git a/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmin.py b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmin.py index ab20ebddb..513f023da 100644 --- a/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmin.py +++ b/pcmdi_metrics/drcdm/param/PRISM/drcdm_PRISM_tasmin.py @@ -10,7 +10,7 @@ metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmin/PRISM_2025/" -ModUnitsAdjust=(True, 'CtoF', 0, 'F') +ModUnitsAdjust = (True, "CtoF", 0, "F") dec_mode = "DJF" annual_strict = False drop_incomplete_djf = False diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py index 54e47eca4..5428d3e7f 100644 --- a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py @@ -3,7 +3,9 @@ # These settings are required vars = ["pr"] # Choices are 'pr','tasmax', 'tasmin' test_data_path = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/%(model)/CMIP/historical/%(realization)/day/%(variable)/v20241130/" -filename_template = "pr_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" +filename_template = ( + "pr_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" +) # Units are actually mm in data so no conversion diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py index a8b25cb96..3c31fc2ec 100644 --- a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py @@ -3,10 +3,12 @@ # These settings are required vars = ["tasmax"] # Choices are 'pr','tasmax', 'tasmin' test_data_path = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/%(model)/CMIP/historical/%(realization)/day/%(variable)/v20241130/" -filename_template = "tasmax_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" +filename_template = ( + "tasmax_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" +) -ModUnitsAdjust=(True, 'KtoF', 0, 'F') +ModUnitsAdjust = (True, "KtoF", 0, "F") dec_mode = "DJF" annual_strict = False drop_incomplete_djf = False diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py index bd411c118..349bdf4e5 100644 --- a/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py @@ -3,10 +3,12 @@ # These settings are required vars = ["tasmin"] # Choices are 'pr','tasmax', 'tasmin' test_data_path = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/%(model)/CMIP/historical/%(realization)/day/%(variable)/v20241130/" -filename_template = "tasmin_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" +filename_template = ( + "tasmin_NAM_CMIP6_%(model)_historical_%(realization)_TTU_STAR-ESDM-V1_day_*.nc" +) -ModUnitsAdjust=(True, 'KtoF', 0, 'F') +ModUnitsAdjust = (True, "KtoF", 0, "F") dec_mode = "DJF" annual_strict = False drop_incomplete_djf = False diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/launch_all_for_model.py b/pcmdi_metrics/drcdm/param/STAR-ESDM/launch_all_for_model.py index 0a96b9cc4..59347693d 100644 --- a/pcmdi_metrics/drcdm/param/STAR-ESDM/launch_all_for_model.py +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/launch_all_for_model.py @@ -4,46 +4,60 @@ # Before running for the first time, create the following subdirectories: # pr/, tasmax/, and tasmin/ -# Edit model name in L11 and run script to generate job submission scripts for that model. +# Edit model name in L11 and run script to generate job submission scripts for that model. # Uncomment the os.system() calls to submit jobs. # To change the job script contents, edit the template files in this directory # (pr_template.txt, tasmax_template.txt, and tasmin_template.txt) model = "ACCESS-CM2" -dpath = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/{0}/CMIP/historical/".format(model) -reals = glob.glob(os.path.join(dpath,"*")) - +dpath = "/global/cfs/projectdirs/m3522/project_downscale/CMIP6/NAM/TTU/STAR-ESDM-V1/{0}/CMIP/historical/".format( + model +) +reals = glob.glob(os.path.join(dpath, "*")) + for rpath in reals: realization = os.path.basename(rpath) params = { "mymodel": '"{0}"'.format(model), "myreal": '"{0}"'.format(realization), - "NR": "{0}".format(realization.split("i")[0].replace("r","")), + "NR": "{0}".format(realization.split("i")[0].replace("r", "")), "modabbr": model[0:3], - } - + } + # PR - with open("/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/pr_template.txt","r") as f: + with open( + "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/pr_template.txt", "r" + ) as f: tmpl = Template(f.read()) pf = tmpl.substitute(params) - outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/pr/run_STAR-ESDM_pr_{0}_{1}.sh".format(model, realization) - with open(outpath,"w") as fout: + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/pr/run_STAR-ESDM_pr_{0}_{1}.sh".format( + model, realization + ) + with open(outpath, "w") as fout: fout.write(pf) os.system("sbatch {0}".format(outpath)) - + # TASMIN - with open("/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmin_template.txt","r") as f: + with open( + "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmin_template.txt", "r" + ) as f: tmpl = Template(f.read()) pf = tmpl.substitute(params) - outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmin/run_STAR-ESDM_tasmin_{0}_{1}.sh".format(model, realization) - with open(outpath,"w") as fout: + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmin/run_STAR-ESDM_tasmin_{0}_{1}.sh".format( + model, realization + ) + with open(outpath, "w") as fout: fout.write(pf) os.system("sbatch {0}".format(outpath)) - + # TASMAX - with open("/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmax_template.txt","r") as f: + with open( + "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmax_template.txt", "r" + ) as f: tmpl = Template(f.read()) pf = tmpl.substitute(params) - outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmax/run_STAR-ESDM_tasmax_{0}_{1}.sh".format(model, realization) - with open(outpath,"w") as fout: + outpath = "/global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/tasmax/run_STAR-ESDM_tasmax_{0}_{1}.sh".format( + model, realization + ) + with open(outpath, "w") as fout: fout.write(pf) - os.system("sbatch {0}".format(outpath)) \ No newline at end of file + os.system("sbatch {0}".format(outpath)) diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py index 3bc814b3d..030f5283d 100644 --- a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py @@ -7,7 +7,9 @@ test_data_path = "/pscratch/sd/a/aordonez/nclim/pr/" filename_template = "pr_day_nClimGrid-Daily-1-0_PCMDI_5km_*.nc" -metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/pr/nClimGrid-Daily-1-0/" +metrics_output_path = ( + "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/pr/nClimGrid-Daily-1-0/" +) ModUnitsAdjust = ( True, diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py index 9fce8f6ce..5b723c8a6 100644 --- a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py @@ -7,10 +7,12 @@ test_data_path = "/pscratch/sd/a/aordonez/nclim/tasmax/" filename_template = "tasmax_day_nClimGrid-Daily-1-0_PCMDI_5km_*.nc" -metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmax/nClimGrid-Daily-1-0/" +metrics_output_path = ( + "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmax/nClimGrid-Daily-1-0/" +) -ModUnitsAdjust=(True, 'KtoF', 0, 'F') +ModUnitsAdjust = (True, "KtoF", 0, "F") dec_mode = "DJF" annual_strict = False drop_incomplete_djf = False diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py index 45f231c86..845de04ac 100644 --- a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py @@ -7,10 +7,12 @@ test_data_path = "/pscratch/sd/a/aordonez/nclim/tasmin/" filename_template = "tasmin_day_nClimGrid-Daily-1-0_PCMDI_5km_*.nc" -metrics_output_path = "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmin/nClimGrid-Daily-1-0/" +metrics_output_path = ( + "/pscratch/sd/a/aordonez/pmp_data/drcdm/obs/tasmin/nClimGrid-Daily-1-0/" +) -ModUnitsAdjust=(True, 'KtoF', 0, 'F') +ModUnitsAdjust = (True, "KtoF", 0, "F") dec_mode = "DJF" annual_strict = False drop_incomplete_djf = False From abcf0f54ee77ae1d8bb77d48681d6e4358e097e2 Mon Sep 17 00:00:00 2001 From: Jiwoo Lee Date: Thu, 27 Feb 2025 14:02:22 -0800 Subject: [PATCH 87/88] pre-commit fix --- pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh | 1 - pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh | 1 - pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh | 1 - pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh | 1 - .../drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh | 1 - .../drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh | 1 - .../drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh | 1 - 7 files changed, 7 deletions(-) diff --git a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh index 1b72c5cc5..31134a89c 100644 --- a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh +++ b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_pr.sh @@ -11,4 +11,3 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/PRISM/drcdm_PRISM_pr.py - diff --git a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh index 7d7e2fa5f..f78bb0288 100644 --- a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh +++ b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmax.sh @@ -11,4 +11,3 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/PRISM//drcdm_PRISM_tasmax.py - diff --git a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh index 4288bc8be..1dda1b0ab 100644 --- a/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh +++ b/pcmdi_metrics/drcdm/param/PRISM/run_CONUS_obs_tasmin.sh @@ -11,4 +11,3 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/PRISM/drcdm_PRISM_tasmin.py - diff --git a/pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh b/pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh index e4472473e..c4da8bc61 100644 --- a/pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh +++ b/pcmdi_metrics/drcdm/param/livneh-unsplit/run_CONUS_obs_pr.sh @@ -11,4 +11,3 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/livneh-unsplit/drcdm_livneh-unsplit_pr.py - diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh index e3c66dd4d..03af426b5 100644 --- a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_pr.sh @@ -11,4 +11,3 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/nClimGrid-Daily-1-0/drcdm_nClimGrid_pr.py - diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh index 870225ad9..fc981b913 100644 --- a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmax.sh @@ -11,4 +11,3 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmax.py - diff --git a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh index 78e568c22..68d3eabfb 100644 --- a/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh +++ b/pcmdi_metrics/drcdm/param/nClimGrid-Daily-1-0/run_CONUS_obs_tasmin.sh @@ -11,4 +11,3 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/obs/nClimGrid-Daily-1-0/drcdm_nClimGrid_tasmin.py - From a9c7e71ed4df1e44d527b91dd28eaeef402b8e5e Mon Sep 17 00:00:00 2001 From: Jiwoo Lee Date: Thu, 27 Feb 2025 14:03:29 -0800 Subject: [PATCH 88/88] pre-commit fix --- pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt | 2 +- pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt | 2 +- pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt | 2 +- pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt | 2 +- pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt | 2 +- pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt b/pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt index 5d476da4e..38b487b4a 100644 --- a/pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt +++ b/pcmdi_metrics/drcdm/param/LOCA2/pr_template.txt @@ -10,4 +10,4 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm -srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_pr.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/pr/${mymodel}/${myreal}/ \ No newline at end of file +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_pr.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/pr/${mymodel}/${myreal}/ diff --git a/pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt b/pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt index 50647d1b0..b0fd94595 100644 --- a/pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt +++ b/pcmdi_metrics/drcdm/param/LOCA2/tasmax_template.txt @@ -10,4 +10,4 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm -srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_tasmax.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/tasmax/${mymodel}/${myreal}/ \ No newline at end of file +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_tasmax.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/tasmax/${mymodel}/${myreal}/ diff --git a/pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt b/pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt index a442b8f0e..24782d8ce 100644 --- a/pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt +++ b/pcmdi_metrics/drcdm/param/LOCA2/tasmin_template.txt @@ -10,4 +10,4 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm -srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_tasmin.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/tasmin/${mymodel}/${myreal}/ \ No newline at end of file +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/LOCA2/drcdm_LOCA2_template_tasmin.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/LOCA2/tasmin/${mymodel}/${myreal}/ diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt b/pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt index 237ee9e04..7814105c6 100644 --- a/pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/pr_template.txt @@ -10,4 +10,4 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm -srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/pr/${mymodel}/${myreal}/ \ No newline at end of file +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_pr.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/pr/${mymodel}/${myreal}/ diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt b/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt index 6ef265328..d86eb89a0 100644 --- a/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmax_template.txt @@ -10,4 +10,4 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm -srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/tasmax/${mymodel}/${myreal}/ \ No newline at end of file +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_tasmax.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/tasmax/${mymodel}/${myreal}/ diff --git a/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt b/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt index d41d055b2..f56d557d1 100644 --- a/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt +++ b/pcmdi_metrics/drcdm/param/STAR-ESDM/tasmin_template.txt @@ -10,4 +10,4 @@ source /global/homes/a/aordonez/miniconda3/etc/profile.d/conda.sh conda activate pmp_drcdm -srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/tasmin/${mymodel}/${myreal}/ \ No newline at end of file +srun time drcdm_driver.py -p /global/homes/a/aordonez/pmp_param/drcdm/STAR-ESDM/drcdm_STAR-ESDM_template_tasmin.py --test_data_set ${mymodel} --case_id ${mymodel} --realization ${myreal} --metrics_output_path /pscratch/sd/a/aordonez/pmp_data/drcdm/downscaled/STAR-ESDM/tasmin/${mymodel}/${myreal}/