From 37b3574ea1f1ec8ece192c8b9767c6f239c37fc7 Mon Sep 17 00:00:00 2001 From: Julian Florez Date: Wed, 28 May 2025 10:11:57 -0600 Subject: [PATCH 01/11] main seed --- rex/renewable_resource.py | 8 ++++---- rex/resource_extraction/resource_extraction.py | 15 +++++++-------- rex/sam_resource.py | 3 ++- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/rex/renewable_resource.py b/rex/renewable_resource.py index e592940ca..10b0050ed 100644 --- a/rex/renewable_resource.py +++ b/rex/renewable_resource.py @@ -1360,7 +1360,7 @@ def get_SAM_df(self, site, height, require_wind_dir=False, icing=False, variables.append('relativehumidity_2m') for var in variables: - var_name = "{}_{}m".format(var, height) + var_name = "{}_{}m".format(var, height) if var != 'relativehumidity_2m' else var ds_slice = (slice(None), site) var_array = self._get_ds(var_name, ds_slice) var_array = SAMResource.roll_timeseries(var_array, time_zone, @@ -1371,9 +1371,9 @@ def get_SAM_df(self, site, height, require_wind_dir=False, icing=False, var, res_df[var], SAMResource.WIND_DATA_RANGES[var], [site]) - col_map = {'pressure': 'Pressure', 'temperature': 'Temperature', - 'windspeed': 'Speed', 'winddirection': 'Direction', - 'relativehumidity_2m': 'Relative Humidity'} + col_map = {'pressure': f'pres_{height}', 'temperature': f'temp_{height}', + 'windspeed': f'speed_{height}', 'winddirection': f'dir_{height}', + 'relativehumidity_2m': 'rhum'} res_df = res_df.rename(columns=col_map) res_df.name = "SAM_{}m-{}".format(height, site) diff --git a/rex/resource_extraction/resource_extraction.py b/rex/resource_extraction/resource_extraction.py index 62d4c46dd..d7f2f17a5 100644 --- a/rex/resource_extraction/resource_extraction.py +++ b/rex/resource_extraction/resource_extraction.py @@ -503,15 +503,14 @@ def _to_SAM_csv(sam_df, site_meta, out_path, write_time=True): col_map[c] = c.capitalize() site_meta = site_meta.rename(columns=col_map) - cols = ','.join(site_meta.columns) - values = site_meta.values[0].astype(str) - values = ','.join([value.replace(',', '') for value in values]) - values = values.replace('\n', '').replace('\r', '').replace('\t', '') + meta_line = ','.join( + f"{col},{value}" for col, value in site_meta.iloc[0].items() + ) with open(out_path, 'r+') as f: content = f.read() f.seek(0, 0) - f.write(cols + '\n' + values + '\n' + content) + f.write(meta_line + '\n' + content) def _init_tree(self, tree): """ @@ -1750,9 +1749,9 @@ def get_SAM_gid(self, hub_height, gid, out_path=None, write_time=True, returned """ kwargs['height'] = hub_height - if out_path is not None: - write_time = False - kwargs.update({'add_header': True}) + #if out_path is not None: + # write_time = False + # kwargs.update({'add_header': True}) SAM_df = super().get_SAM_gid(gid, out_path=out_path, write_time=write_time, diff --git a/rex/sam_resource.py b/rex/sam_resource.py index 99284207a..7bc3296c1 100644 --- a/rex/sam_resource.py +++ b/rex/sam_resource.py @@ -136,7 +136,8 @@ class SAMResource: 'winddirection': (0, 360), 'pressure': (0.5, 1.099), 'temperature': (-200, 100), - 'rh': (0.1, 99.9)} + 'rh': (0.1, 99.9), + 'relativehumidity_2m': (0, 100.0)} # prevent negative wave data; some negative periods are observed on the # west coast along the shore. These are small wave areas and should be fine From f1b19330f26438083254450e489143511e61929e Mon Sep 17 00:00:00 2001 From: Julian Florez Date: Wed, 28 May 2025 10:13:24 -0600 Subject: [PATCH 02/11] remove commented code --- rex/resource_extraction/resource_extraction.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/rex/resource_extraction/resource_extraction.py b/rex/resource_extraction/resource_extraction.py index d7f2f17a5..ff048b0bc 100644 --- a/rex/resource_extraction/resource_extraction.py +++ b/rex/resource_extraction/resource_extraction.py @@ -1749,9 +1749,6 @@ def get_SAM_gid(self, hub_height, gid, out_path=None, write_time=True, returned """ kwargs['height'] = hub_height - #if out_path is not None: - # write_time = False - # kwargs.update({'add_header': True}) SAM_df = super().get_SAM_gid(gid, out_path=out_path, write_time=write_time, From 0438e135c8491d287ab30948d065fb1851b26c7b Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 11:48:51 -0600 Subject: [PATCH 03/11] Update column map --- rex/renewable_resource.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rex/renewable_resource.py b/rex/renewable_resource.py index 10b0050ed..92e02e945 100644 --- a/rex/renewable_resource.py +++ b/rex/renewable_resource.py @@ -1371,9 +1371,11 @@ def get_SAM_df(self, site, height, require_wind_dir=False, icing=False, var, res_df[var], SAMResource.WIND_DATA_RANGES[var], [site]) - col_map = {'pressure': f'pres_{height}', 'temperature': f'temp_{height}', - 'windspeed': f'speed_{height}', 'winddirection': f'dir_{height}', - 'relativehumidity_2m': 'rhum'} + col_map = {'pressure': f'Pressure {height}m', + 'temperature': f'Temperature {height}m', + 'windspeed': f'Wind Speed {height}m', + 'winddirection': f'Wind Direction {height}m', + 'relativehumidity_2m': 'Relative Humidity 2m'} res_df = res_df.rename(columns=col_map) res_df.name = "SAM_{}m-{}".format(height, site) From e9f5f9a35fb8406e75fb6fe2cbcb92465e577ca4 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 11:48:58 -0600 Subject: [PATCH 04/11] Formatting --- rex/renewable_resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rex/renewable_resource.py b/rex/renewable_resource.py index 92e02e945..a8be2bd69 100644 --- a/rex/renewable_resource.py +++ b/rex/renewable_resource.py @@ -1360,7 +1360,8 @@ def get_SAM_df(self, site, height, require_wind_dir=False, icing=False, variables.append('relativehumidity_2m') for var in variables: - var_name = "{}_{}m".format(var, height) if var != 'relativehumidity_2m' else var + var_name = ("{}_{}m".format(var, height) + if var != 'relativehumidity_2m' else var) ds_slice = (slice(None), site) var_array = self._get_ds(var_name, ds_slice) var_array = SAMResource.roll_timeseries(var_array, time_zone, From 6f27adf84ca7194dde2f5057c14ab7665a005cd9 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 12:00:21 -0600 Subject: [PATCH 05/11] Add default option in CSV --- rex/resource_extraction/resource_extraction.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rex/resource_extraction/resource_extraction.py b/rex/resource_extraction/resource_extraction.py index ff048b0bc..40032ca65 100644 --- a/rex/resource_extraction/resource_extraction.py +++ b/rex/resource_extraction/resource_extraction.py @@ -1748,12 +1748,14 @@ def get_SAM_gid(self, hub_height, gid, out_path=None, write_time=True, If multiple lat, lon pairs are given a list of DatFrames is returned """ - kwargs['height'] = hub_height - + # SAM CSV requires wind direction, so leave it on by default but + # allow users to switch off explicitly + get_sam_df_kwargs = {"require_wind_dir": True, "height": hub_height} + get_sam_df_kwargs.update(kwargs) SAM_df = super().get_SAM_gid(gid, out_path=out_path, write_time=write_time, extra_meta_data=extra_meta_data, - **kwargs) + **get_sam_df_kwargs) return SAM_df From 9316ea11e0696995225a0805168268e3af841f0c Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 12:35:38 -0600 Subject: [PATCH 06/11] Add tests for SAM files --- tests/test_resource_extraction.py | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_resource_extraction.py b/tests/test_resource_extraction.py index 187b2848b..7f6eeec67 100644 --- a/tests/test_resource_extraction.py +++ b/tests/test_resource_extraction.py @@ -11,6 +11,9 @@ import pytest from click.testing import CliRunner from pandas.testing import assert_frame_equal +import PySAM.Windpower as PySamWindPower +import PySAM.Pvwattsv8 as PySamPV8 + from rex import TESTDATADIR from rex.resource_extraction.resource_extraction import ( NSRDBX, @@ -895,6 +898,33 @@ def test_windx_make_SAM_files(WindX_cls): LOGGERS.clear() +def test_windx_run_SAM_files(): + """ + Test running WindX files through SAM + """ + + h5_path = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') + with tempfile.TemporaryDirectory() as td: + out_path = os.path.join(td, 'truth.csv') + WindX.make_SAM_files(h5_path, 0, hub_height=80, out_path=out_path) + + obj = PySamWindPower.default('WindPowerNone') + obj.Resource.wind_resource_filename = out_path + obj.Resource.wind_resource_model_choice = 0 + obj.execute() + energy_no_icing = obj.Outputs.annual_energy + assert energy_no_icing > 1e8 + + obj.Losses.env_icing_loss = 1 + obj.Losses.icing_cutoff_rh = 80 + obj.Losses.icing_cutoff_temp = 10 + obj.execute() + assert obj.Outputs.annual_energy < energy_no_icing + + + LOGGERS.clear() + + def test_nsrdbx_make_SAM_files(NSRDBX_cls): """ Test nsrdbx make_SAM_files method @@ -921,6 +951,24 @@ def test_nsrdbx_make_SAM_files(NSRDBX_cls): LOGGERS.clear() +def test_nsrdbx_run_SAM_files(): + """ + Test running nsrdbx files through SAM + """ + + h5_path = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') + with tempfile.TemporaryDirectory() as td: + out_path = os.path.join(td, 'truth.csv') + NSRDBX.make_SAM_files(h5_path, 0, out_path=out_path) + + obj = PySamPV8.default('PVWattsNone') + obj.SolarResource.solar_resource_file = out_path + obj.execute() + assert obj.Outputs.annual_energy > 1.3e8 + + LOGGERS.clear() + + def test_cli_region(runner, WindX_cls): """ Test rex CLI region get From 55a1a337e8ce972d6317b583030d09b8858aec36 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 12:35:52 -0600 Subject: [PATCH 07/11] Split methods --- .../resource_extraction.py | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/rex/resource_extraction/resource_extraction.py b/rex/resource_extraction/resource_extraction.py index 068fb632f..259192e0f 100644 --- a/rex/resource_extraction/resource_extraction.py +++ b/rex/resource_extraction/resource_extraction.py @@ -459,8 +459,7 @@ def _get_ds_slice(dset, gids, ds_ndim): return ds_slice - @staticmethod - def _to_SAM_csv(sam_df, site_meta, out_path, write_time=True): + def _to_SAM_csv(self, sam_df, site_meta, out_path, write_time=True): """ Save SAM dataframe to disk and add meta data to header to make SAM compliant @@ -489,6 +488,10 @@ def _to_SAM_csv(sam_df, site_meta, out_path, write_time=True): cols = [c for c in sam_df if c.lower() not in time_cols] sam_df[cols].to_csv(out_path, index=False) + self._add_sam_csv_header(out_path, site_meta) + + def _add_sam_csv_header(self, out_path, site_meta): + """Add site meta info header to CSV file""" if 'gid' not in site_meta: site_meta.index.name = 'gid' site_meta = site_meta.reset_index() @@ -503,14 +506,15 @@ def _to_SAM_csv(sam_df, site_meta, out_path, write_time=True): col_map[c] = c.capitalize() site_meta = site_meta.rename(columns=col_map) - meta_line = ','.join( - f"{col},{value}" for col, value in site_meta.iloc[0].items() - ) + cols = ','.join(site_meta.columns) + values = site_meta.values[0].astype(str) + values = ','.join([value.replace(',', '') for value in values]) + values = values.replace('\n', '').replace('\r', '').replace('\t', '') with open(out_path, 'r+') as f: content = f.read() f.seek(0, 0) - f.write(meta_line + '\n' + content) + f.write(cols + '\n' + values + '\n' + content) def _init_tree(self, tree): """ @@ -1759,6 +1763,31 @@ def get_SAM_gid(self, gid, hub_height, out_path=None, write_time=True, return SAM_df + def _add_sam_csv_header(self, out_path, site_meta): + """Add site meta info header (like WTK) to CSV file""" + if 'gid' not in site_meta: + site_meta.index.name = 'gid' + site_meta = site_meta.reset_index() + + col_map = {} + for c in site_meta.columns: + if c.lower() == 'timezone': + col_map[c] = 'Time Zone' + elif c.lower() == 'gid': + col_map[c] = 'Location ID' + elif c.islower(): + col_map[c] = c.capitalize() + + site_meta = site_meta.rename(columns=col_map) + meta_line = ','.join( + f"{col},{value}" for col, value in site_meta.iloc[0].items() + ) + + with open(out_path, 'r+') as f: + content = f.read() + f.seek(0, 0) + f.write(meta_line + '\n' + content) + def get_SAM_lat_lon(self, lat_lon, hub_height, check_lat_lon=True, out_path=None, **kwargs): """ From 0a6c8b894e50292854883ba629ce8dbcdbd5c946 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 12:39:37 -0600 Subject: [PATCH 08/11] Fix other tests --- tests/test_resource_hh.py | 2 +- tests/test_resource_invalid.py | 6 +++--- tests/test_sam_resource.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_resource_hh.py b/tests/test_resource_hh.py index dd32ab887..3a868fded 100644 --- a/tests/test_resource_hh.py +++ b/tests/test_resource_hh.py @@ -86,7 +86,7 @@ def test_sam_df_hh(): arr1 = wind['pressure_100m', :, 0] * 9.86923e-6 arr1 = SAMResource.roll_timeseries(arr1, -5, 1) - arr2 = sam_df['Pressure'].values + arr2 = sam_df['Pressure 100m'].values msg1 = ('Error: pressure should have been loaded at 100m ' 'b/c there is only windspeed at 100m.') diff --git a/tests/test_resource_invalid.py b/tests/test_resource_invalid.py index b2d98db3f..b2f8684da 100644 --- a/tests/test_resource_invalid.py +++ b/tests/test_resource_invalid.py @@ -21,7 +21,7 @@ def test_min_pressure(): with WindResource(h5) as wind: og_min = np.min(wind['pressure_100m']) * 9.86923e-6 sam_df = wind.get_SAM_df(site, 100) - patched_min = np.min(sam_df['Pressure'].values) + patched_min = np.min(sam_df['Pressure 100m'].values) msg1 = 'Not a good test set. Min pressure is {}'.format(og_min) msg2 = ('Physical range enforcement failed. ' @@ -43,7 +43,7 @@ def test_min_temp(): with WindResource(h5) as wind: og_min = np.min(wind['temperature_100m']) sam_df = wind.get_SAM_df(site, 100) - patched_min = np.min(sam_df['Temperature'].values) + patched_min = np.min(sam_df['Temperature 100m'].values) msg1 = 'Not a good test set. Min temp is {}'.format(og_min) msg2 = ('Physical range enforcement failed. ' @@ -65,7 +65,7 @@ def test_max_ws(): with WindResource(h5) as wind: og_max = np.max(wind['windspeed_100m']) sam_df = wind.get_SAM_df(site, 100) - patched_max = np.max(sam_df['Speed'].values) + patched_max = np.max(sam_df['Wind Speed 100m'].values) msg1 = 'Not a good test set. Min wind speed is {}'.format(og_max) msg2 = ('Physical range enforcement failed. ' diff --git a/tests/test_sam_resource.py b/tests/test_sam_resource.py index c50f75215..4b32ddfa1 100644 --- a/tests/test_sam_resource.py +++ b/tests/test_sam_resource.py @@ -111,7 +111,7 @@ def test_roll(): if 'Minute' in sam_df: mask &= sam_df['Minute'] == time_index.minute - assert np.isclose(sam_df.loc[mask, 'Speed'], wspd) + assert np.isclose(sam_df.loc[mask, 'Wind Speed 100m'], wspd) def test_roll_timeseries(): From 516a644d3211df98057751ac1cde12b4fed5b30c Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 12:41:34 -0600 Subject: [PATCH 09/11] Use test env --- .github/workflows/h5pyd_tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/h5pyd_tests.yml b/.github/workflows/h5pyd_tests.yml index 942a4b901..193ec0748 100644 --- a/.github/workflows/h5pyd_tests.yml +++ b/.github/workflows/h5pyd_tests.yml @@ -34,9 +34,7 @@ jobs: shell: bash run: | python -m pip install --upgrade pip - pip install pytest - pip install pytest-cov - pip install -e .[hsds] + pip install -e .[test,hsds] - name: Start HSDS and run tests shell: bash From 2f9f287845ca38dc8b50241428a0f5b2dc189841 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 12:41:39 -0600 Subject: [PATCH 10/11] Add test req --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 73863c069..14a4c89de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ test = [ "pytest>=8.3.5,<9", "pytest-timeout>=2.3.1,<3", "flaky>=3.8.1,<4", + "NREL-PySAM>=7.0.0", ] dev = [ "flake8", From 64e449492ef42f2b475805f2847cdcf9c1743ff0 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Fri, 30 May 2025 12:43:53 -0600 Subject: [PATCH 11/11] Match ranges --- rex/sam_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rex/sam_resource.py b/rex/sam_resource.py index 7bc3296c1..dc315bab9 100644 --- a/rex/sam_resource.py +++ b/rex/sam_resource.py @@ -137,7 +137,7 @@ class SAMResource: 'pressure': (0.5, 1.099), 'temperature': (-200, 100), 'rh': (0.1, 99.9), - 'relativehumidity_2m': (0, 100.0)} + 'relativehumidity_2m': (0.1, 99.9)} # prevent negative wave data; some negative periods are observed on the # west coast along the shore. These are small wave areas and should be fine