From 48817082def4dc4c71366cf97fb82a0e16b19da9 Mon Sep 17 00:00:00 2001 From: arbennett Date: Tue, 17 Mar 2020 20:41:22 -0700 Subject: [PATCH 01/36] Add file managers to ensemble products --- pysumma/ensemble.py | 18 +++++++++++++++--- pysumma/simulation.py | 4 +++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/pysumma/ensemble.py b/pysumma/ensemble.py index 472a7501..48ad53b1 100644 --- a/pysumma/ensemble.py +++ b/pysumma/ensemble.py @@ -218,16 +218,26 @@ def attribute_product(list_config): {'attributes': d} for d in product_dict(**list_config)} -def total_product(dec_conf={}, param_conf={}, attr_conf={}): +def file_manager_product(list_config): + return {'++'+'++'.join('{}={}'.format(k, v) for k, v in d.items())+'++': + {'file_manager': d} for d in product_dict(**list_config)} + + +def total_product(dec_conf={}, param_conf={}, attr_conf={}, fman_conf={}, + sequential_keys=False): full_conf = deepcopy(dec_conf) full_conf.update(param_conf) full_conf.update(attr_conf) + full_conf.update(fman_conf) prod_dict = product_dict(**full_conf) config = {} - for d in prod_dict: + for i, d in enumerate(prod_dict): name = '++' + '++'.join( - '{}={}'.format(k, v) if k in param_conf or k in attr_conf else v + '{}={}'.format(k, v) if k in param_conf or k in attr_conf + else v.replace('/', '_').replace('\\', '_') for k, v in d.items()) + '++' + if sequential_keys: + name = f'run_{i}' config[name] = {'decisions': {}, 'parameters': {}, 'attributes': {}} for k, v in d.items(): if k in dec_conf: @@ -236,4 +246,6 @@ def total_product(dec_conf={}, param_conf={}, attr_conf={}): config[name]['parameters'][k] = v elif k in attr_conf: config[name]['attributes'][k] = v + elif k in fman_conf: + config[name]['file_manager'] = v return config diff --git a/pysumma/simulation.py b/pysumma/simulation.py index b509de13..a7b67875 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -6,7 +6,6 @@ import xarray as xr from pathlib import Path from typing import List -from collections.abc import Iterable from .decisions import Decisions from .file_manager import FileManager @@ -227,6 +226,9 @@ def _write_configuration(self, name, write_netcdf: str=False): self.config_path.mkdir(parents=True, exist_ok=True) manager_path = str(self.manager_path.parent) settings_path = str(self.manager['settings_path'].value) + print(settings_path) + print(manager_path) + print(self.config_path) settings_path = Path(settings_path.replace(manager_path, str(self.config_path))) self.manager_path = self.config_path / self.manager.file_name self.manager['settings_path'] = str(settings_path) + os.sep From e03ac97bdc5611899433615f7386f7cac32252b1 Mon Sep 17 00:00:00 2001 From: arbennett Date: Tue, 17 Mar 2020 20:42:58 -0700 Subject: [PATCH 02/36] More options for layer plots --- pysumma/plotting/layers.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pysumma/plotting/layers.py b/pysumma/plotting/layers.py index aecbdc27..ad804441 100644 --- a/pysumma/plotting/layers.py +++ b/pysumma/plotting/layers.py @@ -7,8 +7,9 @@ from .utils import justify -def layers(var, depth, ax=None, colormap='viridis', - plot_soil=False, plot_snow=True, variable_range=None, **kwargs): +def layers(var, depth, ax=None, colormap='viridis', plot_soil=False, + plot_snow=True, variable_range=None, add_colorbar=True, + line_kwargs={}, cbar_kwargs={}): # Preprocess the data vmask = var != -9999 dmask = depth != -9999 @@ -42,7 +43,7 @@ def layers(var, depth, ax=None, colormap='viridis', for l in lo_depth.ifcToto.values[:-1][::-1]: y = lo_depth[l] y[np.isnan(y)] = 0 - ax.vlines(time, ymin=-y, ymax=0, color=rgba[l], **kwargs) + ax.vlines(time, ymin=-y, ymax=0, color=rgba[l], **line_kwargs) # Plot snow layers - plot top down if plot_snow: @@ -50,10 +51,15 @@ def layers(var, depth, ax=None, colormap='viridis', y = hi_depth[l] y[np.isnan(y)] = 0 if (y != 0).any(): - ax.vlines(time, ymin=0, ymax=-y, color=rgba[l], **kwargs) + ax.vlines(time, ymin=0, ymax=-y, color=rgba[l], **line_kwargs) # Add the colorbar mappable = cm.ScalarMappable(norm=norm, cmap=cmap) mappable.set_array(var.values.flatten()) - plt.gcf().colorbar(mappable, label=var.long_name, ax=ax) - return ax + try: + label = var.long_name + except: + label = var.name + if add_colorbar: + plt.gcf().colorbar(mappable, label=label, ax=ax, **cbar_kwargs) + return ax, mappable From 24fbf4ffdd4d0b8c29f429d49e353c52243310e9 Mon Sep 17 00:00:00 2001 From: arbennett Date: Tue, 17 Mar 2020 21:35:45 -0700 Subject: [PATCH 03/36] Add template for distributed runs --- pysumma/distributed.py | 71 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 pysumma/distributed.py diff --git a/pysumma/distributed.py b/pysumma/distributed.py new file mode 100644 index 00000000..161c3c05 --- /dev/null +++ b/pysumma/distributed.py @@ -0,0 +1,71 @@ +from copy import deepcopy +from distributed import Client, get_client +from typing import List +import os +import pandas as pd +import time +import xarray as xr + +from .simulation import Simulation +from .utils import ChainDict, product_dict + +OMP_NUM_THREADS = int(os.environ.get('OMP_NUM_THREADS', 1)) + +class Distributed(object): + ''' + Distributed objects represent SUMMA configurations where + there are multiple GRU/HRU which are expected to be run + in parallel. + + Currently only supports GRU based parallelization. + ''' + + def __init__(self, executable: str, filemanager: str, initialize: bool=True, + num_workers: int=1, threads_per_worker: int=OMP_NUM_THREADS, + chunk_size: int=None, num_chunks: int=None, scheduler: str=None): + """Initialize a new distributed object""" + self._status = 'Initialized' + self.simulation = Simulation(executable, filemanager) + self.submissions: list = [] + self.num_gru = self.count_gru() + # Try to get a client, and if none exists then start a new one + try: + self._client = get_client() + # Start more workers if necessary: + workers = len(self._client.get_worker_logs()) + if workers <= self.num_workers: + self._client.cluster.scale(workers) + except ValueError: + self._client = Client(n_workers=self.num_workers, + threads_per_worker=threads_per_worker) + self.chunks_args = self._gen_args(chunk_size, num_chunks) + + def _gen_args(self, chunk_size: int=None, num_chunks: int=None): + ''' + Generate the arguments that will be used to start multiple + runs from the base ``self.simulation`` + ''' + assert not chunk_size and num_chunks, \ + "Only specify at most one of `chunk_size` or `num_chunks`!" + start, stop = 0, 0 + if chunk_size: + # TODO: FIXME: implement + chunks = [(start, stop)] + elif num_chunks: + # TODO: FIXME: implement + chunks = [(start, stop)] + else: + # TODO: FIXME: implement + chunks = [(start, stop)] + return chunks + + +def _submit(s: Simulation, name: str, run_option: str, + prerun_cmds: List[str], run_args: dict, **kwargs): + s.initialize() + s.apply_config(config) + s.run(run_option, run_suffix=name, prerun_cmds=prerun_cmds, **run_args, **kwargs) + s.process = None + return s + + From 7f62d8abbb494073adc35f2fb8926d2d9d8e4868 Mon Sep 17 00:00:00 2001 From: Andrew Bennett Date: Wed, 18 Mar 2020 14:02:05 -0700 Subject: [PATCH 04/36] Update simulation.py --- pysumma/simulation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysumma/simulation.py b/pysumma/simulation.py index a7b67875..81b915d7 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -226,9 +226,6 @@ def _write_configuration(self, name, write_netcdf: str=False): self.config_path.mkdir(parents=True, exist_ok=True) manager_path = str(self.manager_path.parent) settings_path = str(self.manager['settings_path'].value) - print(settings_path) - print(manager_path) - print(self.config_path) settings_path = Path(settings_path.replace(manager_path, str(self.config_path))) self.manager_path = self.config_path / self.manager.file_name self.manager['settings_path'] = str(settings_path) + os.sep From cf3efde757f9e5dba0fd86d2e85adfd452bcaddd Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 19 Mar 2020 00:16:14 -0700 Subject: [PATCH 05/36] Implement argument generation for distributed --- pysumma/distributed.py | 69 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/pysumma/distributed.py b/pysumma/distributed.py index 161c3c05..0d64645c 100644 --- a/pysumma/distributed.py +++ b/pysumma/distributed.py @@ -4,6 +4,7 @@ import os import pandas as pd import time +import numpy as np import xarray as xr from .simulation import Simulation @@ -20,7 +21,7 @@ class Distributed(object): Currently only supports GRU based parallelization. ''' - def __init__(self, executable: str, filemanager: str, initialize: bool=True, + def __init__(self, executable: str, filemanager: str, num_workers: int=1, threads_per_worker: int=OMP_NUM_THREADS, chunk_size: int=None, num_chunks: int=None, scheduler: str=None): """Initialize a new distributed object""" @@ -45,19 +46,69 @@ def _gen_args(self, chunk_size: int=None, num_chunks: int=None): Generate the arguments that will be used to start multiple runs from the base ``self.simulation`` ''' - assert not chunk_size and num_chunks, \ + assert not (chunk_size and num_chunks), \ "Only specify at most one of `chunk_size` or `num_chunks`!" start, stop = 0, 0 + sim_size = len(self.simulation.local_attributes['gru']) + if not (chunk_size or num_chunks): + chunk_size = 12 if chunk_size: - # TODO: FIXME: implement - chunks = [(start, stop)] + sim_truncated = chunk_size * (sim_size // chunk_size) + starts = np.linspace(1, sim_truncated, chunk_size).astype(int) + stops = np.append(starts[1:] + 1, sim_size) + chunks = np.vstack([starts, stops]).T elif num_chunks: - # TODO: FIXME: implement - chunks = [(start, stop)] + chunk_size = np.ceil(sim_size / num_chunks).astype(int) + starts = np.arange(1, sim_size, chunk_size) + stops = np.append(starts[1:] + 1, sim_size) + chunks = np.vstack([starts, stops]).T + return [{'startGRU': start, 'countGRU': stop - start} + for start, stop in chunks] + + def start(self, run_option: str, prerun_cmds: list=None): + """ + Start running the ensemble members. + + Parameters + ---------- + run_option: + The run type. Should be either 'local' or 'docker' + prerun_cmds: + A list of preprocessing commands to run + """ + for n, s in self.simulations.items(): + # Sleep calls are to ensure writeout happens + config = self.configuration[n] + self.submissions.append(self._client.submit( + _submit, s, n, run_option, prerun_cmds, config)) + + def run(self, run_option: str, prerun_cmds=None, monitor: bool=True): + """ + Run the ensemble + + Parameters + ---------- + run_option: + Where to run the simulation. Can be ``local`` or ``docker`` + prerun_cmds: + A list of shell commands to run before running SUMMA + monitor: + Whether to halt operation until runs are complete + """ + self.start(run_option, prerun_cmds) + if monitor: + return self.monitor() else: - # TODO: FIXME: implement - chunks = [(start, stop)] - return chunks + return True + + def monitor(self): + """ + Halt computation until submitted simulations are complete + """ + simulations = self._client.gather(self.submissions) + for s in simulations: + self.simulations[s.run_suffix] = s + def _submit(s: Simulation, name: str, run_option: str, From 017917d96880f34dc9be895c079a1c8b1b062812 Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 19 Mar 2020 12:58:06 -0700 Subject: [PATCH 06/36] Working prototype of distributed object --- pysumma/__init__.py | 1 + pysumma/distributed.py | 45 ++++++++++++++++++++++++++------------ pysumma/force_file_list.py | 5 ++++- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/pysumma/__init__.py b/pysumma/__init__.py index 221a662b..049410f7 100644 --- a/pysumma/__init__.py +++ b/pysumma/__init__.py @@ -1,5 +1,6 @@ from .simulation import Simulation from .ensemble import Ensemble +from .distributed import Distributed from .file_manager import FileManager from .decisions import Decisions from .output_control import OutputControl diff --git a/pysumma/distributed.py b/pysumma/distributed.py index 0d64645c..b64d9209 100644 --- a/pysumma/distributed.py +++ b/pysumma/distributed.py @@ -1,12 +1,14 @@ from copy import deepcopy from distributed import Client, get_client -from typing import List +from typing import List, Dict import os +from pathlib import Path import pandas as pd import time import numpy as np import xarray as xr +from .file_manager import FileManager from .simulation import Simulation from .utils import ChainDict, product_dict @@ -26,9 +28,13 @@ def __init__(self, executable: str, filemanager: str, chunk_size: int=None, num_chunks: int=None, scheduler: str=None): """Initialize a new distributed object""" self._status = 'Initialized' - self.simulation = Simulation(executable, filemanager) - self.submissions: list = [] - self.num_gru = self.count_gru() + self.executable = executable + self.manager_path = Path(os.path.abspath(filemanager)) + self.manager = FileManager( + self.manager_path.parent, self.manager_path.name) + self.simulations: Dict[str, Simulation] = {} + self.submissions: List = [] + self.num_workers: int = num_workers # Try to get a client, and if none exists then start a new one try: self._client = get_client() @@ -39,9 +45,22 @@ def __init__(self, executable: str, filemanager: str, except ValueError: self._client = Client(n_workers=self.num_workers, threads_per_worker=threads_per_worker) - self.chunks_args = self._gen_args(chunk_size, num_chunks) + self.chunk_args = self._generate_args(chunk_size, num_chunks) + self._generate_simulation_objects() - def _gen_args(self, chunk_size: int=None, num_chunks: int=None): + def _generate_simulation_objects(self): + """ + Create each of the required simulation objects + """ + for argdict in self.chunk_args: + start = argdict['startGRU'] + stop = argdict['startGRU'] + argdict['countGRU'] + name = f"g{start}-{stop}" + self.simulations[name] = Simulation(self.executable, + self.manager_path, + False) + + def _generate_args(self, chunk_size: int=None, num_chunks: int=None): ''' Generate the arguments that will be used to start multiple runs from the base ``self.simulation`` @@ -49,12 +68,12 @@ def _gen_args(self, chunk_size: int=None, num_chunks: int=None): assert not (chunk_size and num_chunks), \ "Only specify at most one of `chunk_size` or `num_chunks`!" start, stop = 0, 0 - sim_size = len(self.simulation.local_attributes['gru']) + sim_size = len(self.manager.local_attributes['gru']) if not (chunk_size or num_chunks): chunk_size = 12 if chunk_size: sim_truncated = chunk_size * (sim_size // chunk_size) - starts = np.linspace(1, sim_truncated, chunk_size).astype(int) + starts = np.arange(1, sim_truncated, chunk_size).astype(int) stops = np.append(starts[1:] + 1, sim_size) chunks = np.vstack([starts, stops]).T elif num_chunks: @@ -65,7 +84,7 @@ def _gen_args(self, chunk_size: int=None, num_chunks: int=None): return [{'startGRU': start, 'countGRU': stop - start} for start, stop in chunks] - def start(self, run_option: str, prerun_cmds: list=None): + def start(self, run_option: str, prerun_cmds: List=None): """ Start running the ensemble members. @@ -76,11 +95,10 @@ def start(self, run_option: str, prerun_cmds: list=None): prerun_cmds: A list of preprocessing commands to run """ - for n, s in self.simulations.items(): - # Sleep calls are to ensure writeout happens - config = self.configuration[n] + for idx, (name, sim) in enumerate(self.simulations.items()): + kwargs = self.chunk_args[idx] self.submissions.append(self._client.submit( - _submit, s, n, run_option, prerun_cmds, config)) + _submit, sim, name, run_option, prerun_cmds, kwargs)) def run(self, run_option: str, prerun_cmds=None, monitor: bool=True): """ @@ -114,7 +132,6 @@ def monitor(self): def _submit(s: Simulation, name: str, run_option: str, prerun_cmds: List[str], run_args: dict, **kwargs): s.initialize() - s.apply_config(config) s.run(run_option, run_suffix=name, prerun_cmds=prerun_cmds, **run_args, **kwargs) s.process = None return s diff --git a/pysumma/force_file_list.py b/pysumma/force_file_list.py index 75cde357..533ad60c 100644 --- a/pysumma/force_file_list.py +++ b/pysumma/force_file_list.py @@ -9,7 +9,7 @@ class ForceFileListOption(BaseOption): def __init__(self, name): super().__init__(name) - self.set_value(xr.open_dataset(name)) + self.set_value(name) def set_value(self, new_value): self.value = new_value @@ -49,3 +49,6 @@ def forcing_paths(self): @property def forcing_data(self): return [o.value for o in self.options] + + def open_forcing_data(self): + return [xr.open_dataset(o) for o in self.forcing_data] From 77dd3a812d82eb6a68032823b629a3220d8526a6 Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 19 Mar 2020 15:03:51 -0700 Subject: [PATCH 07/36] Fix indexing on Distributed --- pysumma/distributed.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pysumma/distributed.py b/pysumma/distributed.py index b64d9209..4906143a 100644 --- a/pysumma/distributed.py +++ b/pysumma/distributed.py @@ -54,7 +54,7 @@ def _generate_simulation_objects(self): """ for argdict in self.chunk_args: start = argdict['startGRU'] - stop = argdict['startGRU'] + argdict['countGRU'] + stop = argdict['startGRU'] + argdict['countGRU'] - 1 name = f"g{start}-{stop}" self.simulations[name] = Simulation(self.executable, self.manager_path, @@ -72,14 +72,14 @@ def _generate_args(self, chunk_size: int=None, num_chunks: int=None): if not (chunk_size or num_chunks): chunk_size = 12 if chunk_size: - sim_truncated = chunk_size * (sim_size // chunk_size) - starts = np.arange(1, sim_truncated, chunk_size).astype(int) - stops = np.append(starts[1:] + 1, sim_size) + sim_truncated = (chunk_size-1) * (sim_size // (chunk_size-1)) + starts = np.arange(1, sim_truncated+1, chunk_size).astype(int) + stops = np.append(starts[1:], sim_size) chunks = np.vstack([starts, stops]).T elif num_chunks: chunk_size = np.ceil(sim_size / num_chunks).astype(int) starts = np.arange(1, sim_size, chunk_size) - stops = np.append(starts[1:] + 1, sim_size) + stops = np.append(starts[1:], sim_size+1) chunks = np.vstack([starts, stops]).T return [{'startGRU': start, 'countGRU': stop - start} for start, stop in chunks] @@ -127,6 +127,8 @@ def monitor(self): for s in simulations: self.simulations[s.run_suffix] = s + def merge_output(self): + pass def _submit(s: Simulation, name: str, run_option: str, From 7dcd207977688b23e7fabfe7cfc4a50b365e87d8 Mon Sep 17 00:00:00 2001 From: Andrew Bennett Date: Thu, 19 Mar 2020 15:29:51 -0700 Subject: [PATCH 08/36] Update setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 1f1c246c..09aedf58 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,6 @@ 'shapely', 'seaborn', 'pandas>=0.25', - 'hs_restclient==1.3.4', 'distributed', 'fiona', 'netcdf4' From 2ec243e3e25c1e8a3cf8fd44693797fd668065a1 Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 19 Mar 2020 17:39:18 -0700 Subject: [PATCH 09/36] update setup.py --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 1f1c246c..5c51abf4 100644 --- a/setup.py +++ b/setup.py @@ -2,10 +2,10 @@ setup(name='pysumma', version='2.0.0', - description='an Object-Oriented Python wrapper for SUMMA model', - url='https://github.com/uva-hydroinformatics/pysumma.git', - author='YoungDon Choi', - author_email='choiyd1115@gmail.com', + description='A python wrapper for SUMMA', + url='https://github.com/UW-Hydro/pysumma.git', + author='YoungDon Choi, Andrew Bennett', + author_email='choiyd1115@gmail.com, andrbenn@uw.edu', license='MIT', packages=find_packages(), install_requires=[ @@ -16,8 +16,8 @@ 'shapely', 'seaborn', 'pandas>=0.25', - 'hs_restclient==1.3.4', 'distributed', + 'cartopy', 'fiona', 'netcdf4' ], From c9e900a989e401b021b955d58c0d94f1df9b1cf9 Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 19 Mar 2020 17:47:04 -0700 Subject: [PATCH 10/36] Add docstrings --- pysumma/ensemble.py | 76 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/pysumma/ensemble.py b/pysumma/ensemble.py index 48ad53b1..07c70e04 100644 --- a/pysumma/ensemble.py +++ b/pysumma/ensemble.py @@ -204,27 +204,103 @@ def _submit(s: Simulation, name: str, run_option: str, prerun_cmds, config): def decision_product(list_config): + """ + Create a dictionary of runs based on a simpler list configuration + of decision options + + Parameters + ---------- + list_config: + A dictionary of the sort + {key1: [list of values], key2: [list of values]} + + Returns + -------- + A dictionary of the sort: + {name: {key1: value1, key2: value1}, + name: {key1: value2, key2: value1}, + ... + name: {key1: valueN, key2: valueN}} + """ return {'++'+'++'.join(d.values())+'++': {'decisions': d} for d in product_dict(**list_config)} def parameter_product(list_config): + """ + Create a dictionary of runs based on a simpler list configuration + of parameter values + + Parameters + ---------- + list_config: + A dictionary of the sort + {key1: [list of values], key2: [list of values]} + + Returns + -------- + A dictionary of the sort: + {name: {key1: value1, key2: value1}, + name: {key1: value2, key2: value1}, + ... + name: {key1: valueN, key2: valueN}} + """ return {'++'+'++'.join('{}={}'.format(k, v) for k, v in d.items())+'++': {'parameters': d} for d in product_dict(**list_config)} def attribute_product(list_config): + """ + Create a dictionary of runs based on a simpler list configuration + of attribute values + + Parameters + ---------- + list_config: + A dictionary of the sort + {key1: [list of values], key2: [list of values]} + + Returns + -------- + A dictionary of the sort: + {name: {key1: value1, key2: value1}, + name: {key1: value2, key2: value1}, + ... + name: {key1: valueN, key2: valueN}} + """ return {'++'+'++'.join('{}={}'.format(k, v) for k, v in d.items())+'++': {'attributes': d} for d in product_dict(**list_config)} def file_manager_product(list_config): + """ + Create a dictionary of runs based on a simpler list configuration + of file managers + + Parameters + ---------- + list_config: + A dictionary of the sort + {key1: [list of values], key2: [list of values]} + + Returns + -------- + A dictionary of the sort: + {name: {key1: value1, key2: value1}, + name: {key1: value2, key2: value1}, + ... + name: {key1: valueN, key2: valueN}} + """ return {'++'+'++'.join('{}={}'.format(k, v) for k, v in d.items())+'++': {'file_manager': d} for d in product_dict(**list_config)} def total_product(dec_conf={}, param_conf={}, attr_conf={}, fman_conf={}, sequential_keys=False): + """ + Combines multiple types of model changes into a single configuration + for the Ensemble object. + """ full_conf = deepcopy(dec_conf) full_conf.update(param_conf) full_conf.update(attr_conf) From eb7cdfa9f91bf3c29f590976c34edba5e05b72ef Mon Sep 17 00:00:00 2001 From: arbennett Date: Fri, 20 Mar 2020 01:33:46 -0600 Subject: [PATCH 11/36] Resolve symlinks through simulation file write --- pysumma/simulation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysumma/simulation.py b/pysumma/simulation.py index b509de13..0414d75e 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -24,7 +24,7 @@ def __init__(self, executable, filemanager, initialize=True): self.stderr = None self.process = None self.executable = executable - self.manager_path = Path(os.path.abspath(filemanager)) + self.manager_path = Path(os.path.abspath(os.path.realpath(filemanager))) self.config_path = self.manager_path.parent / '.pysumma' self.status = 'Uninitialized' if initialize: @@ -226,8 +226,11 @@ def _write_configuration(self, name, write_netcdf: str=False): self.config_path = self.config_path / name self.config_path.mkdir(parents=True, exist_ok=True) manager_path = str(self.manager_path.parent) - settings_path = str(self.manager['settings_path'].value) + settings_path = os.path.abspath(os.path.realpath(str(self.manager['settings_path'].value))) + print(settings_path) settings_path = Path(settings_path.replace(manager_path, str(self.config_path))) + print(manager_path, self.config_path) + print(settings_path) self.manager_path = self.config_path / self.manager.file_name self.manager['settings_path'] = str(settings_path) + os.sep self.manager.write(path=self.config_path) From 5343c039a27f1148eaad106b73cad11ac016aab7 Mon Sep 17 00:00:00 2001 From: arbennett Date: Fri, 20 Mar 2020 11:48:31 -0700 Subject: [PATCH 12/36] Add realpath to distributed --- pysumma/distributed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysumma/distributed.py b/pysumma/distributed.py index d9d2962d..206ddb26 100644 --- a/pysumma/distributed.py +++ b/pysumma/distributed.py @@ -50,7 +50,7 @@ def __init__(self, executable: str, filemanager: str, """ self._status = 'Initialized' self.executable = executable - self.manager_path = Path(os.path.abspath(filemanager)) + self.manager_path = Path(os.path.abspath(os.path.realpath(filemanager))) self.manager = FileManager( self.manager_path.parent, self.manager_path.name) self.simulations: Dict[str, Simulation] = {} From 3b0511d206948b09b1546ed43638ea0bb3086e49 Mon Sep 17 00:00:00 2001 From: Andrew Bennett Date: Fri, 20 Mar 2020 15:59:09 -0700 Subject: [PATCH 13/36] Feature/docs (#101) * Update docs directory * Add readthedocs config --- README.md | 127 ++++++++----------------------- docs/Makefile | 20 +++++ docs/api.rst | 27 +++++++ docs/conf.py | 168 +++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 65 ++++++++++++++++ pysumma/__init__.py | 1 - pysumma/distributed.py | 13 ++++ pysumma/ensemble.py | 13 ++++ pysumma/simulation.py | 132 +++++++++++++++++++++++++++++--- readthedocs.yml | 5 ++ setup.py | 1 - 11 files changed, 465 insertions(+), 107 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/api.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 readthedocs.yml diff --git a/README.md b/README.md index bd7f99b3..ef2d0189 100644 --- a/README.md +++ b/README.md @@ -1,109 +1,46 @@ -[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/DavidChoi76/pysumma_binder_test.git/master) - -# pySUMMA - -The pySUMMA is an Object-Oriented Python wrapper for the manipulation, display and analysis of -SUMMA model (Structure for Unifying Multiple Modeling Alternatives) +pysumma is a Python wrapper for manipulating, running, managing, and analyzing +of SUMMA (Structure for Unifying Multiple Modeling Alternatives) * [SUMMA web site at UCAR ](https://www.rap.ucar.edu/projects/summa) -## The pySUMMA is intended to provide - - - Get and set model parameters and method (fileManager and Decision file) - - Run SUMMA Model - - Visualize netcdf of SUMMA outputs - - Operate pySUMMA with jupyter notebook environment - - Interact Hydorshare to download SUMMA TestCases and post the output of SUMMA - - Automate model calibration or sensitivity analysis (Future work) - -## How to run pySUMMA on HydroShare - (Link YouTube: https://www.youtube.com/watch?v=pL-LNd474Tw) - 1) log in HydoShare(https://www.hydroshare.org) - 2) Start CUAHSI JupyterHub from APPS menu on HydroShare(https://www.hydroshare.org/apps/) - 3) Open "Welcome.ipynb" to download pySUMMA resources from HydroShare - - run the code "1.How to connect with HydroShare" - - change the code from "resid = os.environ['HS_RES_ID']" to "resid = 'c1bb4a12bff44bf08c5958cba7947348'". - 4) You can see the list of Jupyter Notebooks and click one of Jupyter Notebook. - 5) Run one of Jupyter Notebooks. - -### Examples of manipulating and running pySUMMA : - -Refereed paper : Clark, M. P., B. Nijssen, J. D. Lundquist, D. Kavetski, D. E. Rupp, R. A. Woods, -J. E. Freer, E. D. Gutmann, A. W. Wood, D. J. Gochis, R. M. Rasmussen, D. G. Tarboton, V. Mahat, -G. N. Flerchinger, D. G. Marks, 2015b: A unified approach for process-based hydrologic modeling: -Part 2. Model implementation and case studies. Water Resources Research, -[doi:10.1002/2015WR017200](https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1002/2015WR017200). - -#### HydroShare resources -**(Composite Resource)** [Jupyter Notebooks to demonstrate SUMMA Model at Reynolds Mountain East on HydroShare](https://www.hydroshare.org/resource/c1bb4a12bff44bf08c5958cba7947348/) - -**(Composite Resource)** [Procedure and Supplementary documents Collaborative hydrologic modeling on HydroShare](https://www.hydroshare.org/resource/184eea3d3412418a886db87ffdb510b6/) - -**(Model Program)** [SUMMA 2.0.0 Sopron version (lubuntu-16.10)](https://www.hydroshare.org/resource/a5dbd5b198c9468387f59f3fefc11e22/) - -**(Model Program)** [SUMMA 2.0.0 Sopron version (lubuntu-16.04.4)](https://www.hydroshare.org/resource/041671fbc8a544cd8a979af6c2227f92/) - -**(Model Instance)** [Sensitivity to Stomatal Resistance Parameterization of SUMMA Model in Aspen stand at Reynolds Mountain East)](https://www.hydroshare.org/resource/e1a73bc4e7c34166895ff20ae53371f5/) - -**(Model Instance)** [The Impact of Root Distributions Parameters of SUMMA Model in Aspen stand at Reynolds Mountain East)](https://www.hydroshare.org/resource/eed6f3faedad4c17992bb361bd492caa/) - -**(Model Instance)** [The Impact of Lateral Flow Parameterizations on ET of SUMMA Model at Reynolds Mountain East)](https://www.hydroshare.org/resource/11d471b6096d4eaa81068256d281a919/) - -**(Model Instance)** [The Impact of Lateral Flow Parameterizations on Runoff of SUMMA Model at Reynolds Mountain East)](https://www.hydroshare.org/resource/5d20a87ecc5b495097e073e4d5f58d0c/) +pysumma provides methods for: + - Running SUMMA + - Modifying SUMMA input files + - Automatically parallelizing distributed and sensitivity analysis type experiments + - Visualizing output -**(Model Instance)** [The Impact of the canopy shortwave radiation parameterizations of SUMMA Model at Reynolds Mountain East)](https://www.hydroshare.org/resource/0c4fd861a9694b2f9fcdf19eb33a6b54/) - -**(Model Instance)** [The Impact of LAI parameter on the below canopy shortwave radiation of SUMMA Model at Reynolds Mountain East)](https://www.hydroshare.org/resource/2bedc3b88f3547d5b9b0ade7248dfdd5/) - -**(Model Instance)** [The Impact of the canopy wind parameter for the exponential wind profile of SUMMA Model at Reynolds Mountain East)](https://www.hydroshare.org/resource/4064a7b014094f50aa63730e4a3ff976/) - -**(Collection Resource)** [Test Cases of SUMMA modeling that include model instances and Jupyter notebooks for SUMMA 2nd Paper(2015))](https://www.hydroshare.org/resource/1b7a9af74daa4a449190f922b5db366e/) - -## How to run pySUMMA locally - -### Installation and Usage - -#### pySUMMA requires Python 3.6 and following packages : - - - xarray 0.10.7 : N-D labeled arrays and datasets in python - - numpy 1.16.1 : the fundamental package for scientific computing with Python - - matplotlib 3.0.2 : a Python 2D plotting library - - seaborn 0.9.0 : statistical data visualization - - jupyterthemes 0.20.0 : select and install a Jupyter notebook theme - - hs-restclient 1.3.3 : HydroShare REST API python client library - - ipyleaflet 0.9.2 : A jupyter widget for dynamic Leaflet maps - - Linux Environment (VirtualBox 5.2.8) - - [lubuntu-16.10 executable](https://www.hydroshare.org/resource/a5dbd5b198c9468387f59f3fefc11e22/) - - [lubuntu-16.04.4 executable](https://www.hydroshare.org/resource/041671fbc8a544cd8a979af6c2227f92/) - -### Download and Install pySUMMA: - -**1.)** Download pySUMMA -```python -~/Downloads$ git clone https://github.com/uva-hydroinformatics/pysumma.git +# Installation +Currently we only support building pysumma from source. This can be accomplished by +running: ``` - -**2.)** change directory into pysumma folder same level with setup.py. -```python -~/Downloads/pysumma$ pip install . +git clone https://github.com/UW-Hydro/pysumma.git +cd pysumma +python setup.py install ``` -#### The UML of pySUMMA -![Image of UML](UML.png) - -## Reference of SUMMA +If you plan on helping to develop pysumma you may wish to use the following to install +``` +git clone https://github.com/UW-Hydro/pysumma.git +cd pysumma +python setup.py develop +``` +# Additional SUMMA References - [Document](http://summa.readthedocs.io/en/latest/) : SUMMA documentation is available online and remains a work in progress. - [Source Code](https://github.com/NCAR/summa) : NCAR github - -## Bugs - Our issue tracker is at https://github.com/uva-hydroinformatics/pysumma/issues. + +# Bugs + Our issue tracker is at https://github.com/UW-Hydro/pysumma/issues. Please report any bugs that you find. Or, even better, fork the repository on GitHub and create a pull request. All changes are welcome, big or small, and we will help you make the pull request if you are new to git (just ask on the issue). -## License - Distributed with a MIT license; see LICENSE.txt:: - - Copyright (C) 2017 pySUMMA Developers - YoungDon Choi +# How to run pySUMMA on HydroShare + (Link YouTube: https://www.youtube.com/watch?v=pL-LNd474Tw) + 1) log in HydoShare(https://www.hydroshare.org) + 2) Start CUAHSI JupyterHub from APPS menu on HydroShare(https://www.hydroshare.org/apps/) + 3) Open "Welcome.ipynb" to download pySUMMA resources from HydroShare + - run the code "1.How to connect with HydroShare" + - change the code from "resid = os.environ['HS_RES_ID']" to "resid = 'c1bb4a12bff44bf08c5958cba7947348'". + 4) You can see the list of Jupyter Notebooks and click one of Jupyter Notebook. + 5) Run one of Jupyter Notebooks. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..14fa8f27 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = pysumma +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..91bdbc3a --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,27 @@ +.. currentmodule:: pysumma +.. _api: + +############# +API reference +############# + +This page provides an auto-generated summary of pysumma's API. For more details +and examples, refer to the main documentation. + +Simulation +========== +.. autoclass:: pysumma.Simulation + :members: + +Ensemble +======= + +.. autoclass:: pysumma.Ensemble + :members: + +Distributed +======= + +.. autoclass:: pysumma.Distributed + :members: + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..1101efce --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +import sphinx_rtd_theme + +sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('../')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.extlinks', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', + 'sphinx.ext.autosummary'] + +extlinks = {'issue': ('https://github.com/UW-Hydro/pysumma/issues/%s', 'GH'), + 'pull': ('https://github.com/UW-Hydro/pysumma/pull/%s', 'PR'), + } + +napoleon_google_docstring = False +napoleon_use_param = False +napoleon_use_ivar = False + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'pysumma' +copyright = '2020, YoungDon Choi, Andrew Bennett' +author = 'YoungDon Choi, Andrew Bennett' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.0.0' +# The full version, including alpha/beta/rc tags. +release = '0.0.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'pysummadoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'pysumma.tex', 'pysumma Documentation', + 'YounDon Choi, Andrew Bennett', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'pysumma', 'pysumma Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'pysumma', 'pysumma Documentation', + author, 'pysumma ', 'One line description of project.', + 'Miscellaneous'), +] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..88d4b54e --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,65 @@ +.. pysumma documentation master file +.. _index: + +pysumma +======= +pysumma is a Python wrapper for manipulating, running, managing, and analyzing +of SUMMA (Structure for Unifying Multiple Modeling Alternatives) + + `SUMMA web site at UCAR `_ + +pysumma provides methods for: + - Running SUMMA + - Modifying SUMMA input files + - Automatically parallelizing distributed and sensitivity analysis type experiments + - Visualizing output + +Installation +============ +Currently we only support building pysumma from source. This can be accomplished by +running: + +:: + + git clone https://github.com/UW-Hydro/pysumma.git + cd pysumma + python setup.py install + +If you plan on helping to develop pysumma you may wish to use the following to install + +:: + + git clone https://github.com/UW-Hydro/pysumma.git + cd pysumma + python setup.py develop + +Additional SUMMA References +=========================== + - `Documentation `_ : SUMMA documentation is available online and remains a work in progress. + - `Source Code `_ : NCAR github + +Bugs +==== + Our issue tracker is at https://github.com/UW-Hydro/pysumma/issues. + Please report any bugs that you find. Or, even better, fork the repository on + GitHub and create a pull request. All changes are welcome, big or small, and we + will help you make the pull request if you are new to git + (just ask on the issue). + +How to run pySUMMA on HydroShare +================================ + (Link YouTube: https://www.youtube.com/watch?v=pL-LNd474Tw) + 1) log in HydoShare(https://www.hydroshare.org) + 2) Start CUAHSI JupyterHub from APPS menu on HydroShare(https://www.hydroshare.org/apps/) + 3) Open "Welcome.ipynb" to download pySUMMA resources from HydroShare + - run the code "1.How to connect with HydroShare" + - change the code from "resid = os.environ['HS_RES_ID']" to "resid = 'c1bb4a12bff44bf08c5958cba7947348'". + 4) You can see the list of Jupyter Notebooks and click one of Jupyter Notebook. + 5) Run one of Jupyter Notebooks. + +Sitemap +======= +.. toctree:: + :maxdepth: 3 + + api diff --git a/pysumma/__init__.py b/pysumma/__init__.py index 049410f7..273c972d 100644 --- a/pysumma/__init__.py +++ b/pysumma/__init__.py @@ -6,5 +6,4 @@ from .output_control import OutputControl from .local_param_info import LocalParamInfo from .force_file_list import ForceFileList -from . import plotting from . import utils diff --git a/pysumma/distributed.py b/pysumma/distributed.py index 206ddb26..58497347 100644 --- a/pysumma/distributed.py +++ b/pysumma/distributed.py @@ -21,6 +21,19 @@ class Distributed(object): in parallel. Currently only supports GRU based parallelization. + + Attributes + ---------- + executable: + Path to the SUMMA executable + manager: + FileManager object + num_workers: + Number of parallel workers to use + chunk_args: + List of dictionaries containing ``startGRU`` and ``countGRU`` values + simulations: + Dictionary of run names and Simulation objects ''' def __init__(self, executable: str, filemanager: str, diff --git a/pysumma/ensemble.py b/pysumma/ensemble.py index 07c70e04..b91618f1 100644 --- a/pysumma/ensemble.py +++ b/pysumma/ensemble.py @@ -15,6 +15,19 @@ class Ensemble(object): ''' Ensembles represent an multiple SUMMA configurations based on changing the decisions or parameters of a given run. + + Attributes + ---------- + executable: + Path to the SUMMA executable + filemanager: (optional) + Path to the file manager + configuration: + Dictionary of runs, along with settings + num_workers: + Number of parallel workers to use + simulations: + Dictionary of run names and Simulation objects ''' def __init__(self, executable: str,configuration: dict, diff --git a/pysumma/simulation.py b/pysumma/simulation.py index 201b678e..ffcf58a9 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -15,7 +15,40 @@ class Simulation(): - """The simulation object provides a wrapper for SUMMA simulations""" + """ + The simulation object provides a wrapper for SUMMA simulations + + Attributes + ---------- + stdout: + Store standard output of the run + stderr: + Handle to the process during runtime + manager_path: + Path to the file manager + config_path: + Path to where configuration will be written + status: + Current status of the simulation + manager: + File manager object (populated after calling ``initialize``) + decisions: + Decisions object (populated after calling ``initialize``) + output_control: + OutputControl object (populated after calling ``initialize``) + parameter_trial: + Parameter trial object (populated after calling ``initialize``) + force_file_list: + Forcing file list object (populated after calling ``initialize``) + local_param_info: + LocalParamInfo object (populated after calling ``initialize``) + basin_param_info: + BasinParamInfo object (populated after calling ``initialize``) + local_attributes: + LocalAttributes object (populated after calling ``initialize``) + initial_conditions: + InitialConditions object (populated after calling ``initialize``) + """ def __init__(self, executable, filemanager, initialize=True): """Initialize a new simulation object""" @@ -30,6 +63,13 @@ def __init__(self, executable, filemanager, initialize=True): self.initialize() def initialize(self): + """ + Initialize reads in all of the relevant files. This may not + be desired on instantiation, so the ``initialize`` parameter + can be set in the constructor. Calling this will also create + a backup of the configuration that can be restored via the + ``reset`` method. + """ self.manager = FileManager( self.manager_path.parent, self.manager_path.name) self.status = 'Initialized' @@ -48,7 +88,26 @@ def initialize(self): self.create_backup() self.status = 'Initialized' - def apply_config(self, config): + def apply_config(self, config: dict): + """ + Change the settings of the simulation based on a configuration + dictionary. + + Parameters + ---------- + config: + A dictionary where keys represent the type of change and + the values represent the changes to be applied. A representative + example might be: + + :: + config = { + 'file_manager': '/home/user/cool_setup/file_manager_new.txt', + 'decisions': {'snowLayers': 'CLM_2010'}, + 'parameters': {'albedoDecayRate': 1e-6}, + 'attributes': {'mHeight': 15} + } + """ if 'file_manager' in config: self.manager_path = Path(os.path.abspath(config['file_manager'])) for k, v in config.get('decisions', {}).items(): @@ -63,6 +122,17 @@ def apply_config(self, config): self.validate_layer_params(self.local_param_info) def assign_attributes(self, name, data): + """ + Assign new data to the ``local_attributes`` dataset. + + Parameters + ---------- + name: + The name (or key) of the attribute to modify + data: + The data to change the attribute to. The shape + must match the shape in the local attributes file + """ required_shape = self.local_attributes[name].shape try: self.local_attributes[name].values = np.array(data).reshape(required_shape) @@ -81,6 +151,7 @@ def create_backup(self): self.backup['manager_path'] = copy.deepcopy(self.manager_path) def reset(self): + """Restores the original settings of the Simulation""" self.manager = copy.deepcopy(self.backup['manager']) self.manager_path = copy.deepcopy(self.backup['manager_path']) self.config_path = self.manager_path.parent / '.pysumma' @@ -98,6 +169,7 @@ def reset(self): self.vegparm = self.manager.vegparm def validate_layer_params(self, params): + """Ensure that the layer parameters are valid""" for i in range(1, 5): assert (params[f'zmaxLayer{i}_upper'] <= params[f'zmaxLayer{i}_lower'], i) @@ -168,7 +240,10 @@ def _run_docker(self, run_suffix, processes=1, def start(self, run_option, run_suffix='pysumma_run', processes=1, prerun_cmds=[], startGRU=None, countGRU=None, iHRU=None, freq_restart=None, progress=None): - """Run a SUMMA simulation""" + """ + Run a SUMMA simulation without halting. Most likely this should + not be used. Use the ``run`` method for most common use cases. + """ #TODO: Implement running on hydroshare here if not prerun_cmds: prerun_cmds = [] @@ -187,11 +262,46 @@ def start(self, run_option, run_suffix='pysumma_run', processes=1, def run(self, run_option, run_suffix='pysumma_run', processes=1, prerun_cmds=None, startGRU=None, countGRU=None, iHRU=None, freq_restart=None, progress=None): + """ + Run a SUMMA simulation and halt until completion or error. + + Parameters + ---------- + run_option: + Method to run SUMMA, must be one of either local or docker + run_suffix: + Name to append to the output files for this SUMMA run + processes: + Number of openmp processes to use for this run. For this + to do anything SUMMA must be compiled with openmp support + prerun_cmds: + A list of commands to execute before running SUMMA. May be + necessary to set environment variables or do any preprocessing + startGRU: + GRU index to start the simulation on (must also set ``countGRU`` + if this argument is set) + countGRU: + Number of GRU to run, starting at ``startGRU`` (must also set + ``startGRU`` if this argument is set) + iHRU: + Index of HRU to run (cannot be used with ``startGRU`` and + ``countGRU``) + freq_restart: + Frequency to write restart files. Options include + ``[y, m, d, never]`` for yearly, monthly, and daily restart + files. Defaults to ``never`` + progress: + Frequency to write stdout progress. Note this is not printed + during runtime via pysumma, but can be checked after completion. + Options include ``[m, d, h, never]`` for monthly, daily, and + hourly output. + """ self.start(run_option, run_suffix, processes, prerun_cmds, startGRU, countGRU, iHRU, freq_restart, progress) self.monitor() def monitor(self): + '''Halt execution until Simulation either finishes or errors''' # Simulation already run if self.status in ['Error', 'Success']: return self.status @@ -218,7 +328,6 @@ def monitor(self): except Exception: self._output = None - return self.status def _write_configuration(self, name, write_netcdf: str=False): @@ -226,10 +335,7 @@ def _write_configuration(self, name, write_netcdf: str=False): self.config_path.mkdir(parents=True, exist_ok=True) manager_path = str(self.manager_path.parent) settings_path = os.path.abspath(os.path.realpath(str(self.manager['settings_path'].value))) - print(settings_path) settings_path = Path(settings_path.replace(manager_path, str(self.config_path))) - print(manager_path, self.config_path) - print(settings_path) self.manager_path = self.config_path / self.manager.file_name self.manager['settings_path'] = str(settings_path) + os.sep self.manager.write(path=self.config_path) @@ -250,7 +356,8 @@ def _write_configuration(self, name, write_netcdf: str=False): with open(settings_path / 'VEGPARM.TBL', 'w+') as f: f.writelines(self.vegparm) - def get_output(self) -> List[str]: + def get_output_files(self) -> List[str]: + """Find output files given the ``stdout`` generated from a run""" new_file_text = 'Created output file:' out_files = [] for l in self.stdout.split('\n'): @@ -261,6 +368,7 @@ def get_output(self) -> List[str]: @property def output(self): + """Get the output as an xarray dataset""" if self.status == 'Success': return self._output elif self.status == 'Error': @@ -275,6 +383,10 @@ def __repr__(self): repr = [] repr.append("Executable path: {}".format(self.executable)) repr.append("Simulation status: {}".format(self.status)) - repr.append("File manager configuration:") - repr.append(str(self.manager)) + try: + repr.append("File manager configuration:") + repr.append(str(self.manager)) + except: + repr.append("Use Simulation.initialize() to " + "read input files for more information") return '\n'.join(repr) diff --git a/readthedocs.yml b/readthedocs.yml new file mode 100644 index 00000000..a7ad271a --- /dev/null +++ b/readthedocs.yml @@ -0,0 +1,5 @@ +#conda: +# file: environment.yml +python: + version: 3 + setup_py_install: true diff --git a/setup.py b/setup.py index e458a695..38474d6c 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ 'pandas>=0.25', 'hs_restclient', 'distributed', - 'cartopy', 'fiona', 'netcdf4' ], From 19592015f87be8beb2557442b47bf4cf7663745a Mon Sep 17 00:00:00 2001 From: arbennett Date: Mon, 23 Mar 2020 16:28:32 -0700 Subject: [PATCH 14/36] Add environment.yml for conda users. --- environment.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 environment.yml diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..820a01ee --- /dev/null +++ b/environment.yml @@ -0,0 +1,23 @@ +name: pysumma +channels: + - conda-forge +dependencies: + - python>=3.5 + - xarray>=0.11.0 + - pandas + - netcdf4>=1.2.5 + - numpy>=1.11.2 + - dask + - distributed + - toolz + - pytest + - fiona + - cartopy + - shapely + - seaborn + - matplotlib + - geopandas + - jupyter + - pip + - pip: + - hs_restclient From d151ee65bf53440282573d69c716f8d41a70cf86 Mon Sep 17 00:00:00 2001 From: Andrew Bennett Date: Mon, 23 Mar 2020 16:29:02 -0700 Subject: [PATCH 15/36] Add environment.yml for conda users. (#102) --- environment.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 environment.yml diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..820a01ee --- /dev/null +++ b/environment.yml @@ -0,0 +1,23 @@ +name: pysumma +channels: + - conda-forge +dependencies: + - python>=3.5 + - xarray>=0.11.0 + - pandas + - netcdf4>=1.2.5 + - numpy>=1.11.2 + - dask + - distributed + - toolz + - pytest + - fiona + - cartopy + - shapely + - seaborn + - matplotlib + - geopandas + - jupyter + - pip + - pip: + - hs_restclient From e9c23bf09c4cc7b4a13713625de8d93e02d28be7 Mon Sep 17 00:00:00 2001 From: Andrew Bennett Date: Mon, 23 Mar 2020 17:12:02 -0700 Subject: [PATCH 16/36] Feature/docs (#103) * Add environment.yml for conda users. * Add environment.yml to readme * Update docs for readme * Update docs --- README.md | 19 ++++ docs/configuration.rst | 185 +++++++++++++++++++++++++++++++++++++ docs/index.rst | 22 +++++ pysumma/force_file_list.py | 2 +- pysumma/option.py | 4 + 5 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 docs/configuration.rst diff --git a/README.md b/README.md index ef2d0189..eff185e5 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,25 @@ pysumma provides methods for: - Visualizing output # Installation + +## Dependencies +A conda environment is available for management of pysumma's dependencies. +You can create your own environment from this file by running: +``` +conda env create -f environment.yml +``` + +Then, you can activate this environment with `conda activate pysumma`. +Before installing pysumma into this environment you may also wish to install it as a kernel in your Jupyter environments. +This can be accomplished by running: + +``` +python -m ipykernel install --user --name=pysumma +``` + +With this environment active you can install pysumma this environment with the instructions below. + +## Installing pysumma Currently we only support building pysumma from source. This can be accomplished by running: ``` diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 00000000..4dd0ce39 --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,185 @@ +.. _configuration.rst: + +SUMMA model setups require a number of configuration files and data sources to run. +pysumma provides interfaces to each of these files in a standardized fashion, allowing you to quickly manipulate existing SUMMA configurations. +For more information about the in depth details of each of the required inputs for SUMMA see the `SUMMA documentation on input `_ +This page shows some basic examples of how you can interact with these configuration objects as well as extremely concise descriptions of what each object does. +For more detailed information about each of the objects you can browse our API documentation :here:`./api.rst`. + + +Text based files +================ +All of the text-based input files are implemented around a base object named the ``OptionContainer``. +In turn, each specific option within these text-based inputs are implemented as an ``Option``. +Most of the code for this grouping of option types are in these base classes. +Specifics of the implementations are in each file's specific class. + +The generic way to interact with these types of files is similar to how you would interact with a Python dictionary. +It is possible to list all of the available attributes for each of these configuration types by using the ``list_options()`` method. +As an example we will first show basic usage with the ``FileManager`` class. +The other classes are shown in more condensed forms only to show the various differences between them. + +File manager +------------ +The filemanager tells SUMMA where to find each of the other required inputs. +It can be thought of the entry point to a SUMMA simulation. +The pysumma ``FileManager`` object stores each of these paths as well as provides an interface to the datastructres for easier manipulation. + +``FileManager`` objects are instantiated by providing the path to them as well as the file name as separate arguments. + +:: + + import pysumma as ps + fm = ps.FileManager('./summa_setup_template', 'file_manager.txt') + + +Then, you can see what is in it simply by printing it out: + +:: + + print(fm) + + > 'SUMMA_FILE_MANAGER_V1.0' ! filemanager_version + > '/home/user/summa_setup_template/settings/' ! settings_path + > '/home/user/summa_setup_template/forcing/' ! input_path + > '/home/user/summa_setup_template/output/' ! output_path + > 'decisions.txt' ! decisions_path + > '[notUsed]' ! meta_time + > '[notUsed]' ! meta_attr + > '[notUsed]' ! meta_type + > '[notUsed]' ! meta_force + > '[notUsed]' ! meta_localparam + > 'output_control.txt' ! output_control + > '[notUsed]' ! meta_localindex + > '[notUsed]' ! meta_basinparam + > '[notUsed]' ! meta_basinmvar + > 'local_attributes.nc' ! local_attributes + > 'local_param_info.txt' ! local_param_info + > 'basin_param_info.txt' ! basin_param_info + > 'forcing_file_list.txt' ! forcing_file_list + > 'initial_conditions.nc' ! model_init_cond + > 'parameter_trial.nc' ! parameter_trial + > 'test' ! output_prefix + +To see how to access each of these specific options you can use the ``list_options`` method. + +:: + + print(fm.list_options) + + > ['filemanager_version', 'settings_path', 'input_path', + > 'output_path', 'decisions_path', 'meta_time', + > 'meta_attr', 'meta_type', 'meta_force', 'meta_localparam', + > 'output_control', 'meta_localindex', 'meta_basinparam', + > 'meta_basinmvar', 'local_attributes', 'local_param_info', + > 'basin_param_info', 'forcing_file_list', 'model_init_cond', + > 'parameter_trial', 'output_prefix'] + +Then, each of these keys can be accessed directly similarly to how is done with python dictionaries. +This can be used to inspect the values of each option as well as modify their values. + +:: + + print(fm['output_prefix']) + + > 'test' ! output_prefix + + fm['output_prefix'] = 'tutorial' + + print(fm['output_prefix']) + + > 'tutorial' ! output_prefix + +Decisions +--------- +The decisions file contains the specification of the various physics options to use. +It also contains the run times and other algorithmic control options. +See `the SUMMA documentation `_ for a more complete description of the decisions. + +Instantiation of ``Decisions`` objects is similar to that of the other ``ObjectContainers``. +Once instantiated you can inspect the available decisions and the options available for each of them as follows. + +:: + dec = ps.Decisions('.', 'decisions.txt') + print(dec['snowLayers']) + + > snowLayers CLM_2010 ! choice of method to combine and sub-divide snow layers + + print(dec.list_options()) + + > ['simulStart', 'simulFinsh', 'tmZoneInfo', 'soilCatTbl', + > 'vegeParTbl', 'soilStress', 'stomResist', 'num_method', + > 'fDerivMeth', 'LAI_method', 'f_Richards', 'groundwatr', + > 'hc_profile', 'bcUpprTdyn', 'bcLowrTdyn', 'bcUpprSoiH', + > 'bcLowrSoiH', 'veg_traits', 'canopyEmis', 'snowIncept', + > 'windPrfile', 'astability', 'canopySrad', 'alb_method', + > 'compaction', 'snowLayers', 'thCondSnow', 'thCondSoil', + > 'spatial_gw', 'subRouting'] + + print(dec['snowLayers']) + + > snowLayers CLM_2010 ! choice of method to combine and sub-divide snow layers + + print(dec['snowLayers'].available_options) + + > ['jrdn1991', 'CLM_2010'] + + dec['snowLayers'] = 'jrdn1991' + +Forcing file list +----------------- +The forcing file list contains a listing of each of the forcing files available for use as SUMMA input. +To instantiate the `ForceFileList` you will have to specify the path that is set as the ``input_path`` in your ``FileManager``. Below we show using the ``FileManager`` (``fm``) to do so. +Once instantiated you can also use the `ForceFileList` object to inspect the forcing files themselves. + +:: + + ff = ps.ForceFileList('.', 'forcingFileList.1hr.txt', fm['input_path']) + print(ff) + + >> 'forcing_file.nc' + + print(ff.open_forcing_data()) + + [ + + Dimensions: (hru: 671, time: 744) + Coordinates: + * time (time) datetime64[ns] 1980-01-01 ... 1980-01-31T23:00:00 + Dimensions without coordinates: hru + Data variables: + LWRadAtm (time, hru) float32 ... + SWRadAtm (time, hru) float32 ... + airpres (time, hru) float32 ... + airtemp (time, hru) float32 ... + data_step timedelta64[ns] ... + hruId (hru) int64 ... + pptrate (time, hru) float32 ... + spechum (time, hru) float32 ... + windspd (time, hru) float32 ... + ] + +Output control +-------------- +The output control file contains a listing of all of the variables desired to be written to output, +along with how often and whether any aggregation needs to be done before writeout. +Because there are many available output variables that you can choose from we do not exhaustively list them. + + +Local parameter info +-------------------- +The local parameter info file contains a listing of global parameters. Spatially dependent parameters are specified +in the parameter trial NetCDF file. Values which are specified in the local parameter info file will be overwritten +by those specified in the parameter trial file. +As with the output control file, there are many parameters which can be specified, so we omit them for brevity. + + +NetCDF based files +================== +The following input files are NetCDF-based and therefore, should be interacted with via ``xarray`` when using pysumma: + + - Parameter trial + - Basin parameters + - Local attributes + - Initial conditions + diff --git a/docs/index.rst b/docs/index.rst index 88d4b54e..57a66734 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,28 @@ pysumma provides methods for: Installation ============ + +Dependencies +------------ + +A conda environment is available for management of pysumma's dependencies. +You can create your own environment from this file by running: + +:: + conda env create -f environment.yml + +Then, you can activate this environment with ``conda activate pysumma``. +Before installing pysumma into this environment you may also wish to install it as a kernel in your Jupyter environments. +This can be accomplished by running: + +:: + python -m ipykernel install --user --name=pysumma + +With this environment active you can install pysumma this environment with the instructions below. + +Installing pysumma +------------------ + Currently we only support building pysumma from source. This can be accomplished by running: diff --git a/pysumma/force_file_list.py b/pysumma/force_file_list.py index 533ad60c..4a2f1980 100644 --- a/pysumma/force_file_list.py +++ b/pysumma/force_file_list.py @@ -31,7 +31,7 @@ class ForceFileList(OptionContainer): prefix: str = '' def __init__(self, dirpath, filepath, force_file_prefix_path): - self.prefix = force_file_prefix_path + self.prefix = str(force_file_prefix_path) super().__init__(ForceFileListOption, dirpath, filepath) def set_option(self, key, value): diff --git a/pysumma/option.py b/pysumma/option.py index 6103ebf0..537fc8eb 100644 --- a/pysumma/option.py +++ b/pysumma/option.py @@ -188,6 +188,10 @@ def remove_option(self, name, strict=False): raise ValueError("Could not find option {}!".format(name)) return None + def list_options(self): + """Return a list of all available option keys""" + return [o.name for o in self.options] + def clear(self): self.options = [] From 599e6436d15dd5b2650b5be2d697d756f9d83f6f Mon Sep 17 00:00:00 2001 From: Andrew Bennett Date: Mon, 23 Mar 2020 17:16:29 -0700 Subject: [PATCH 17/36] Update index.rst --- docs/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 57a66734..c21d063c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,7 @@ A conda environment is available for management of pysumma's dependencies. You can create your own environment from this file by running: :: + conda env create -f environment.yml Then, you can activate this environment with ``conda activate pysumma``. @@ -31,6 +32,7 @@ Before installing pysumma into this environment you may also wish to install it This can be accomplished by running: :: + python -m ipykernel install --user --name=pysumma With this environment active you can install pysumma this environment with the instructions below. @@ -85,3 +87,4 @@ Sitemap :maxdepth: 3 api + configuration From 42b6aac2e75607e3d19b5b935d2a2c2add77a507 Mon Sep 17 00:00:00 2001 From: arbennett Date: Mon, 23 Mar 2020 19:33:41 -0700 Subject: [PATCH 18/36] update configuration docs --- docs/configuration.rst | 88 +++++++++++++++++++++++++++++++++--------- docs/index.rst | 1 + 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 4dd0ce39..8ebf30ad 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,10 +1,13 @@ .. _configuration.rst: +Interfaces to configuration files +********************************* + SUMMA model setups require a number of configuration files and data sources to run. pysumma provides interfaces to each of these files in a standardized fashion, allowing you to quickly manipulate existing SUMMA configurations. For more information about the in depth details of each of the required inputs for SUMMA see the `SUMMA documentation on input `_ This page shows some basic examples of how you can interact with these configuration objects as well as extremely concise descriptions of what each object does. -For more detailed information about each of the objects you can browse our API documentation :here:`./api.rst`. +For more detailed information about each of the objects you can browse our API documentation `here `_. Text based files @@ -26,6 +29,8 @@ It can be thought of the entry point to a SUMMA simulation. The pysumma ``FileManager`` object stores each of these paths as well as provides an interface to the datastructres for easier manipulation. ``FileManager`` objects are instantiated by providing the path to them as well as the file name as separate arguments. +The ``FileManager`` contains references to all of the other configuration files through the various attributes. +See the `API documentation `_ for more information about what attributes are available. :: @@ -90,6 +95,7 @@ This can be used to inspect the values of each option as well as modify their va > 'tutorial' ! output_prefix + Decisions --------- The decisions file contains the specification of the various physics options to use. @@ -100,6 +106,7 @@ Instantiation of ``Decisions`` objects is similar to that of the other ``ObjectC Once instantiated you can inspect the available decisions and the options available for each of them as follows. :: + dec = ps.Decisions('.', 'decisions.txt') print(dec['snowLayers']) @@ -141,29 +148,60 @@ Once instantiated you can also use the `ForceFileList` object to inspect the for print(ff.open_forcing_data()) - [ - - Dimensions: (hru: 671, time: 744) - Coordinates: - * time (time) datetime64[ns] 1980-01-01 ... 1980-01-31T23:00:00 - Dimensions without coordinates: hru - Data variables: - LWRadAtm (time, hru) float32 ... - SWRadAtm (time, hru) float32 ... - airpres (time, hru) float32 ... - airtemp (time, hru) float32 ... - data_step timedelta64[ns] ... - hruId (hru) int64 ... - pptrate (time, hru) float32 ... - spechum (time, hru) float32 ... - windspd (time, hru) float32 ... - ] + >> [ + >> + >> Dimensions: (hru: 671, time: 744) + >> Coordinates: + >> * time (time) datetime64[ns] 1980-01-01 ... 1980-01-31T23:00:00 + >> Dimensions without coordinates: hru + >> Data variables: + >> LWRadAtm (time, hru) float32 ... + >> SWRadAtm (time, hru) float32 ... + >> airpres (time, hru) float32 ... + >> airtemp (time, hru) float32 ... + >> data_step timedelta64[ns] ... + >> hruId (hru) int64 ... + >> pptrate (time, hru) float32 ... + >> spechum (time, hru) float32 ... + >> windspd (time, hru) float32 ... + >> ] Output control -------------- The output control file contains a listing of all of the variables desired to be written to output, along with how often and whether any aggregation needs to be done before writeout. Because there are many available output variables that you can choose from we do not exhaustively list them. +The format of the output control file mirrors the way that it is described in the +`SUMMA docs `_. + +:: + + oc = ps.OutputControl('.', 'output_control.txt') + print(oc) + + >> ! varName | outFreq | sum | inst | mean | var | min | max | mode + >> pptrate | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> airtemp | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarSWE | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarRainPlusMelt | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarTotalET | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarTotalRunoff | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarSurfaceRunoff | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarTotalSoilWat | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarAquiferStorage | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarAquiferBaseflow | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarNetRadiation | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarLatHeatTotal | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + >> scalarSenHeatTotal | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 + + print(oc['scalarTotalRunoff'].statistic) + + >> instant + + oc['scalarTotalRunoff'] = [24, 1, 0, 0, 0, 0, 0, 0] + print(oc['scalarTotalRunoff'].statistic) + + >> sum Local parameter info @@ -172,7 +210,21 @@ The local parameter info file contains a listing of global parameters. Spatially in the parameter trial NetCDF file. Values which are specified in the local parameter info file will be overwritten by those specified in the parameter trial file. As with the output control file, there are many parameters which can be specified, so we omit them for brevity. +Additionally, we currently do not have descriptions of what each of the parameters represent - the best way to figure +this out currently is by looking at the SUMMA source code directly. + +:: + + lpi = ps.LocalParamInfo('.', 'local_param_info.txt') + print(lpi.list_options()) + + >> ['upperBoundHead', 'lowerBoundHead', 'upperBoundTheta', 'lowerBoundTheta', + >> 'upperBoundTemp', 'lowerBoundTemp', 'tempCritRain', 'tempRangeTimestep', + >> ... + >> 'zmaxLayer1_lower', 'zmaxLayer2_lower', 'zmaxLayer3_lower', 'zmaxLayer4_lower', + >> 'zmaxLayer1_upper', 'zmaxLayer2_upper', 'zmaxLayer3_upper', 'zmaxLayer4_upper'] + lpi['tempCritRain'] = 273.3 NetCDF based files ================== diff --git a/docs/index.rst b/docs/index.rst index 57a66734..76db56f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -85,3 +85,4 @@ Sitemap :maxdepth: 3 api + configuration From 7251f4ac5638047c3b159a2a3af1e519a85d8833 Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 26 Mar 2020 14:19:56 -0600 Subject: [PATCH 19/36] Update ensemble & distributed to allow existing clients --- pysumma/distributed.py | 21 ++++++++++++++------- pysumma/ensemble.py | 20 +++++++++++++------- pysumma/simulation.py | 2 +- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/pysumma/distributed.py b/pysumma/distributed.py index 58497347..97f9b5fd 100644 --- a/pysumma/distributed.py +++ b/pysumma/distributed.py @@ -38,7 +38,8 @@ class Distributed(object): def __init__(self, executable: str, filemanager: str, num_workers: int=1, threads_per_worker: int=OMP_NUM_THREADS, - chunk_size: int=None, num_chunks: int=None, scheduler: str=None): + chunk_size: int=None, num_chunks: int=None, scheduler: str=None, + client: Client=None): """ Initialize a new distributed object @@ -70,15 +71,21 @@ def __init__(self, executable: str, filemanager: str, self.submissions: List = [] self.num_workers: int = num_workers # Try to get a client, and if none exists then start a new one - try: - self._client = get_client() - # Start more workers if necessary: + if client: + self._client = client workers = len(self._client.get_worker_logs()) if workers <= self.num_workers: self._client.cluster.scale(workers) - except ValueError: - self._client = Client(n_workers=self.num_workers, - threads_per_worker=threads_per_worker) + else: + try: + self._client = get_client() + # Start more workers if necessary: + workers = len(self._client.get_worker_logs()) + if workers <= self.num_workers: + self._client.cluster.scale(workers) + except ValueError: + self._client = Client(n_workers=self.num_workers, + threads_per_worker=threads_per_worker) self.chunk_args = self._generate_args(chunk_size, num_chunks) self._generate_simulation_objects() diff --git a/pysumma/ensemble.py b/pysumma/ensemble.py index b91618f1..67f53fe8 100644 --- a/pysumma/ensemble.py +++ b/pysumma/ensemble.py @@ -33,7 +33,7 @@ class Ensemble(object): def __init__(self, executable: str,configuration: dict, filemanager: str=None, num_workers: int=1, threads_per_worker: int=OMP_NUM_THREADS, - scheduler: str=None): + scheduler: str=None, client: Client=None): """ Create a new Ensemble object. The API mirrors that of the Simulation object. @@ -46,15 +46,21 @@ def __init__(self, executable: str,configuration: dict, self.simulations: dict = {} self.submissions: list = [] # Try to get a client, and if none exists then start a new one - try: - self._client = get_client() - # Start more workers if necessary: + if client: + self._client = client workers = len(self._client.get_worker_logs()) if workers <= self.num_workers: self._client.cluster.scale(workers) - except ValueError: - self._client = Client(n_workers=self.num_workers, - threads_per_worker=threads_per_worker) + else: + try: + self._client = get_client() + # Start more workers if necessary: + workers = len(self._client.get_worker_logs()) + if workers <= self.num_workers: + self._client.cluster.scale(workers) + except ValueError: + self._client = Client(n_workers=self.num_workers, + threads_per_worker=threads_per_worker) self._generate_simulation_objects() def _generate_simulation_objects(self): diff --git a/pysumma/simulation.py b/pysumma/simulation.py index ffcf58a9..f287372c 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -322,7 +322,7 @@ def monitor(self): self.status = 'Success' try: - self._output = [xr.open_dataset(f) for f in self.get_output()] + self._output = [xr.open_dataset(f) for f in self.get_output_files()] if len(self._output) == 1: self._output = self._output[0] except Exception: From 35cd8486c59ce432112bd517fa02895470dac24f Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 9 Apr 2020 09:31:53 -0700 Subject: [PATCH 20/36] First pass at allowing manipulation of parameter trial files --- pysumma/ensemble.py | 9 ++++++--- pysumma/simulation.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/pysumma/ensemble.py b/pysumma/ensemble.py index 67f53fe8..4250cf1f 100644 --- a/pysumma/ensemble.py +++ b/pysumma/ensemble.py @@ -315,7 +315,7 @@ def file_manager_product(list_config): def total_product(dec_conf={}, param_conf={}, attr_conf={}, fman_conf={}, - sequential_keys=False): + param_trial_conf={}, sequential_keys=False): """ Combines multiple types of model changes into a single configuration for the Ensemble object. @@ -323,17 +323,18 @@ def total_product(dec_conf={}, param_conf={}, attr_conf={}, fman_conf={}, full_conf = deepcopy(dec_conf) full_conf.update(param_conf) full_conf.update(attr_conf) + full_conf.update(param_trial_conf) full_conf.update(fman_conf) prod_dict = product_dict(**full_conf) config = {} for i, d in enumerate(prod_dict): name = '++' + '++'.join( - '{}={}'.format(k, v) if k in param_conf or k in attr_conf + '{}={}'.format(k, v) if k in param_conf or k in attr_conf or k in param_trial_conf else v.replace('/', '_').replace('\\', '_') for k, v in d.items()) + '++' if sequential_keys: name = f'run_{i}' - config[name] = {'decisions': {}, 'parameters': {}, 'attributes': {}} + config[name] = {'decisions': {}, 'parameters': {}, 'attributes': {}, 'trial_parameters': {}} for k, v in d.items(): if k in dec_conf: config[name]['decisions'][k] = v @@ -341,6 +342,8 @@ def total_product(dec_conf={}, param_conf={}, attr_conf={}, fman_conf={}, config[name]['parameters'][k] = v elif k in attr_conf: config[name]['attributes'][k] = v + elif k in param_trial_conf: + config[name]['trial_parameters'][k] = v elif k in fman_conf: config[name]['file_manager'] = v return config diff --git a/pysumma/simulation.py b/pysumma/simulation.py index f287372c..6db5e4b8 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -118,6 +118,8 @@ def apply_config(self, config: dict): self.output_control.set_option(k, **v) for k, v in config.get('attributes', {}).items(): self.assign_attributes(k, v) + for k, v in config.get('trial_parameters', {}).items(): + self.assign_trial_params(k, v) if self.decisions['snowLayers'] == 'CLM_2010': self.validate_layer_params(self.local_param_info) @@ -145,6 +147,36 @@ def assign_attributes(self, name, data): 'io/en/latest/input_output/SUMMA_input/#attribute-and-', 'parameter-files for more information', e) + def assign_trial_params(self, name, data, dim='hru', create=True): + """ + Assign new data to the ``parameter_trial`` dataset. + + Parameters + ---------- + name: + The name (or key) of the attribute to modify + data: + The data to change the parameter to. The shape + must match the shape in the parameter trial file + """ + # Create the variable if we need + print(name in self.parameter_trial.variables) + if create and name not in self.parameter_trial.variables: + self.parameter_trial[name] = self.parameter_trial[dim].astype(float).copy() + required_shape = self.parameter_trial[name].shape + try: + self.parameter_trial[name].values = np.array(data).reshape(required_shape) + except ValueError as e: + raise ValueError('The shape of the provided replacement data does', + ' not match the shape of the original data.', e) + except KeyError as e: + raise KeyError(f'The key {name} does not exist in this trial parameter', + 'file. See the documentation at https://summa.readthedocs.', + 'io/en/latest/input_output/SUMMA_input/#attribute-and-', + 'parameter-files for more information', e) + + + def create_backup(self): self.backup = {} self.backup['manager'] = copy.deepcopy(self.manager) From 943b644e171061c8381c8769c10e030030c7a3eb Mon Sep 17 00:00:00 2001 From: arbennett Date: Fri, 10 Apr 2020 09:43:26 -0700 Subject: [PATCH 21/36] Clean up, add documentation, helper function --- pysumma/ensemble.py | 24 ++++++++++++++++++++++++ pysumma/simulation.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pysumma/ensemble.py b/pysumma/ensemble.py index 4250cf1f..ce4ecab6 100644 --- a/pysumma/ensemble.py +++ b/pysumma/ensemble.py @@ -291,6 +291,30 @@ def attribute_product(list_config): {'attributes': d} for d in product_dict(**list_config)} +def trial_parameter_product(list_config): + """ + Create a dictionary of runs based on a simpler list configuration + of trial parameter values + + Parameters + ---------- + list_config: + A dictionary of the sort + {key1: [list of values], key2: [list of values]} + + Returns + -------- + A dictionary of the sort: + {name: {key1: value1, key2: value1}, + name: {key1: value2, key2: value1}, + ... + name: {key1: valueN, key2: valueN}} + """ + return {'++'+'++'.join('{}={}'.format(k, v) for k, v in d.items())+'++': + {'trial_parameters': d} for d in product_dict(**list_config)} + + + def file_manager_product(list_config): """ Create a dictionary of runs based on a simpler list configuration diff --git a/pysumma/simulation.py b/pysumma/simulation.py index 6db5e4b8..5f070720 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -105,6 +105,7 @@ def apply_config(self, config: dict): 'file_manager': '/home/user/cool_setup/file_manager_new.txt', 'decisions': {'snowLayers': 'CLM_2010'}, 'parameters': {'albedoDecayRate': 1e-6}, + 'trial_parameters': {'theta_mp': 0.4}, 'attributes': {'mHeight': 15} } """ @@ -160,7 +161,6 @@ def assign_trial_params(self, name, data, dim='hru', create=True): must match the shape in the parameter trial file """ # Create the variable if we need - print(name in self.parameter_trial.variables) if create and name not in self.parameter_trial.variables: self.parameter_trial[name] = self.parameter_trial[dim].astype(float).copy() required_shape = self.parameter_trial[name].shape From 851775ade5002f884020b2cb1ebb0c67a16e17c0 Mon Sep 17 00:00:00 2001 From: arbennett Date: Mon, 18 May 2020 15:54:21 -0700 Subject: [PATCH 22/36] Fix last gru index for distributed --- pysumma/distributed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysumma/distributed.py b/pysumma/distributed.py index 97f9b5fd..344ef590 100644 --- a/pysumma/distributed.py +++ b/pysumma/distributed.py @@ -115,7 +115,7 @@ def _generate_args(self, chunk_size: int=None, num_chunks: int=None): if chunk_size: sim_truncated = (chunk_size-1) * (sim_size // (chunk_size-1)) starts = np.arange(1, sim_truncated+1, chunk_size).astype(int) - stops = np.append(starts[1:], sim_size) + stops = np.append(starts[1:], sim_size+1) chunks = np.vstack([starts, stops]).T elif num_chunks: chunk_size = np.ceil(sim_size / num_chunks).astype(int) From c6c238aead7ecbaeddfdf0e39c5d04d31c830bf5 Mon Sep 17 00:00:00 2001 From: arbennett Date: Mon, 18 May 2020 21:57:49 -0700 Subject: [PATCH 23/36] WIP: Adding basic framework for calibration with OSTRICH --- pysumma/__init__.py | 1 + pysumma/calibration/__init__.py | 1 + .../meta/model_executable.template | 0 .../meta/objective_function.template | 22 +++++ pysumma/calibration/meta/ostIn.template | 64 ++++++++++++ .../calibration/meta/save_parameters.template | 6 ++ pysumma/calibration/ostrich.py | 97 +++++++++++++++++++ pysumma/evaluation.py | 23 +++++ pysumma/validation.py | 13 --- 9 files changed, 214 insertions(+), 13 deletions(-) create mode 100644 pysumma/calibration/__init__.py create mode 100644 pysumma/calibration/meta/model_executable.template create mode 100644 pysumma/calibration/meta/objective_function.template create mode 100644 pysumma/calibration/meta/ostIn.template create mode 100644 pysumma/calibration/meta/save_parameters.template create mode 100644 pysumma/calibration/ostrich.py create mode 100644 pysumma/evaluation.py delete mode 100644 pysumma/validation.py diff --git a/pysumma/__init__.py b/pysumma/__init__.py index 273c972d..d6d2645e 100644 --- a/pysumma/__init__.py +++ b/pysumma/__init__.py @@ -7,3 +7,4 @@ from .local_param_info import LocalParamInfo from .force_file_list import ForceFileList from . import utils +from calibration import Ostrich diff --git a/pysumma/calibration/__init__.py b/pysumma/calibration/__init__.py new file mode 100644 index 00000000..596f8ca7 --- /dev/null +++ b/pysumma/calibration/__init__.py @@ -0,0 +1 @@ +from ostrich import Ostrich diff --git a/pysumma/calibration/meta/model_executable.template b/pysumma/calibration/meta/model_executable.template new file mode 100644 index 00000000..e69de29b diff --git a/pysumma/calibration/meta/objective_function.template b/pysumma/calibration/meta/objective_function.template new file mode 100644 index 00000000..a80a1b03 --- /dev/null +++ b/pysumma/calibration/meta/objective_function.template @@ -0,0 +1,22 @@ +""" +Template objective function script for pysumma calibration with ostrich +""" + +import pysumma.evaluation as pse +import xarray as xr + +if __name__ == '__main__': + sim_file = $simFile + obs_file = $obsFile + calib_var = $calibVar + out_file = $outFile + + sim_ds = xr.open_dataset($simFile) + obs_ds = xr.open_dataset($obsFile) + kge = pse.kling_gupta_efficiency(sim_ds[calib_var], obs_ds[calib_var]) + mae = pse.mean_absolute_error(sim_ds[calib_var], obs_ds[calib_var]) + rmse = pse.root_mean_square_error(sim_ds[calib_var], obs_ds[calib_var]) + with open(out_file, 'w+') as f: + f.writeline('%.6f'%kge + '\t #KGE') + f.writeline('%.6f'%mae + '\t #MAE') + f.writeline('%.6f'%rmse + '\t #RMSE') diff --git a/pysumma/calibration/meta/ostIn.template b/pysumma/calibration/meta/ostIn.template new file mode 100644 index 00000000..6d7d5658 --- /dev/null +++ b/pysumma/calibration/meta/ostIn.template @@ -0,0 +1,64 @@ +# ----------------------------------------------------------------- +# Ostrich configuration file template for pysumma calibration +# ----------------------------------------------------------------- +# This file is automatically populated by pysumma and provides +# a high level interface to the OSTRICH optimization package. +# We do not implement many of the available options for the +# sake of ease of use (and implementation). +# ----------------------------------------------------------------- +# Currently only allow DDS optimization routine +ProgramType DDS +ModelExecutable $runScript +ObjectiveFunction $objectiveFun +PreserveBestModel $saveScript +PreserveModelOutput $preserveOutput +RandomSeed $seed +OnObsError $errval + +BeginFilePairs +$weightTemplateFile; $weigthtValueFile +EndFilePairs + +# ------------------------------------------------- +# Parameter/DV Specification - note: what are txInN, txOst ...? +# Format: +# parameter start minimum maximum txInN txOst txOut fmt +# ------------------------------------------------- +BeginParams +$paramSection +EndParams + +# ------------------------------------------------- +# Response variable specification - note: keyworld should aways be OST_NULL +# Format: +# name filename keyword line col token +# ------------------------------------------------- +BeginResponseVars +$responseSection +EndResponseVars + +# ------------------------------------------------- +# Compound response variable specification +# Format is complicated - see OSTRICH documentation +# ------------------------------------------------- +BeginTiedRespVars +$tiedResponseSection +EndTiedRespVars + +# ------------------------------------------------- +# Definition of cost function and penalty. +# Currently we do not support constraints so penalty is hard coded to APM +# ------------------------------------------------- +BeginGCOP +CostFunction $costFunction +PenaltyFunction APM +EndGCOP + +# ------------------------------------------------- +# DDS algorithm control +# ------------------------------------------------- +BeginDDSAlg +PerturbationValue $perturbVal +MaxIterations $maxIters +UseInitialParamValues +EndDDSAlg diff --git a/pysumma/calibration/meta/save_parameters.template b/pysumma/calibration/meta/save_parameters.template new file mode 100644 index 00000000..631cbe31 --- /dev/null +++ b/pysumma/calibration/meta/save_parameters.template @@ -0,0 +1,6 @@ +import pysumma as ps +import xarray as xr +import os + +if __name__ == '__main__': + pass diff --git a/pysumma/calibration/ostrich.py b/pysumma/calibration/ostrich.py new file mode 100644 index 00000000..0ccb67e3 --- /dev/null +++ b/pysumma/calibration/ostrich.py @@ -0,0 +1,97 @@ +from pkg_resources import resource_filename as resource +from functools import partial +from string import Template +from typing import List, Dict + +def read_template(path): + with open(path, 'r') as f: + OST_FILE= f.read() + return Template(OST_FILE) + +resource = partial(resource, __name__) + +INPT_META = read_template(resource('meta/ostIn.template')) +EXEC_META = read_template(resource('meta/model_executable.template')) +LOSS_META = read_template(resource('meta/objective_function.template')) +SAVE_META = read_template(resource('meta/save_parameters.template')) + + +class Ostrich(): + + def __init__(self, ostrich_executable, summa_executable): + self.ostrich: str = ostrich_executable + self.summa: str = summa_executable + self.template: Template = INPT_META + self.preserve_output: str ='no' + self.seed: int = 42 + self.errval: float = -9999 + self.perturb_val: float = 0.2 + self.max_iters: int = 100 + self.calib_params: List[OstrichParam] = [] + self.cost_function = 'KGE' + self.maximize = True + + def read_config(self, config_file): + raise NotImplementedError() + + def write_config(self, path): + raise NotImplementedError() + + def run(self): + raise NotImplementedError() + + def to_simulation(self): + raise NotImplementedError() + + def save_setup(self, path): + raise NotImplementedError() + + @property + def param_section(self) -> str: + return '\n'.join(str(param) for param in self.calib_params) + + def write_weight_template_section(self, path='./param_mapping.tpl') -> str: + with open(path) as f: + f.write('\n'.join([f'{cp.realname} | {cp.weightname}' + for cp in self.calib_params])) + return path + + def write_weight_value_section(self, path='./param_weights.txt') -> str: + with open(path) as f: + f.write('\n'.join([f'{cp.realname} | {cp.value}' + for cp in self.calib_params])) + return path + + @property + def tied_response_section(self) -> str: + if self.maximize: + return f'neg{self.cost_function} 1 {self.cost_function} wsum -1.0.0' + else: + return '# nothing to do here' + + @property + def map_vars_to_template(self) -> Dict[str: str]: + return {'runScript': self.run_script, + 'objectiveFun': self.objective_function, + 'saveScript': self.save_script, + 'preserveOutput': self.preserve_output, + 'seed': self.seed, + 'errval': self.errval, + 'perturbVal': self.perturb_val, + 'maxIters': self.max_iters, + 'paramSection': self.param_section, + 'responseSection': self.response_section, + 'tiedResponseSection': self.tied_response_section, + 'costFunction': self.cost_function + } + +class OstrichParam(): + + def __init__(self, name, value, val_range): + self.realname = name + self.weightname = f'{name}_mtp' + self.value = value + self.lower, self.upper = val_range + + def __str__(self): + return f"{self.weightname} {self.value} {self.lower} none none none free" diff --git a/pysumma/evaluation.py b/pysumma/evaluation.py new file mode 100644 index 00000000..d13c8126 --- /dev/null +++ b/pysumma/evaluation.py @@ -0,0 +1,23 @@ +from sklearn.metrics import mean_absolute_error, mean_squared_error +import math +import numpy as np + + +def kling_gupta_efficiency(sim, obs): + obs = np.asarray(obs) + sim = np.asarray(sim) + obs_filtered = obs[~np.isnan(obs)] + sim_filtered = sim[~np.isnan(obs)] + sim_std = np.std(sim_filtered, ddof=1) + obs_std = np.std(obs_filtered, ddof=1) + sim_mu = np.mean(sim_filtered) + obs_mu = np.mean(obs_filtered) + r = np.corrcoef(sim_filtered, obs_filtered)[0, 1] + var = sim_std / obs_std + bias = sim_mu / obs_mu + kge = 1 - np.sqrt((bias-1)**2 + (var-1)**2 + (r-1)**2) + return kge + + +def root_mean_square_error(sim, obs): + return np.sqrt(mean_squared_error(sim, obs)) diff --git a/pysumma/validation.py b/pysumma/validation.py deleted file mode 100644 index d7b0f841..00000000 --- a/pysumma/validation.py +++ /dev/null @@ -1,13 +0,0 @@ -from sklearn.metrics import mean_absolute_error, mean_squared_error -import math - - -class Validation(object): - - def analysis(observation, simulation): - MAE_B = mean_absolute_error(observation, simulation) - MSE_B = mean_squared_error(observation, simulation) - RMSE_B = math.sqrt(MSE_B) - print('Mean Absolute Error: %f' % MAE_B) - print('Mean Squared Error: %f' % MSE_B) - print('Root Mean Squared Error: %f' % RMSE_B) From c011015fce6f7478dbdda59bfcd23e782afff574 Mon Sep 17 00:00:00 2001 From: arbennett Date: Tue, 19 May 2020 16:03:45 -0700 Subject: [PATCH 24/36] Working setup of Ostrich! --- pysumma/__init__.py | 2 +- pysumma/calibration/__init__.py | 2 +- .../meta/model_executable.template | 58 ++++++++ .../meta/objective_function.template | 22 --- pysumma/calibration/meta/ostIn.template | 2 +- .../calibration/meta/save_parameters.template | 8 +- pysumma/calibration/ostrich.py | 140 ++++++++++++++---- pysumma/evaluation.py | 9 ++ pysumma/simulation.py | 6 +- 9 files changed, 191 insertions(+), 58 deletions(-) delete mode 100644 pysumma/calibration/meta/objective_function.template diff --git a/pysumma/__init__.py b/pysumma/__init__.py index d6d2645e..e4cf1fd0 100644 --- a/pysumma/__init__.py +++ b/pysumma/__init__.py @@ -7,4 +7,4 @@ from .local_param_info import LocalParamInfo from .force_file_list import ForceFileList from . import utils -from calibration import Ostrich +from .calibration import Ostrich, OstrichParam diff --git a/pysumma/calibration/__init__.py b/pysumma/calibration/__init__.py index 596f8ca7..f63917e3 100644 --- a/pysumma/calibration/__init__.py +++ b/pysumma/calibration/__init__.py @@ -1 +1 @@ -from ostrich import Ostrich +from .ostrich import Ostrich, OstrichParam diff --git a/pysumma/calibration/meta/model_executable.template b/pysumma/calibration/meta/model_executable.template index e69de29b..e5c16c6f 100644 --- a/pysumma/calibration/meta/model_executable.template +++ b/pysumma/calibration/meta/model_executable.template @@ -0,0 +1,58 @@ +#!$pythonPath +import pysumma as ps +import pysumma.evaluation as pse +import shutil +import xarray as xr + +if __name__ == '__main__': + # Template variables + summa_exe = '$summaExe' + file_manager = '$fileManager' + obs_data_file = '$obsDataFile' + sim_calib_var = '$simVarName' + obs_calib_var = '$obsVarName' + out_file = '$outFile' + param_mapping_file = '$paramMappingFile' + param_weight_file = '$paramWeightFile' + param_file = '$paramFile' + + # read in parameters from ostrich files and summa setup + with xr.open_dataset(param_file) as temp: + trial_params = temp.load() + + param_dict = {} + with open(param_weight_file) as weights: + for line in weights: + name, value = line.split('|') + param_dict[name.strip()] = float(value.strip()) + + # insert calibration parameters from ostrich + for k, v in param_dict.items(): + trial_params[k] = xr.full_like(trial_params['hruIndex'], fill_value=v) + trial_params.to_netcdf(param_file) + + # initialize simulation object + s = ps.Simulation(summa_exe, file_manager) + + # run the simulation + s.run('local') + assert s.status == 'Success' + + # open output and calculate diagnostics + sim_ds = s.output + obs_ds = xr.open_dataset(obs_data_file) + + # trim sim and obs to common time length + time_slice = pse.trim_time(sim_ds, obs_ds) + sim_ds = sim_ds.sel(time=time_slice) + obs_ds = obs_ds.sel(time=time_slice) + + kge = pse.kling_gupta_efficiency(sim_ds[sim_calib_var], obs_ds[obs_calib_var]) + mae = pse.mean_absolute_error(sim_ds[sim_calib_var], obs_ds[obs_calib_var]) + rmse = pse.root_mean_square_error(sim_ds[sim_calib_var], obs_ds[obs_calib_var]) + + # save diagnostics in form that ostrich can read + with open(out_file, 'w+') as f: + f.write('%.6f'%kge + '\t #KGE\n') + f.write('%.6f'%mae + '\t #MAE\n') + f.write('%.6f'%rmse + '\t #RMSE\n') diff --git a/pysumma/calibration/meta/objective_function.template b/pysumma/calibration/meta/objective_function.template deleted file mode 100644 index a80a1b03..00000000 --- a/pysumma/calibration/meta/objective_function.template +++ /dev/null @@ -1,22 +0,0 @@ -""" -Template objective function script for pysumma calibration with ostrich -""" - -import pysumma.evaluation as pse -import xarray as xr - -if __name__ == '__main__': - sim_file = $simFile - obs_file = $obsFile - calib_var = $calibVar - out_file = $outFile - - sim_ds = xr.open_dataset($simFile) - obs_ds = xr.open_dataset($obsFile) - kge = pse.kling_gupta_efficiency(sim_ds[calib_var], obs_ds[calib_var]) - mae = pse.mean_absolute_error(sim_ds[calib_var], obs_ds[calib_var]) - rmse = pse.root_mean_square_error(sim_ds[calib_var], obs_ds[calib_var]) - with open(out_file, 'w+') as f: - f.writeline('%.6f'%kge + '\t #KGE') - f.writeline('%.6f'%mae + '\t #MAE') - f.writeline('%.6f'%rmse + '\t #RMSE') diff --git a/pysumma/calibration/meta/ostIn.template b/pysumma/calibration/meta/ostIn.template index 6d7d5658..f3baaa15 100644 --- a/pysumma/calibration/meta/ostIn.template +++ b/pysumma/calibration/meta/ostIn.template @@ -16,7 +16,7 @@ RandomSeed $seed OnObsError $errval BeginFilePairs -$weightTemplateFile; $weigthtValueFile +$weightTemplateFile; $weightValueFile EndFilePairs # ------------------------------------------------- diff --git a/pysumma/calibration/meta/save_parameters.template b/pysumma/calibration/meta/save_parameters.template index 631cbe31..640ff3aa 100644 --- a/pysumma/calibration/meta/save_parameters.template +++ b/pysumma/calibration/meta/save_parameters.template @@ -1,6 +1,12 @@ +#!$pythonPath import pysumma as ps import xarray as xr +import shutil import os if __name__ == '__main__': - pass + save_dir = '$saveDir' + model_dir = '$modelDir' + if not os.path.exists(save_dir): + os.mkdir(save_dir) + shutil.copytree(model_dir, save_dir) diff --git a/pysumma/calibration/ostrich.py b/pysumma/calibration/ostrich.py index 0ccb67e3..09447aa9 100644 --- a/pysumma/calibration/ostrich.py +++ b/pysumma/calibration/ostrich.py @@ -2,6 +2,12 @@ from functools import partial from string import Template from typing import List, Dict +from pathlib import Path +from pysumma import Simulation +import subprocess +import shutil +import stat +import os def read_template(path): with open(path, 'r') as f: @@ -10,67 +16,113 @@ def read_template(path): resource = partial(resource, __name__) -INPT_META = read_template(resource('meta/ostIn.template')) -EXEC_META = read_template(resource('meta/model_executable.template')) -LOSS_META = read_template(resource('meta/objective_function.template')) -SAVE_META = read_template(resource('meta/save_parameters.template')) +INPT_FILE = resource('meta/ostIn.template') +EXEC_FILE = resource('meta/model_executable.template') +SAVE_FILE = resource('meta/save_parameters.template') + +INPT_META = read_template(INPT_FILE) +EXEC_META = read_template(EXEC_FILE) +SAVE_META = read_template(SAVE_FILE) class Ostrich(): - def __init__(self, ostrich_executable, summa_executable): + def __init__(self, ostrich_executable, summa_executable, file_manager, python_path='python'): self.ostrich: str = ostrich_executable + self.python_path: str = python_path self.summa: str = summa_executable self.template: Template = INPT_META + self.save_template: Template = SAVE_META + self.run_template: Template = EXEC_META + self.config_path: Path = Path(os.path.abspath(file_manager)).parent / 'calibration' + self.simulation = Simulation(summa_executable, file_manager, + config_dir=self.config_path) + self.file_manager = self.simulation.manager + self.run_script: Path = self.config_path / 'run_script.py' + self.save_script: Path = self.config_path / 'save_script.py' + self.metrics_file: Path = self.config_path / 'metrics.txt' self.preserve_output: str ='no' self.seed: int = 42 self.errval: float = -9999 self.perturb_val: float = 0.2 self.max_iters: int = 100 self.calib_params: List[OstrichParam] = [] - self.cost_function = 'KGE' - self.maximize = True + self.cost_function: str = 'KGE' + self.objective_function: str = 'gcop' + self.maximize: bool = True + + def run(self, prerun_cmds=[]): + if len(prerun_cmds): + preprocess_cmd = " && ".join(prerun_cmds) + " && " + else: + preprocess_cmd = "" + cmd = preprocess_cmd + f'cd {str(self.config_path)} && ./ostrich' + self.process = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + self.stdout, self.stderr = self.process.communicate() + if isinstance(self.stdout, bytes): + self.stderr = self.stderr.decode('utf-8', 'ignore') + self.stdout = self.stdout.decode('utf-8', 'ignore') def read_config(self, config_file): - raise NotImplementedError() + raise NotImplementedError('We do not yet support importing OSTRICH configurations!') - def write_config(self, path): - raise NotImplementedError() + def write_config(self): + if not os.path.exists(self.config_path): + os.mkdir(self.config_path) - def run(self): - raise NotImplementedError() + # Substitue templates and save + self.weightTemplateFile = self.write_weight_template_section() + self.weightValueFile = self.write_weight_value_section() - def to_simulation(self): - raise NotImplementedError() + with open(self.config_path / 'ostIn.txt', 'w') as f: + f.write(self.template.substitute(self.map_vars_to_template)) + with open(self.save_script, 'w') as f: + f.write(self.save_template.substitute(self.map_vars_to_save_template)) - def save_setup(self, path): - raise NotImplementedError() + self.simulation._write_configuration() + with open(self.run_script, 'w') as f: + f.write(self.run_template.substitute(self.map_vars_to_run_template)) - @property - def param_section(self) -> str: - return '\n'.join(str(param) for param in self.calib_params) + shutil.copy(self.ostrich, self.config_path / 'ostrich') - def write_weight_template_section(self, path='./param_mapping.tpl') -> str: - with open(path) as f: + # Make sure we set permissions for execution + st = os.stat(self.config_path / 'ostrich') + os.chmod(self.config_path / 'ostrich', st.st_mode | stat.S_IEXEC) + st = os.stat(self.run_script) + os.chmod(self.run_script, st.st_mode | stat.S_IEXEC) + st = os.stat(self.save_script) + os.chmod(self.save_script, st.st_mode | stat.S_IEXEC) + + def write_weight_template_section(self, file_name=Path('param_mapping.tpl')) -> Path: + with open(self.config_path / file_name, 'w') as f: f.write('\n'.join([f'{cp.realname} | {cp.weightname}' for cp in self.calib_params])) - return path + return Path('.') / file_name - def write_weight_value_section(self, path='./param_weights.txt') -> str: - with open(path) as f: + def write_weight_value_section(self, file_name='param_weights.txt') -> Path: + with open(self.config_path / file_name, 'w') as f: f.write('\n'.join([f'{cp.realname} | {cp.value}' for cp in self.calib_params])) - return path + return Path('.') / file_name + + @property + def param_section(self) -> str: + return '\n'.join(str(param) for param in self.calib_params) + + @property + def response_section(self) -> str: + return f"{self.cost_function} {self.metrics_file}; OST_NULL 0 1 ' '" @property def tied_response_section(self) -> str: if self.maximize: - return f'neg{self.cost_function} 1 {self.cost_function} wsum -1.0.0' + return f'neg{self.cost_function} 1 {self.cost_function} wsum -1.00' else: return '# nothing to do here' @property - def map_vars_to_template(self) -> Dict[str: str]: + def map_vars_to_template(self): return {'runScript': self.run_script, 'objectiveFun': self.objective_function, 'saveScript': self.save_script, @@ -82,9 +134,35 @@ def map_vars_to_template(self) -> Dict[str: str]: 'paramSection': self.param_section, 'responseSection': self.response_section, 'tiedResponseSection': self.tied_response_section, - 'costFunction': self.cost_function + 'costFunction': f'neg{self.cost_function}' if self.maximize else self.cost_function, + 'weightTemplateFile': self.weightTemplateFile, + 'weightValueFile': self.weightValueFile + } + + @property + def map_vars_to_save_template(self): + return { + 'pythonPath': self.python_path, + 'saveDir': self.config_path.parent / 'best_calibration', + 'modelDir': self.config_path} + + @property + def map_vars_to_run_template(self): + return { + 'pythonPath': self.python_path, + 'summaExe': self.summa, + 'fileManager': self.simulation.manager_path, + 'obsDataFile': self.obs_data_file, + 'simVarName': self.sim_calib_var, + 'obsVarName': self.obs_calib_var, + 'outFile': self.metrics_file, + 'paramMappingFile': self.weightTemplateFile, + 'paramWeightFile': self.weightValueFile, + 'paramFile': (self.simulation.manager['settings_path'].value + + self.simulation.manager['parameter_trial'].value), } + class OstrichParam(): def __init__(self, name, value, val_range): @@ -94,4 +172,8 @@ def __init__(self, name, value, val_range): self.lower, self.upper = val_range def __str__(self): - return f"{self.weightname} {self.value} {self.lower} none none none free" + return f"{self.weightname} {self.value} {self.lower} {self.upper} none none none free" + + +def read_ostrich_params(path): + pass diff --git a/pysumma/evaluation.py b/pysumma/evaluation.py index d13c8126..9109695b 100644 --- a/pysumma/evaluation.py +++ b/pysumma/evaluation.py @@ -2,6 +2,15 @@ import math import numpy as np +def trim_time(sim, obs): + sim_start = sim['time'].values[0] + sim_stop = sim['time'].values[-1] + obs_start = obs['time'].values[0] + obs_stop = obs['time'].values[-1] + start = max(sim_start, obs_start) + stop = min(sim_stop, obs_stop) + return slice(start, stop) + def kling_gupta_efficiency(sim, obs): obs = np.asarray(obs) diff --git a/pysumma/simulation.py b/pysumma/simulation.py index f287372c..d3d211e9 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -50,14 +50,14 @@ class Simulation(): InitialConditions object (populated after calling ``initialize``) """ - def __init__(self, executable, filemanager, initialize=True): + def __init__(self, executable, filemanager, initialize=True, config_dir='.pysumma'): """Initialize a new simulation object""" self.stdout = None self.stderr = None self.process = None self.executable = executable self.manager_path = Path(os.path.abspath(os.path.realpath(filemanager))) - self.config_path = self.manager_path.parent / '.pysumma' + self.config_path = self.manager_path.parent / config_dir self.status = 'Uninitialized' if initialize: self.initialize() @@ -330,7 +330,7 @@ def monitor(self): return self.status - def _write_configuration(self, name, write_netcdf: str=False): + def _write_configuration(self, name=''): self.config_path = self.config_path / name self.config_path.mkdir(parents=True, exist_ok=True) manager_path = str(self.manager_path.parent) From 3f000968396075bb9a8321ca2c59d8dddc21d62a Mon Sep 17 00:00:00 2001 From: arbennett Date: Wed, 20 May 2020 07:08:57 -0700 Subject: [PATCH 25/36] Add Ostrich documentation --- pysumma/calibration/ostrich.py | 113 ++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 15 deletions(-) diff --git a/pysumma/calibration/ostrich.py b/pysumma/calibration/ostrich.py index 09447aa9..2bb19752 100644 --- a/pysumma/calibration/ostrich.py +++ b/pysumma/calibration/ostrich.py @@ -1,13 +1,14 @@ -from pkg_resources import resource_filename as resource +import os +import numpy as np +import shutil +import stat +import subprocess from functools import partial -from string import Template -from typing import List, Dict from pathlib import Path +from pkg_resources import resource_filename as resource from pysumma import Simulation -import subprocess -import shutil -import stat -import os +from string import Template +from typing import List, Dict def read_template(path): with open(path, 'r') as f: @@ -16,18 +17,80 @@ def read_template(path): resource = partial(resource, __name__) +# Paths to template files INPT_FILE = resource('meta/ostIn.template') EXEC_FILE = resource('meta/model_executable.template') SAVE_FILE = resource('meta/save_parameters.template') +# Templates INPT_META = read_template(INPT_FILE) EXEC_META = read_template(EXEC_FILE) SAVE_META = read_template(SAVE_FILE) class Ostrich(): + """ + Provides a high level interface to the OSTRICH optimization package. + This class can currently only be used for single-objective optimization + using the DDS algorithm as defined in the template file. Currently the + metrics calculated are KGE, MAE, and MSE as defined in the evaluation + package, though more metrics can be implemmented quite easily. + + A basic workflow for this object is: + + :: + import pysumma as ps + summa_exe = './summa.exe' + ostrich_exe = './ostrich.exe' + file_manager = './file_manager.txt' + python_exe = '/pool0/data/andrbenn/.conda/all/bin/python' + ostrich = ps.Ostrich(ostrich_exe, summa_exe, file_manager, python_path=python_exe) + ostrich.calib_params = [ + ps.OstrichParam('paramName', starValue, (minValue, maxValue)), + ] + ostrich.obs_data_file = 'obs_data.nc' + ostrich.sim_calib_var = 'sim_varname' + ostrich.obs_calib_var = 'obs_varname' + ostrich.write_config() + ostrich.run() + + Attributes + ---------- + ostrich: + Path to OSTRICH executable + python_path: + Path to Python executable used for the ``run_script`` + summa: + Path to the SUMMA executable + template: + OSTRICH configuration file template + save_template: + Template for script to save best parameters + run_template: + Template for script to run and evaluate SUMMA + config_path: + Path to location of calibration runs/logs + simulation: + pysumma Simulation object used as template + file_manager: + File manager file for SUMMA simulation + seed: + Random seed for calibration + errval: + Error value for OSTRICH + perturb_val: + Strength of parameter perturbations during calibration + max_iters: + Number of calibration trial runs + cost_function: + Metric to use when ranking calibration runs + maximize: + Whether to maximize the ``cost_function`` + """ def __init__(self, ostrich_executable, summa_executable, file_manager, python_path='python'): + """Initialize a new Ostrich object""" + self.available_metrics: np.ndarray = np.array(['KGE', 'MAE', 'RMSE']) self.ostrich: str = ostrich_executable self.python_path: str = python_path self.summa: str = summa_executable @@ -52,6 +115,7 @@ def __init__(self, ostrich_executable, summa_executable, file_manager, python_pa self.maximize: bool = True def run(self, prerun_cmds=[]): + """Start calibration run""" if len(prerun_cmds): preprocess_cmd = " && ".join(prerun_cmds) + " && " else: @@ -64,10 +128,8 @@ def run(self, prerun_cmds=[]): self.stderr = self.stderr.decode('utf-8', 'ignore') self.stdout = self.stdout.decode('utf-8', 'ignore') - def read_config(self, config_file): - raise NotImplementedError('We do not yet support importing OSTRICH configurations!') - def write_config(self): + """Writes all necessary files for calibration""" if not os.path.exists(self.config_path): os.mkdir(self.config_path) @@ -95,12 +157,14 @@ def write_config(self): os.chmod(self.save_script, st.st_mode | stat.S_IEXEC) def write_weight_template_section(self, file_name=Path('param_mapping.tpl')) -> Path: + """Write the parameter name mapping for OSTRICH""" with open(self.config_path / file_name, 'w') as f: f.write('\n'.join([f'{cp.realname} | {cp.weightname}' for cp in self.calib_params])) return Path('.') / file_name def write_weight_value_section(self, file_name='param_weights.txt') -> Path: + """Write the parameter values for OSTRICH""" with open(self.config_path / file_name, 'w') as f: f.write('\n'.join([f'{cp.realname} | {cp.value}' for cp in self.calib_params])) @@ -108,14 +172,18 @@ def write_weight_value_section(self, file_name='param_weights.txt') -> Path: @property def param_section(self) -> str: + """Write list of calibration parameters""" return '\n'.join(str(param) for param in self.calib_params) @property def response_section(self) -> str: - return f"{self.cost_function} {self.metrics_file}; OST_NULL 0 1 ' '" + """Write section of OSTRICH configuration for selecting metric""" + metric_row = np.argwhere(self.cost_function == self.available_metrics)[0][0] + return f"{self.cost_function} {self.metrics_file}; OST_NULL {metric_row} 1 ' '" @property def tied_response_section(self) -> str: + """Write section for determining if we are maximizing or minimizing the metric""" if self.maximize: return f'neg{self.cost_function} 1 {self.cost_function} wsum -1.00' else: @@ -123,6 +191,7 @@ def tied_response_section(self) -> str: @property def map_vars_to_template(self): + """For completion of the OSTRICH input template""" return {'runScript': self.run_script, 'objectiveFun': self.objective_function, 'saveScript': self.save_script, @@ -141,6 +210,7 @@ def map_vars_to_template(self): @property def map_vars_to_save_template(self): + """For completion of the parameter saving template""" return { 'pythonPath': self.python_path, 'saveDir': self.config_path.parent / 'best_calibration', @@ -148,6 +218,7 @@ def map_vars_to_save_template(self): @property def map_vars_to_run_template(self): + """For completion of the model run script template""" return { 'pythonPath': self.python_path, 'summaExe': self.summa, @@ -164,6 +235,22 @@ def map_vars_to_run_template(self): class OstrichParam(): + """ + Definition of a SUMMA parameter to be optimized by OSTRICH + + Parameters + ---------- + realname: + Parameter name as seen by SUMMA + weightname: + Parameter name as seen by OSTRICH + value: + Default value + lower: + Lower bound for parameter value + upper: + Upper bound for parameter value + """ def __init__(self, name, value, val_range): self.realname = name @@ -173,7 +260,3 @@ def __init__(self, name, value, val_range): def __str__(self): return f"{self.weightname} {self.value} {self.lower} {self.upper} none none none free" - - -def read_ostrich_params(path): - pass From 7221cc2e0e415308ab94f6a7d2af7de6ffdf05fa Mon Sep 17 00:00:00 2001 From: arbennett Date: Wed, 27 May 2020 15:17:50 -0700 Subject: [PATCH 26/36] Allow for tied parameters to constrain optimization --- .../meta/model_executable.template | 7 ++- pysumma/calibration/meta/ostIn.template | 9 +++ pysumma/calibration/ostrich.py | 58 +++++++++++++++++-- pysumma/evaluation.py | 8 +-- 4 files changed, 72 insertions(+), 10 deletions(-) diff --git a/pysumma/calibration/meta/model_executable.template b/pysumma/calibration/meta/model_executable.template index e5c16c6f..b6a65956 100644 --- a/pysumma/calibration/meta/model_executable.template +++ b/pysumma/calibration/meta/model_executable.template @@ -15,6 +15,7 @@ if __name__ == '__main__': param_mapping_file = '$paramMappingFile' param_weight_file = '$paramWeightFile' param_file = '$paramFile' + simulation_args = $simulationArgs # read in parameters from ostrich files and summa setup with xr.open_dataset(param_file) as temp: @@ -32,10 +33,14 @@ if __name__ == '__main__': trial_params.to_netcdf(param_file) # initialize simulation object - s = ps.Simulation(summa_exe, file_manager) + s = ps.Simulation(summa_exe, file_manager, **simulation_args) # run the simulation s.run('local') + if s.status != 'Success': + print(s.stdout) + print('--------------------------------------------') + print(s.stderr) assert s.status == 'Success' # open output and calculate diagnostics diff --git a/pysumma/calibration/meta/ostIn.template b/pysumma/calibration/meta/ostIn.template index f3baaa15..9e3362dd 100644 --- a/pysumma/calibration/meta/ostIn.template +++ b/pysumma/calibration/meta/ostIn.template @@ -28,6 +28,15 @@ BeginParams $paramSection EndParams +# ------------------------------------------------- +# Parameter/DV Specification - note: what are txInN, txOst ...? +# Format: +# parameter start minimum maximum txInN txOst txOut fmt +# ------------------------------------------------- +BeginTiedParams +$tiedParamSection +EndTiedParams + # ------------------------------------------------- # Response variable specification - note: keyworld should aways be OST_NULL # Format: diff --git a/pysumma/calibration/ostrich.py b/pysumma/calibration/ostrich.py index 2bb19752..42f6f2de 100644 --- a/pysumma/calibration/ostrich.py +++ b/pysumma/calibration/ostrich.py @@ -86,6 +86,8 @@ class Ostrich(): Metric to use when ranking calibration runs maximize: Whether to maximize the ``cost_function`` + simulation_kwargs: + Keyword arguments to pass to the simulation run function """ def __init__(self, ostrich_executable, summa_executable, file_manager, python_path='python'): @@ -110,9 +112,11 @@ def __init__(self, ostrich_executable, summa_executable, file_manager, python_pa self.perturb_val: float = 0.2 self.max_iters: int = 100 self.calib_params: List[OstrichParam] = [] + self.tied_params: List[OstrichTiedParam] = [] self.cost_function: str = 'KGE' self.objective_function: str = 'gcop' self.maximize: bool = True + self.simulation_kwargs: Dict = {} def run(self, prerun_cmds=[]): """Start calibration run""" @@ -159,22 +163,38 @@ def write_config(self): def write_weight_template_section(self, file_name=Path('param_mapping.tpl')) -> Path: """Write the parameter name mapping for OSTRICH""" with open(self.config_path / file_name, 'w') as f: - f.write('\n'.join([f'{cp.realname} | {cp.weightname}' - for cp in self.calib_params])) + for cp in self.calib_params: + if cp.weightname.endswith('mtp'): + f.write(f'{cp.realname} | {cp.weightname}\n') + for tp in self.tied_params: + if tp.realname.endswith('mtp'): + f.write(f'{tp.realname.replace("_mtp", "")} | {tp.realname}\n') return Path('.') / file_name def write_weight_value_section(self, file_name='param_weights.txt') -> Path: """Write the parameter values for OSTRICH""" with open(self.config_path / file_name, 'w') as f: f.write('\n'.join([f'{cp.realname} | {cp.value}' - for cp in self.calib_params])) + for cp in self.calib_params]) + '\n') return Path('.') / file_name + def add_tied_param(self, param_name, lower_bound, upper_bound): + self.calib_params.append(OstrichParam(f'{param_name}', 0.5, (0.01, 0.99), weightname=f'{param_name}_scale')) + self.tied_params.append(OstrichTiedParam(param_name, lower_bound, upper_bound)) + @property def param_section(self) -> str: """Write list of calibration parameters""" return '\n'.join(str(param) for param in self.calib_params) + @property + def tied_param_section(self) -> str: + """Write list of tied calibration parameters""" + if len(self.tied_params): + return '\n'.join(str(param) for param in self.tied_params) + else: + return '# nothing to do here' + @property def response_section(self) -> str: """Write section of OSTRICH configuration for selecting metric""" @@ -201,6 +221,7 @@ def map_vars_to_template(self): 'perturbVal': self.perturb_val, 'maxIters': self.max_iters, 'paramSection': self.param_section, + 'tiedParamSection': self.tied_param_section, 'responseSection': self.response_section, 'tiedResponseSection': self.tied_response_section, 'costFunction': f'neg{self.cost_function}' if self.maximize else self.cost_function, @@ -229,6 +250,7 @@ def map_vars_to_run_template(self): 'outFile': self.metrics_file, 'paramMappingFile': self.weightTemplateFile, 'paramWeightFile': self.weightValueFile, + 'simulationArgs': self.simulation_kwargs, 'paramFile': (self.simulation.manager['settings_path'].value + self.simulation.manager['parameter_trial'].value), } @@ -252,11 +274,37 @@ class OstrichParam(): Upper bound for parameter value """ - def __init__(self, name, value, val_range): + def __init__(self, name, value, val_range, weightname=''): self.realname = name - self.weightname = f'{name}_mtp' + if not weightname: + self.weightname = f'{name}_mtp' + else: + self.weightname = weightname self.value = value self.lower, self.upper = val_range def __str__(self): return f"{self.weightname} {self.value} {self.lower} {self.upper} none none none free" + + +class OstrichTiedParam(): + def __init__(self, name, lower_param, upper_param): + self.realname = f'{name}_mtp' + self.weightname = f'{name}_scale' + self.lower_param = f'{lower_param}_mtp' + self.upper_param = f'{upper_param}_mtp' + + @property + def type_data(self): + """This corresponds to the equation y = x2 + x1x3 - x1x2""" + if self.lower_param and self.upper_param: + return "ratio 0 -1 1 0 0 1 0 0 0 0 0 0 0 0 0 1 free" + elif self.lower_param: + raise NotImplementedError() + return "" + elif self.upper_param: + raise NotImplementedError() + return "" + + def __str__(self): + return f"{self.realname} 3 {self.weightname} {self.lower_param} {self.upper_param} {self.type_data}" diff --git a/pysumma/evaluation.py b/pysumma/evaluation.py index 9109695b..8d924069 100644 --- a/pysumma/evaluation.py +++ b/pysumma/evaluation.py @@ -3,10 +3,10 @@ import numpy as np def trim_time(sim, obs): - sim_start = sim['time'].values[0] - sim_stop = sim['time'].values[-1] - obs_start = obs['time'].values[0] - obs_stop = obs['time'].values[-1] + sim_start = sim['time'].values[1] + sim_stop = sim['time'].values[-2] + obs_start = obs['time'].values[1] + obs_stop = obs['time'].values[-2] start = max(sim_start, obs_start) stop = min(sim_stop, obs_stop) return slice(start, stop) From e56b50862b14fe9a8b14b0aee15d9d2912c4f6d7 Mon Sep 17 00:00:00 2001 From: arbennett Date: Wed, 27 May 2020 16:24:50 -0700 Subject: [PATCH 27/36] Add capability to add a simple conversion function to the run script --- pysumma/calibration/meta/model_executable.template | 7 ++++--- pysumma/calibration/ostrich.py | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pysumma/calibration/meta/model_executable.template b/pysumma/calibration/meta/model_executable.template index b6a65956..fcc038cd 100644 --- a/pysumma/calibration/meta/model_executable.template +++ b/pysumma/calibration/meta/model_executable.template @@ -16,6 +16,7 @@ if __name__ == '__main__': param_weight_file = '$paramWeightFile' param_file = '$paramFile' simulation_args = $simulationArgs + conversion = $conversionFunc # read in parameters from ostrich files and summa setup with xr.open_dataset(param_file) as temp: @@ -52,9 +53,9 @@ if __name__ == '__main__': sim_ds = sim_ds.sel(time=time_slice) obs_ds = obs_ds.sel(time=time_slice) - kge = pse.kling_gupta_efficiency(sim_ds[sim_calib_var], obs_ds[obs_calib_var]) - mae = pse.mean_absolute_error(sim_ds[sim_calib_var], obs_ds[obs_calib_var]) - rmse = pse.root_mean_square_error(sim_ds[sim_calib_var], obs_ds[obs_calib_var]) + kge = pse.kling_gupta_efficiency(sim_ds[sim_calib_var], conversion(obs_ds[obs_calib_var])) + mae = pse.mean_absolute_error(sim_ds[sim_calib_var], conversion(obs_ds[obs_calib_var])) + rmse = pse.root_mean_square_error(sim_ds[sim_calib_var], conversion(obs_ds[obs_calib_var])) # save diagnostics in form that ostrich can read with open(out_file, 'w+') as f: diff --git a/pysumma/calibration/ostrich.py b/pysumma/calibration/ostrich.py index 42f6f2de..798b4a3f 100644 --- a/pysumma/calibration/ostrich.py +++ b/pysumma/calibration/ostrich.py @@ -2,6 +2,7 @@ import numpy as np import shutil import stat +import inspect import subprocess from functools import partial from pathlib import Path @@ -106,6 +107,7 @@ def __init__(self, ostrich_executable, summa_executable, file_manager, python_pa self.run_script: Path = self.config_path / 'run_script.py' self.save_script: Path = self.config_path / 'save_script.py' self.metrics_file: Path = self.config_path / 'metrics.txt' + self.conversion_function: callable = lambda x: x self.preserve_output: str ='no' self.seed: int = 42 self.errval: float = -9999 @@ -248,6 +250,7 @@ def map_vars_to_run_template(self): 'simVarName': self.sim_calib_var, 'obsVarName': self.obs_calib_var, 'outFile': self.metrics_file, + 'conversionFunc': inspect.getsource(self.conversion_function).split('=')[-1], 'paramMappingFile': self.weightTemplateFile, 'paramWeightFile': self.weightValueFile, 'simulationArgs': self.simulation_kwargs, From 738dafcc03e6a5cfb5b5e2bb76e01fb74056ef23 Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 28 May 2020 16:00:07 -0700 Subject: [PATCH 28/36] Add ability to set a filter function for calibration analysis --- pysumma/calibration/meta/model_executable.template | 8 +++++--- pysumma/calibration/meta/save_parameters.template | 4 ++-- pysumma/calibration/ostrich.py | 7 ++++++- pysumma/evaluation.py | 4 +++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pysumma/calibration/meta/model_executable.template b/pysumma/calibration/meta/model_executable.template index fcc038cd..b49181e0 100644 --- a/pysumma/calibration/meta/model_executable.template +++ b/pysumma/calibration/meta/model_executable.template @@ -3,6 +3,7 @@ import pysumma as ps import pysumma.evaluation as pse import shutil import xarray as xr +$importStrings if __name__ == '__main__': # Template variables @@ -17,6 +18,7 @@ if __name__ == '__main__': param_file = '$paramFile' simulation_args = $simulationArgs conversion = $conversionFunc + filter = $filterFunc # read in parameters from ostrich files and summa setup with xr.open_dataset(param_file) as temp: @@ -53,9 +55,9 @@ if __name__ == '__main__': sim_ds = sim_ds.sel(time=time_slice) obs_ds = obs_ds.sel(time=time_slice) - kge = pse.kling_gupta_efficiency(sim_ds[sim_calib_var], conversion(obs_ds[obs_calib_var])) - mae = pse.mean_absolute_error(sim_ds[sim_calib_var], conversion(obs_ds[obs_calib_var])) - rmse = pse.root_mean_square_error(sim_ds[sim_calib_var], conversion(obs_ds[obs_calib_var])) + kge = pse.kling_gupta_efficiency( filter(sim_ds[sim_calib_var]), filter(conversion(obs_ds[obs_calib_var]))) + mae = pse.mean_absolute_error( filter(sim_ds[sim_calib_var]), filter(conversion(obs_ds[obs_calib_var]))) + rmse = pse.root_mean_square_error(filter(sim_ds[sim_calib_var]), filter(conversion(obs_ds[obs_calib_var]))) # save diagnostics in form that ostrich can read with open(out_file, 'w+') as f: diff --git a/pysumma/calibration/meta/save_parameters.template b/pysumma/calibration/meta/save_parameters.template index 640ff3aa..40fc684f 100644 --- a/pysumma/calibration/meta/save_parameters.template +++ b/pysumma/calibration/meta/save_parameters.template @@ -7,6 +7,6 @@ import os if __name__ == '__main__': save_dir = '$saveDir' model_dir = '$modelDir' - if not os.path.exists(save_dir): - os.mkdir(save_dir) + if os.path.exists(save_dir): + shutil.rmtree(save_dir) shutil.copytree(model_dir, save_dir) diff --git a/pysumma/calibration/ostrich.py b/pysumma/calibration/ostrich.py index 798b4a3f..a4f258b0 100644 --- a/pysumma/calibration/ostrich.py +++ b/pysumma/calibration/ostrich.py @@ -107,7 +107,9 @@ def __init__(self, ostrich_executable, summa_executable, file_manager, python_pa self.run_script: Path = self.config_path / 'run_script.py' self.save_script: Path = self.config_path / 'save_script.py' self.metrics_file: Path = self.config_path / 'metrics.txt' + self.impot_strings: str = '' self.conversion_function: callable = lambda x: x + self.filter_function: callable = lambda x: x self.preserve_output: str ='no' self.seed: int = 42 self.errval: float = -9999 @@ -250,7 +252,9 @@ def map_vars_to_run_template(self): 'simVarName': self.sim_calib_var, 'obsVarName': self.obs_calib_var, 'outFile': self.metrics_file, - 'conversionFunc': inspect.getsource(self.conversion_function).split('=')[-1], + 'importStrings': self.import_strings, + 'conversionFunc': "=".join(inspect.getsource(self.conversion_function).split('=')[1:]), + 'filterFunc': "=".join(inspect.getsource(self.filter_function).split('=')[1:]), 'paramMappingFile': self.weightTemplateFile, 'paramWeightFile': self.weightValueFile, 'simulationArgs': self.simulation_kwargs, @@ -270,6 +274,7 @@ class OstrichParam(): weightname: Parameter name as seen by OSTRICH value: + Default value lower: Lower bound for parameter value diff --git a/pysumma/evaluation.py b/pysumma/evaluation.py index 8d924069..04abc63f 100644 --- a/pysumma/evaluation.py +++ b/pysumma/evaluation.py @@ -2,7 +2,9 @@ import math import numpy as np -def trim_time(sim, obs): +def trim_time(sim, obs, roundto='min'): + sim['time'] = sim['time'].dt.round(roundto) + obs['time'] = obs['time'].dt.round(roundto) sim_start = sim['time'].values[1] sim_stop = sim['time'].values[-2] obs_start = obs['time'].values[1] From 801a8aee4b0f56508814eb36d2a19be5c1d40542 Mon Sep 17 00:00:00 2001 From: Andrew Bennett Date: Sat, 13 Jun 2020 12:49:41 -0700 Subject: [PATCH 29/36] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 38474d6c..6ced74b6 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ 'hs_restclient', 'distributed', 'fiona', + 'cartopy', 'netcdf4' ], include_package_data=True, From e3c33e75c316c7ecd9d0de4035da88283d44a20c Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 16 Jul 2020 16:44:04 -0700 Subject: [PATCH 30/36] Line endings --- pysumma/evaluation.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pysumma/evaluation.py b/pysumma/evaluation.py index 04abc63f..21f912a9 100644 --- a/pysumma/evaluation.py +++ b/pysumma/evaluation.py @@ -30,5 +30,32 @@ def kling_gupta_efficiency(sim, obs): return kge +def decomposed_kling_gupta_efficiency(sim, obs): + obs = np.asarray(obs) + sim = np.asarray(sim) + obs_filtered = obs[~np.isnan(obs)] + sim_filtered = sim[~np.isnan(obs)] + sim_std = np.std(sim_filtered, ddof=1) + obs_std = np.std(obs_filtered, ddof=1) + sim_mu = np.mean(sim_filtered) + obs_mu = np.mean(obs_filtered) + r = np.corrcoef(sim_filtered, obs_filtered)[0, 1] + var = sim_std / obs_std + bias = sim_mu / obs_mu + kge = 1 - np.sqrt((bias-1)**2 + (var-1)**2 + (r-1)**2) + return kge, bias, var, r + + +def nash_sutcliffe_efficiency(sim, obs): + obs = np.asarray(obs) + sim = np.asarray(sim) + obs_filtered = obs[~np.isnan(obs)] + sim_filtered = sim[~np.isnan(obs)] + obs_mu = np.mean(obs_filtered) + num = np.sum( (sim_filtered - obs_filtered) ** 2 ) + den = np.sum( (obs_filtered - obs_mu) ** 2 ) + return 1 - (num / den) + + def root_mean_square_error(sim, obs): return np.sqrt(mean_squared_error(sim, obs)) From b06521875af14099b96ae604e56e5e1f74c75321 Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 16 Jul 2020 17:31:12 -0700 Subject: [PATCH 31/36] Update file manager and decisions for 3.0.0 --- pysumma/decisions.py | 6 +---- pysumma/file_manager.py | 48 +++++++++++++++++----------------- pysumma/meta/decisions.json | 13 --------- pysumma/meta/file_manager.json | 18 ++++++++++++- 4 files changed, 42 insertions(+), 43 deletions(-) diff --git a/pysumma/decisions.py b/pysumma/decisions.py index b7f18c23..7af9f090 100644 --- a/pysumma/decisions.py +++ b/pysumma/decisions.py @@ -22,11 +22,7 @@ def __init__(self, name, value): self.set_value(value) def set_value(self, new_value): - datestring = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}" - if (self.name in ['simulStart', 'simulFinsh'] and - re.match(datestring, new_value) is not None): - self.value = new_value - elif new_value in self.available_options: + if new_value in self.available_options: self.value = new_value else: raise ValueError(os.linesep.join([ diff --git a/pysumma/file_manager.py b/pysumma/file_manager.py index 77260ceb..1d01d855 100644 --- a/pysumma/file_manager.py +++ b/pysumma/file_manager.py @@ -31,7 +31,7 @@ def set_value(self, new_value): self.value = new_value def __str__(self): - return "'{}' ! {}".format(self.value, self.name) + return "{} '{}'".format(self.value, self.name) class FileManager(OptionContainer): @@ -48,90 +48,90 @@ def set_option(self, key, value): o.set_value(value) def get_constructor_args(self, line): - return (OPTION_NAMES[self.opt_count], - line.split('!')[0].replace("'", "").strip()) + name, value = line.split('!')[0].strip().split('') + return (name.strip(), value.strip(). replace("'", "").strip()) @property def decisions(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('decisions_path') + p1 = self.get_value('settingsPath') + p2 = self.get_value('decisionsFile') self._decisions = Decisions(p1, p2) return self._decisions @property def output_control(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('output_control') + p1 = self.get_value('settingsPath') + p2 = self.get_value('outputDefFile') self._output_control = OutputControl(p1, p2) return self._output_control @property def local_param_info(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('local_param_info') + p1 = self.get_value('settingsPath') + p2 = self.get_value('hruParamFile') self._local_param_info = LocalParamInfo(p1, p2) return self._local_param_info @property def basin_param_info(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('basin_param_info') + p1 = self.get_value('settingsPath') + p2 = self.get_value('gruParamFile') self._basin_param_info = LocalParamInfo(p1, p2) return self._basin_param_info @property def force_file_list(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('forcing_file_list') - p3 = self.get_value('input_path') + p1 = self.get_value('settingsPath') + p2 = self.get_value('forcingList') + p3 = self.get_value('forcingPath') self._force_file_list = ForceFileList(p1, p2, p3) return self._force_file_list @property def local_attributes(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('local_attributes') + p1 = self.get_value('settingsPath') + p2 = self.get_value('attributeFile') self._local_attrs = xr.open_dataset(p1 + p2) return self._local_attrs @property def parameter_trial(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('parameter_trial') + p1 = self.get_value('settingsPath') + p2 = self.get_value('trialParamFile') self._param_trial = xr.open_dataset(p1 + p2) return self._param_trial @property def initial_conditions(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('model_init_cond') + p1 = self.get_value('settingsPath') + p2 = self.get_value('initCondFile') self._init_cond = xr.open_dataset(p1 + p2) return self._init_cond @property def genparm(self): - p1, p2 = self.get_value('settings_path'), 'GENPARM.TBL' + p1, p2 = self.get_value('settingsPath'), 'GENPARM.TBL' with open(p1 + p2, 'r') as f: self._genparm = f.readlines() return self._genparm @property def mptable(self): - p1, p2 = self.get_value('settings_path'), 'MPTABLE.TBL' + p1, p2 = self.get_value('settingsPath'), 'MPTABLE.TBL' with open(p1 + p2, 'r') as f: self._mptable = f.readlines() return self._mptable @property def soilparm(self): - p1, p2 = self.get_value('settings_path'), 'SOILPARM.TBL' + p1, p2 = self.get_value('settingsPath'), 'SOILPARM.TBL' with open(p1 + p2, 'r') as f: self._soilparm = f.readlines() return self._soilparm @property def vegparm(self): - p1, p2 = self.get_value('settings_path'), 'VEGPARM.TBL' + p1, p2 = self.get_value('settingsPath'), 'VEGPARM.TBL' with open(p1 + p2, 'r') as f: self._vegparm = f.readlines() return self._vegparm diff --git a/pysumma/meta/decisions.json b/pysumma/meta/decisions.json index 7500ebfd..f848b1c0 100644 --- a/pysumma/meta/decisions.json +++ b/pysumma/meta/decisions.json @@ -1,17 +1,4 @@ { - - "simulStart": { - "options": ["YYYY-MM-DD hh:mm"], - "description": "simulation start time" - }, - "simulFinsh": { - "options": ["YYYY-MM-DD hh:mm"], - "description": "simulation end time" - }, - "tmZoneInfo": { - "options": ["localTime", "ncTime", "utcTime"], - "description": "time zone information" - }, "soilCatTbl": { "options": ["STAS", "STAS-RUC", "ROSETTA"], "description": "soil-category dataset" diff --git a/pysumma/meta/file_manager.json b/pysumma/meta/file_manager.json index bbb6e4e8..4a492b6a 100644 --- a/pysumma/meta/file_manager.json +++ b/pysumma/meta/file_manager.json @@ -1,3 +1,19 @@ { - "option_names": ["filemanager_version", "settings_path", "input_path", "output_path", "decisions_path", "meta_time", "meta_attr", "meta_type", "meta_force", "meta_localparam", "output_control", "meta_localindex", "meta_basinparam", "meta_basinmvar", "local_attributes", "local_param_info", "basin_param_info", "forcing_file_list", "model_init_cond", "parameter_trial", "output_prefix"] + "option_names": [ + "simStartTime", + "simEndTime", + "tmZoneInfo", + "settingsPath", + "forcingPath", + "outputPath", + "attributeFile", + "decisionsFile", + "hruParamFile", + "gruParamFile", + "forcingList", + "outputDefFile", + "initCondFile", + "trialParamFile", + "outFilePrefix" + ] } From 49e4b6037127fe8acb14c7b2f8292cd371cd632c Mon Sep 17 00:00:00 2001 From: arbennett Date: Thu, 16 Jul 2020 17:35:03 -0700 Subject: [PATCH 32/36] Add check for fman version --- pysumma/file_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysumma/file_manager.py b/pysumma/file_manager.py index 1d01d855..75ead0a2 100644 --- a/pysumma/file_manager.py +++ b/pysumma/file_manager.py @@ -42,6 +42,8 @@ class FileManager(OptionContainer): def __init__(self, path, name): super().__init__(FileManagerOption, path, name) + assert self.get_option('controlVersion') == 'SUMMA_FILE_MANAGER_V3.0.0' + def set_option(self, key, value): o = self.get_option(key) From ffaf47ec5965d6722f046bce5c78df70dd732313 Mon Sep 17 00:00:00 2001 From: arbennett Date: Sun, 19 Jul 2020 10:32:19 -0700 Subject: [PATCH 33/36] Start of update for file manager --- pysumma/file_manager.py | 54 ++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/pysumma/file_manager.py b/pysumma/file_manager.py index 77260ceb..e05e4d38 100644 --- a/pysumma/file_manager.py +++ b/pysumma/file_manager.py @@ -66,72 +66,72 @@ def output_control(self): return self._output_control @property - def local_param_info(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('local_param_info') - self._local_param_info = LocalParamInfo(p1, p2) - return self._local_param_info + def global_hru_params(self): + p1 = self.get_value('settingsPath') + p2 = self.get_value('globalHruParams') + self._hru_params = GlobalParams(p1, p2) + return self._hru_params @property - def basin_param_info(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('basin_param_info') - self._basin_param_info = LocalParamInfo(p1, p2) - return self._basin_param_info + def global_gru_params(self): + p1 = self.get_value('settingsPath') + p2 = self.get_value('GlobalGruParams') + self._gru_params = GlobalParams(p1, p2) + return self._gru_params @property def force_file_list(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('forcing_file_list') - p3 = self.get_value('input_path') - self._force_file_list = ForceFileList(p1, p2, p3) + p1 = self.get_value('settingsPath') + p2 = self.get_value('forcingList') + p3 = self.get_value('forcingPath') + self._force_file_list = ForcingList(p1, p2, p3) return self._force_file_list @property def local_attributes(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('local_attributes') + p1 = self.get_value('settingsPath') + p2 = self.get_value('attributeFile') self._local_attrs = xr.open_dataset(p1 + p2) return self._local_attrs @property - def parameter_trial(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('parameter_trial') - self._param_trial = xr.open_dataset(p1 + p2) - return self._param_trial + def spatial_paramr(self): + p1 = self.get_value('settingsPath') + p2 = self.get_value('spatialParams') + self._spatial_params = xr.open_dataset(p1 + p2) + return self._spatial_params @property def initial_conditions(self): - p1 = self.get_value('settings_path') - p2 = self.get_value('model_init_cond') + p1 = self.get_value('settingsPath') + p2 = self.get_value('initCondFile') self._init_cond = xr.open_dataset(p1 + p2) return self._init_cond @property def genparm(self): - p1, p2 = self.get_value('settings_path'), 'GENPARM.TBL' + p1, p2 = self.get_value('settingsPath'), 'GENPARM.TBL' with open(p1 + p2, 'r') as f: self._genparm = f.readlines() return self._genparm @property def mptable(self): - p1, p2 = self.get_value('settings_path'), 'MPTABLE.TBL' + p1, p2 = self.get_value('settingsPath'), 'MPTABLE.TBL' with open(p1 + p2, 'r') as f: self._mptable = f.readlines() return self._mptable @property def soilparm(self): - p1, p2 = self.get_value('settings_path'), 'SOILPARM.TBL' + p1, p2 = self.get_value('settingsPath'), 'SOILPARM.TBL' with open(p1 + p2, 'r') as f: self._soilparm = f.readlines() return self._soilparm @property def vegparm(self): - p1, p2 = self.get_value('settings_path'), 'VEGPARM.TBL' + p1, p2 = self.get_value('settingsPath'), 'VEGPARM.TBL' with open(p1 + p2, 'r') as f: self._vegparm = f.readlines() return self._vegparm From 3db3c70808fb187e7da2a04afad3f6394e28898b Mon Sep 17 00:00:00 2001 From: arbennett Date: Sun, 19 Jul 2020 11:58:38 -0700 Subject: [PATCH 34/36] Update for summa 3.0.0 --- pysumma/__init__.py | 4 +- pysumma/file_manager.py | 20 ++++--- pysumma/force_file_list.py | 6 +- .../{local_param_info.py => global_params.py} | 8 +-- pysumma/meta/file_manager.json | 8 +-- pysumma/option.py | 5 +- pysumma/output_control.py | 7 ++- pysumma/simulation.py | 58 +++++++++---------- pysumma/{param_trial.py => spatial_params.py} | 0 9 files changed, 62 insertions(+), 54 deletions(-) rename pysumma/{local_param_info.py => global_params.py} (91%) rename pysumma/{param_trial.py => spatial_params.py} (100%) diff --git a/pysumma/__init__.py b/pysumma/__init__.py index e4cf1fd0..627d4c41 100644 --- a/pysumma/__init__.py +++ b/pysumma/__init__.py @@ -4,7 +4,7 @@ from .file_manager import FileManager from .decisions import Decisions from .output_control import OutputControl -from .local_param_info import LocalParamInfo -from .force_file_list import ForceFileList +from .global_params import GlobalParams +from .force_file_list import ForcingList from . import utils from .calibration import Ostrich, OstrichParam diff --git a/pysumma/file_manager.py b/pysumma/file_manager.py index a63b32e3..f50fc77f 100644 --- a/pysumma/file_manager.py +++ b/pysumma/file_manager.py @@ -7,8 +7,8 @@ from .option import BaseOption, OptionContainer from .decisions import Decisions from .output_control import OutputControl -from .local_param_info import LocalParamInfo -from .force_file_list import ForceFileList +from .global_params import GlobalParams +from .force_file_list import ForcingList # Option names for the file manager, this is just a list, # as the order of these values matters. They may also not be @@ -31,7 +31,7 @@ def set_value(self, new_value): self.value = new_value def __str__(self): - return "{} '{}'".format(self.value, self.name) + return "{} '{}'".format(self.name.ljust(36), self.value) class FileManager(OptionContainer): @@ -42,16 +42,18 @@ class FileManager(OptionContainer): def __init__(self, path, name): super().__init__(FileManagerOption, path, name) - assert self.get_option('controlVersion') == 'SUMMA_FILE_MANAGER_V3.0.0' - + assert self.get_value('controlVersion') == 'SUMMA_FILE_MANAGER_V3.0.0' def set_option(self, key, value): o = self.get_option(key) o.set_value(value) def get_constructor_args(self, line): - name, value = line.split('!')[0].strip().split('') - return (name.strip(), value.strip(). replace("'", "").strip()) + name, *value = line.split('!')[0].strip().split() + if isinstance(value, list): + value = " ".join(value).replace("'", "") + print(name, value) + return (name.strip(), value.strip().replace("'", "").strip()) @property def decisions(self): @@ -63,7 +65,7 @@ def decisions(self): @property def output_control(self): p1 = self.get_value('settingsPath') - p2 = self.get_value('outputDefFile') + p2 = self.get_value('outputControl') self._output_control = OutputControl(p1, p2) return self._output_control @@ -77,7 +79,7 @@ def global_hru_params(self): @property def global_gru_params(self): p1 = self.get_value('settingsPath') - p2 = self.get_value('GlobalGruParams') + p2 = self.get_value('globalGruParams') self._gru_params = GlobalParams(p1, p2) return self._gru_params diff --git a/pysumma/force_file_list.py b/pysumma/force_file_list.py index 4a2f1980..872b91e8 100644 --- a/pysumma/force_file_list.py +++ b/pysumma/force_file_list.py @@ -5,7 +5,7 @@ from .option import OptionContainer -class ForceFileListOption(BaseOption): +class ForcingOption(BaseOption): def __init__(self, name): super().__init__(name) @@ -26,13 +26,13 @@ def __str__(self): return "'{}'".format(self.name.split('/')[-1]) -class ForceFileList(OptionContainer): +class ForcingList(OptionContainer): prefix: str = '' def __init__(self, dirpath, filepath, force_file_prefix_path): self.prefix = str(force_file_prefix_path) - super().__init__(ForceFileListOption, dirpath, filepath) + super().__init__(ForcingOption, dirpath, filepath) def set_option(self, key, value): o = self.get_option(key) diff --git a/pysumma/local_param_info.py b/pysumma/global_params.py similarity index 91% rename from pysumma/local_param_info.py rename to pysumma/global_params.py index 96e7c718..4321860b 100644 --- a/pysumma/local_param_info.py +++ b/pysumma/global_params.py @@ -4,7 +4,7 @@ from .option import OptionContainer -class LocalParamOption(BaseOption): +class GlobalParamOption(BaseOption): def __init__(self, name, default, low, high): super().__init__(name) @@ -28,13 +28,13 @@ def _to_string(val): self.name, *map(_to_string, self.value))) -class LocalParamInfo(OptionContainer): +class GlobalParams(OptionContainer): fmt_strings = ["'(a25,1x,3(a1,1x,f12.4,1x))'", "'(a25,1x,a1,1x,3(f12.4,1x,a1,1x))'"] def __init__(self, dirpath, filepath): - super().__init__(LocalParamOption, dirpath, filepath) + super().__init__(GlobalParamOption, dirpath, filepath) def set_option(self, key, value): if not isinstance(value, list): @@ -43,7 +43,7 @@ def set_option(self, key, value): o = self.get_option(key) o.set_value(value) except AttributeError as e: - self.options.append(LocalParamOption(key, *value)) + self.options.append(GlobalParamOption(key, *value)) def read(self, path): """Read the configuration and populate the options""" diff --git a/pysumma/meta/file_manager.json b/pysumma/meta/file_manager.json index 4a492b6a..4342ee15 100644 --- a/pysumma/meta/file_manager.json +++ b/pysumma/meta/file_manager.json @@ -8,12 +8,12 @@ "outputPath", "attributeFile", "decisionsFile", - "hruParamFile", - "gruParamFile", + "globalHruParams", + "globalGruParams", "forcingList", - "outputDefFile", + "outputControl", "initCondFile", - "trialParamFile", + "spatialParams", "outFilePrefix" ] } diff --git a/pysumma/option.py b/pysumma/option.py index 537fc8eb..045ff537 100644 --- a/pysumma/option.py +++ b/pysumma/option.py @@ -143,9 +143,10 @@ def read(self, path): with open(path, 'r') as f: self.original_contents = f.readlines() for line in self.original_contents: - if line.startswith('!') and not self.opt_count: + isnt_empty = len(''.join(map(lambda x: x.strip(), line.split() ))) + if line.startswith('!') and not self.opt_count and not isnt_empty: self.header.append(line) - elif not line.startswith('!'): + elif not line.startswith('!') and isnt_empty: self.options.append(self.OptionType( *self.get_constructor_args(line))) self.opt_count += 1 diff --git a/pysumma/output_control.py b/pysumma/output_control.py index 13b586f2..3030711e 100644 --- a/pysumma/output_control.py +++ b/pysumma/output_control.py @@ -27,7 +27,10 @@ class OutputControlOption(BaseOption): def __init__(self, var=None, period=None, sum=0, instant=1, mean=0, variance=0, min=0, max=0, mode=0): self.name = var - self.period = int(period) + if self.name == 'outputPrecision': + self.period = period + else: + self.period = int(period) self.sum = int(sum) self.instant = int(instant) self.mean = int(mean) @@ -64,6 +67,8 @@ def get_print_list(self): self.validate() plist = [self.name.ljust(36), self.period, self.sum, self.instant, self.mean, self.variance, self.min, self.max, self.mode] + if self.name == 'outputPrecision': + plist = plist[0:2] return [str(p) for p in plist] def __str__(self): diff --git a/pysumma/simulation.py b/pysumma/simulation.py index bbda2477..39c5c66f 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -10,8 +10,8 @@ from .decisions import Decisions from .file_manager import FileManager from .output_control import OutputControl -from .local_param_info import LocalParamInfo -from .force_file_list import ForceFileList +from .global_params import GlobalParams +from .force_file_list import ForcingList class Simulation(): @@ -36,14 +36,14 @@ class Simulation(): Decisions object (populated after calling ``initialize``) output_control: OutputControl object (populated after calling ``initialize``) - parameter_trial: - Parameter trial object (populated after calling ``initialize``) + spatial_params: + Spatially distributed parameters (populated after calling ``initialize``) force_file_list: - Forcing file list object (populated after calling ``initialize``) - local_param_info: - LocalParamInfo object (populated after calling ``initialize``) - basin_param_info: - BasinParamInfo object (populated after calling ``initialize``) + ForcingList object (populated after calling ``initialize``) + global_hru_params: + GlobalParams object for hru (populated after calling ``initialize``) + global_gru_params: + GlobalParams object for gru (populated after calling ``initialize``) local_attributes: LocalAttributes object (populated after calling ``initialize``) initial_conditions: @@ -75,10 +75,10 @@ def initialize(self): self.status = 'Initialized' self.decisions = self.manager.decisions self.output_control = self.manager.output_control - self.parameter_trial = self.manager.parameter_trial + self.spatial_params = self.manager.spatial_params self.force_file_list = self.manager.force_file_list - self.local_param_info = self.manager.local_param_info - self.basin_param_info = self.manager.basin_param_info + self.global_hru_params = self.manager.global_hru_params + self.global_gru_params = self.manager.global_gru_params self.local_attributes = self.manager.local_attributes self.initial_conditions = self.manager.initial_conditions self.genparm = self.manager.genparm @@ -114,7 +114,7 @@ def apply_config(self, config: dict): for k, v in config.get('decisions', {}).items(): self.decisions.set_option(k, v) for k, v in config.get('parameters', {}).items(): - self.local_param_info.set_option(k, v) + self.global_hru_params.set_option(k, v) for k, v in config.get('output_control', {}).items(): self.output_control.set_option(k, **v) for k, v in config.get('attributes', {}).items(): @@ -122,7 +122,7 @@ def apply_config(self, config: dict): for k, v in config.get('trial_parameters', {}).items(): self.assign_trial_params(k, v) if self.decisions['snowLayers'] == 'CLM_2010': - self.validate_layer_params(self.local_param_info) + self.validate_layer_params(self.global_hru_params) def assign_attributes(self, name, data): """ @@ -150,7 +150,7 @@ def assign_attributes(self, name, data): def assign_trial_params(self, name, data, dim='hru', create=True): """ - Assign new data to the ``parameter_trial`` dataset. + Assign new data to the ``spatial_params`` dataset. Parameters ---------- @@ -161,11 +161,11 @@ def assign_trial_params(self, name, data, dim='hru', create=True): must match the shape in the parameter trial file """ # Create the variable if we need - if create and name not in self.parameter_trial.variables: - self.parameter_trial[name] = self.parameter_trial[dim].astype(float).copy() - required_shape = self.parameter_trial[name].shape + if create and name not in self.spatial_params.variables: + self.spatial_params[name] = self.spatial_params[dim].astype(float).copy() + required_shape = self.spatial_params[name].shape try: - self.parameter_trial[name].values = np.array(data).reshape(required_shape) + self.spatial_params[name].values = np.array(data).reshape(required_shape) except ValueError as e: raise ValueError('The shape of the provided replacement data does', ' not match the shape of the original data.', e) @@ -189,10 +189,10 @@ def reset(self): self.config_path = self.manager_path.parent / '.pysumma' self.decisions = self.manager.decisions self.output_control = self.manager.output_control - self.parameter_trial = self.manager.parameter_trial + self.spatial_params = self.manager.spatial_params self.force_file_list = self.manager.force_file_list - self.local_param_info = self.manager.local_param_info - self.basin_param_info = self.manager.basin_param_info + self.global_hru_params = self.manager.global_hru_params + self.global_gru_params = self.manager.global_gru_params self.local_attributes = self.manager.local_attributes self.initial_conditions = self.manager.initial_conditions self.genparm = self.manager.genparm @@ -366,19 +366,19 @@ def _write_configuration(self, name=''): self.config_path = self.config_path / name self.config_path.mkdir(parents=True, exist_ok=True) manager_path = str(self.manager_path.parent) - settings_path = os.path.abspath(os.path.realpath(str(self.manager['settings_path'].value))) + settings_path = os.path.abspath(os.path.realpath(str(self.manager['settingsPath'].value))) settings_path = Path(settings_path.replace(manager_path, str(self.config_path))) self.manager_path = self.config_path / self.manager.file_name - self.manager['settings_path'] = str(settings_path) + os.sep + self.manager['settingsPath'] = str(settings_path) + os.sep self.manager.write(path=self.config_path) self.decisions.write(path=settings_path) self.force_file_list.write(path=settings_path) - self.local_param_info.write(path=settings_path) - self.basin_param_info.write(path=settings_path) + self.global_hru_params.write(path=settings_path) + self.global_gru_params.write(path=settings_path) self.output_control.write(path=settings_path) - self.local_attributes.to_netcdf(settings_path / self.manager['local_attributes'].value) - self.parameter_trial.to_netcdf(settings_path / self.manager['parameter_trial'].value) - self.initial_conditions.to_netcdf(settings_path / self.manager['model_init_cond'].value) + self.local_attributes.to_netcdf(settings_path / self.manager['attributeFile'].value) + self.spatial_params.to_netcdf(settings_path / self.manager['spatialParams'].value) + self.initial_conditions.to_netcdf(settings_path / self.manager['initCondFile'].value) with open(settings_path / 'GENPARM.TBL', 'w+') as f: f.writelines(self.genparm) with open(settings_path / 'MPTABLE.TBL', 'w+') as f: diff --git a/pysumma/param_trial.py b/pysumma/spatial_params.py similarity index 100% rename from pysumma/param_trial.py rename to pysumma/spatial_params.py From 0061a084ac3dba931cb15b70d01508a4ed466f32 Mon Sep 17 00:00:00 2001 From: arbennett Date: Mon, 20 Jul 2020 08:22:50 -0700 Subject: [PATCH 35/36] Remove print statement --- pysumma/file_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysumma/file_manager.py b/pysumma/file_manager.py index f50fc77f..83715335 100644 --- a/pysumma/file_manager.py +++ b/pysumma/file_manager.py @@ -52,7 +52,6 @@ def get_constructor_args(self, line): name, *value = line.split('!')[0].strip().split() if isinstance(value, list): value = " ".join(value).replace("'", "") - print(name, value) return (name.strip(), value.strip().replace("'", "").strip()) @property From 105869a9ebb42a944204acc638e084429708259a Mon Sep 17 00:00:00 2001 From: arbennett Date: Tue, 21 Jul 2020 16:00:34 -0700 Subject: [PATCH 36/36] Update for release --- README.md | 3 ++ docs/configuration.rst | 77 ++++++++++++++-------------------- environment.yml | 1 + pysumma/file_manager.py | 31 ++++++++------ pysumma/global_params.py | 5 ++- pysumma/meta/file_manager.json | 16 ++++--- pysumma/simulation.py | 26 ++++++------ setup.py | 2 +- 8 files changed, 80 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index eff185e5..00a55933 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# pysumma + pysumma is a Python wrapper for manipulating, running, managing, and analyzing of SUMMA (Structure for Unifying Multiple Modeling Alternatives) * [SUMMA web site at UCAR ](https://www.rap.ucar.edu/projects/summa) @@ -6,6 +8,7 @@ pysumma provides methods for: - Running SUMMA - Modifying SUMMA input files - Automatically parallelizing distributed and sensitivity analysis type experiments + - Calibration via OSTRICH - Visualizing output # Installation diff --git a/docs/configuration.rst b/docs/configuration.rst index 8ebf30ad..c11486e3 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -44,52 +44,38 @@ Then, you can see what is in it simply by printing it out: print(fm) - > 'SUMMA_FILE_MANAGER_V1.0' ! filemanager_version - > '/home/user/summa_setup_template/settings/' ! settings_path - > '/home/user/summa_setup_template/forcing/' ! input_path - > '/home/user/summa_setup_template/output/' ! output_path - > 'decisions.txt' ! decisions_path - > '[notUsed]' ! meta_time - > '[notUsed]' ! meta_attr - > '[notUsed]' ! meta_type - > '[notUsed]' ! meta_force - > '[notUsed]' ! meta_localparam - > 'output_control.txt' ! output_control - > '[notUsed]' ! meta_localindex - > '[notUsed]' ! meta_basinparam - > '[notUsed]' ! meta_basinmvar - > 'local_attributes.nc' ! local_attributes - > 'local_param_info.txt' ! local_param_info - > 'basin_param_info.txt' ! basin_param_info - > 'forcing_file_list.txt' ! forcing_file_list - > 'initial_conditions.nc' ! model_init_cond - > 'parameter_trial.nc' ! parameter_trial - > 'test' ! output_prefix + > controlVersion 'SUMMA_FILE_MANAGER_V3.0.0' + > simStartTime '2002-10-01 00:00' + > simEndTime '2003-05-31 00:00' + > tmZoneInfo 'localTime' + > settingsPath '/pool0/data/andrbenn/dana_3_test/.pysumma/_test/settings/' + > forcingPath './forcings/' + > outputPath './output/' + > decisionsFile 'decisions.txt' + > outputControlFile 'output_control.txt' + > globalHruParamFile '../params/local_param_info.txt' + > globalGruParamFile '../params/basin_param_info.txt' + > attributeFile '../params/local_attributes.nc' + > trialParamFile '../params/parameter_trial.nc' + > forcingListFile '../forcings/forcing_file_list.txt' + > initConditionFile '../params/initial_conditions.nc' + > outFilePrefix 'template_output' + > vegTableFile 'VEGPARM.TBL' + > soilTableFile 'SOILPARM.TBL' + > generalTableFile 'GENPARM.TBL' + > noahmpTableFile 'MPTABLE.TBL' To see how to access each of these specific options you can use the ``list_options`` method. - -:: - - print(fm.list_options) - - > ['filemanager_version', 'settings_path', 'input_path', - > 'output_path', 'decisions_path', 'meta_time', - > 'meta_attr', 'meta_type', 'meta_force', 'meta_localparam', - > 'output_control', 'meta_localindex', 'meta_basinparam', - > 'meta_basinmvar', 'local_attributes', 'local_param_info', - > 'basin_param_info', 'forcing_file_list', 'model_init_cond', - > 'parameter_trial', 'output_prefix'] - Then, each of these keys can be accessed directly similarly to how is done with python dictionaries. This can be used to inspect the values of each option as well as modify their values. :: - print(fm['output_prefix']) + print(fm['outputPrefix']) > 'test' ! output_prefix - fm['output_prefix'] = 'tutorial' + fm['outputPrefix'] = 'tutorial' print(fm['output_prefix']) @@ -114,14 +100,13 @@ Once instantiated you can inspect the available decisions and the options availa print(dec.list_options()) - > ['simulStart', 'simulFinsh', 'tmZoneInfo', 'soilCatTbl', - > 'vegeParTbl', 'soilStress', 'stomResist', 'num_method', + > ['soilCatTbl', 'vegeParTbl', 'soilStress', 'stomResist', > 'fDerivMeth', 'LAI_method', 'f_Richards', 'groundwatr', > 'hc_profile', 'bcUpprTdyn', 'bcLowrTdyn', 'bcUpprSoiH', > 'bcLowrSoiH', 'veg_traits', 'canopyEmis', 'snowIncept', > 'windPrfile', 'astability', 'canopySrad', 'alb_method', > 'compaction', 'snowLayers', 'thCondSnow', 'thCondSoil', - > 'spatial_gw', 'subRouting'] + > 'spatial_gw', 'subRouting', 'num_method'] print(dec['snowLayers']) @@ -136,12 +121,12 @@ Once instantiated you can inspect the available decisions and the options availa Forcing file list ----------------- The forcing file list contains a listing of each of the forcing files available for use as SUMMA input. -To instantiate the `ForceFileList` you will have to specify the path that is set as the ``input_path`` in your ``FileManager``. Below we show using the ``FileManager`` (``fm``) to do so. -Once instantiated you can also use the `ForceFileList` object to inspect the forcing files themselves. +To instantiate the `ForcingList` you will have to specify the path that is set as the ``input_path`` in your ``FileManager``. Below we show using the ``FileManager`` (``fm``) to do so. +Once instantiated you can also use the `ForcingList` object to inspect the forcing files themselves. :: - ff = ps.ForceFileList('.', 'forcingFileList.1hr.txt', fm['input_path']) + ff = ps.ForcingList('.', 'forcingFileList.1hr.txt', fm['input_path']) print(ff) >> 'forcing_file.nc' @@ -204,9 +189,9 @@ The format of the output control file mirrors the way that it is described in th >> sum -Local parameter info +GlobalParams -------------------- -The local parameter info file contains a listing of global parameters. Spatially dependent parameters are specified +The GlobalParams object listing of global parameters. Spatially dependent parameters are specified in the parameter trial NetCDF file. Values which are specified in the local parameter info file will be overwritten by those specified in the parameter trial file. As with the output control file, there are many parameters which can be specified, so we omit them for brevity. @@ -215,7 +200,7 @@ this out currently is by looking at the SUMMA source code directly. :: - lpi = ps.LocalParamInfo('.', 'local_param_info.txt') + lpi = ps.GlobalParams('.', 'global_param_info.txt') print(lpi.list_options()) >> ['upperBoundHead', 'lowerBoundHead', 'upperBoundTheta', 'lowerBoundTheta', @@ -230,7 +215,7 @@ NetCDF based files ================== The following input files are NetCDF-based and therefore, should be interacted with via ``xarray`` when using pysumma: - - Parameter trial + - Parameter trial (Spatially distributed parameters) - Basin parameters - Local attributes - Initial conditions diff --git a/environment.yml b/environment.yml index 820a01ee..0ed3b183 100644 --- a/environment.yml +++ b/environment.yml @@ -19,5 +19,6 @@ dependencies: - geopandas - jupyter - pip + - summa=3.0.0 - pip: - hs_restclient diff --git a/pysumma/file_manager.py b/pysumma/file_manager.py index 83715335..c38bee22 100644 --- a/pysumma/file_manager.py +++ b/pysumma/file_manager.py @@ -64,28 +64,28 @@ def decisions(self): @property def output_control(self): p1 = self.get_value('settingsPath') - p2 = self.get_value('outputControl') + p2 = self.get_value('outputControlFile') self._output_control = OutputControl(p1, p2) return self._output_control @property def global_hru_params(self): p1 = self.get_value('settingsPath') - p2 = self.get_value('globalHruParams') + p2 = self.get_value('globalHruParamFile') self._hru_params = GlobalParams(p1, p2) return self._hru_params @property def global_gru_params(self): p1 = self.get_value('settingsPath') - p2 = self.get_value('globalGruParams') + p2 = self.get_value('globalGruParamFile') self._gru_params = GlobalParams(p1, p2) return self._gru_params @property def force_file_list(self): p1 = self.get_value('settingsPath') - p2 = self.get_value('forcingList') + p2 = self.get_value('forcingListFile') p3 = self.get_value('forcingPath') self._force_file_list = ForcingList(p1, p2, p3) return self._force_file_list @@ -98,43 +98,48 @@ def local_attributes(self): return self._local_attrs @property - def spatial_params(self): + def trial_params(self): p1 = self.get_value('settingsPath') - p2 = self.get_value('spatialParams') - self._spatial_params = xr.open_dataset(p1 + p2) - return self._spatial_params + p2 = self.get_value('trialParamFile') + self._trial_params = xr.open_dataset(p1 + p2) + return self._trial_params @property def initial_conditions(self): p1 = self.get_value('settingsPath') - p2 = self.get_value('initCondFile') + p2 = self.get_value('initConditionFile') self._init_cond = xr.open_dataset(p1 + p2) return self._init_cond @property def genparm(self): - p1, p2 = self.get_value('settingsPath'), 'GENPARM.TBL' + p1 = self.get_value('settingsPath') + p2 = self.get_value('generalTableFile') + print(p1, p2) with open(p1 + p2, 'r') as f: self._genparm = f.readlines() return self._genparm @property def mptable(self): - p1, p2 = self.get_value('settingsPath'), 'MPTABLE.TBL' + p1 = self.get_value('settingsPath') + p2 = self.get_value('noahmpTableFile') with open(p1 + p2, 'r') as f: self._mptable = f.readlines() return self._mptable @property def soilparm(self): - p1, p2 = self.get_value('settingsPath'), 'SOILPARM.TBL' + p1 = self.get_value('settingsPath') + p2 = self.get_value('soilTableFile') with open(p1 + p2, 'r') as f: self._soilparm = f.readlines() return self._soilparm @property def vegparm(self): - p1, p2 = self.get_value('settingsPath'), 'VEGPARM.TBL' + p1 = self.get_value('settingsPath') + p2 = self.get_value('vegTableFile') with open(p1 + p2, 'r') as f: self._vegparm = f.readlines() return self._vegparm diff --git a/pysumma/global_params.py b/pysumma/global_params.py index 4321860b..7b62e861 100644 --- a/pysumma/global_params.py +++ b/pysumma/global_params.py @@ -50,10 +50,11 @@ def read(self, path): with open(path, 'r') as f: self.original_contents = f.readlines() for line in self.original_contents: - if ((line.startswith('!') and not self.opt_count) + isnt_empty = len(''.join(map(lambda x: x.strip(), line.split() ))) + if ((line.startswith('!') and not self.opt_count and not isnt_empty) or line.split('!')[0].strip() in self.fmt_strings): self.header.append(line) - elif not line.startswith('!'): + elif not line.startswith('!') and isnt_empty: self.options.append(self.OptionType( *self.get_constructor_args(line))) self.opt_count += 1 diff --git a/pysumma/meta/file_manager.json b/pysumma/meta/file_manager.json index 4342ee15..4a0f9bd6 100644 --- a/pysumma/meta/file_manager.json +++ b/pysumma/meta/file_manager.json @@ -8,12 +8,16 @@ "outputPath", "attributeFile", "decisionsFile", - "globalHruParams", - "globalGruParams", - "forcingList", - "outputControl", - "initCondFile", - "spatialParams", + "globalHruParamFile", + "globalGruParamFile", + "forcingListFile", + "outputControlFile", + "initConditionFile", + "trialParamFile", + "vegTableFile", + "soilTableFile", + "generalTableFile", + "noahmpTableFile", "outFilePrefix" ] } diff --git a/pysumma/simulation.py b/pysumma/simulation.py index 39c5c66f..08744a33 100644 --- a/pysumma/simulation.py +++ b/pysumma/simulation.py @@ -36,7 +36,7 @@ class Simulation(): Decisions object (populated after calling ``initialize``) output_control: OutputControl object (populated after calling ``initialize``) - spatial_params: + trial_params: Spatially distributed parameters (populated after calling ``initialize``) force_file_list: ForcingList object (populated after calling ``initialize``) @@ -75,7 +75,7 @@ def initialize(self): self.status = 'Initialized' self.decisions = self.manager.decisions self.output_control = self.manager.output_control - self.spatial_params = self.manager.spatial_params + self.trial_params = self.manager.trial_params self.force_file_list = self.manager.force_file_list self.global_hru_params = self.manager.global_hru_params self.global_gru_params = self.manager.global_gru_params @@ -161,11 +161,11 @@ def assign_trial_params(self, name, data, dim='hru', create=True): must match the shape in the parameter trial file """ # Create the variable if we need - if create and name not in self.spatial_params.variables: - self.spatial_params[name] = self.spatial_params[dim].astype(float).copy() - required_shape = self.spatial_params[name].shape + if create and name not in self.trial_params.variables: + self.trial_params[name] = self.trial_params[dim].astype(float).copy() + required_shape = self.trial_params[name].shape try: - self.spatial_params[name].values = np.array(data).reshape(required_shape) + self.trial_params[name].values = np.array(data).reshape(required_shape) except ValueError as e: raise ValueError('The shape of the provided replacement data does', ' not match the shape of the original data.', e) @@ -189,7 +189,7 @@ def reset(self): self.config_path = self.manager_path.parent / '.pysumma' self.decisions = self.manager.decisions self.output_control = self.manager.output_control - self.spatial_params = self.manager.spatial_params + self.trial_params = self.manager.trial_params self.force_file_list = self.manager.force_file_list self.global_hru_params = self.manager.global_hru_params self.global_gru_params = self.manager.global_gru_params @@ -377,15 +377,15 @@ def _write_configuration(self, name=''): self.global_gru_params.write(path=settings_path) self.output_control.write(path=settings_path) self.local_attributes.to_netcdf(settings_path / self.manager['attributeFile'].value) - self.spatial_params.to_netcdf(settings_path / self.manager['spatialParams'].value) - self.initial_conditions.to_netcdf(settings_path / self.manager['initCondFile'].value) - with open(settings_path / 'GENPARM.TBL', 'w+') as f: + self.trial_params.to_netcdf(settings_path / self.manager['trialParamFile'].value) + self.initial_conditions.to_netcdf(settings_path / self.manager['initConditionFile'].value) + with open(settings_path / self.manager['generalTableFile'].value, 'w+') as f: f.writelines(self.genparm) - with open(settings_path / 'MPTABLE.TBL', 'w+') as f: + with open(settings_path / self.manager['noahmpTableFile'].value, 'w+') as f: f.writelines(self.mptable) - with open(settings_path / 'SOILPARM.TBL', 'w+') as f: + with open(settings_path / self.manager['soilTableFile'].value, 'w+') as f: f.writelines(self.soilparm) - with open(settings_path / 'VEGPARM.TBL', 'w+') as f: + with open(settings_path / self.manager['vegTableFile'].value, 'w+') as f: f.writelines(self.vegparm) def get_output_files(self) -> List[str]: diff --git a/setup.py b/setup.py index 6ced74b6..964e5c4d 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='pysumma', - version='2.0.0', + version='3.0.0', description='A python wrapper for SUMMA', url='https://github.com/UW-Hydro/pysumma.git', author='YoungDon Choi, Andrew Bennett',