diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json new file mode 100644 index 000000000..73c9de1f5 --- /dev/null +++ b/.github/problem-matchers/sphinx.json @@ -0,0 +1,40 @@ +{ + "problemMatcher": [ + { + "owner": "sphinx-problem-matcher", + "pattern": [ + { + "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$", + "file": 1, + "line": 2, + "severity": 3, + "message": 4 + } + ] + }, + { + "owner": "sphinx-problem-matcher-loose", + "pattern": [ + { + "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst", + "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$", + "file": 1, + "severity": 2, + "message": 3 + } + ] + }, + { + "owner": "sphinx-problem-matcher-loose-no-severity", + "pattern": [ + { + "_comment": "Looks for file names ending with .rst and line numbers but without severity", + "regexp": "^(.*\\.rst):(\\d+):(.*)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + } + ] +} diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml new file mode 100644 index 000000000..b1b0ed5fe --- /dev/null +++ b/.github/workflows/doc-build.yml @@ -0,0 +1,44 @@ +name: Documentation +env: + PYTHON_VERSION: "3.12" + POETRY_VERSION: "1.4" +on: push +jobs: + sphinx: + # Note that we only do this on one platform and with the earliest reasonable Python + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Init Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Init Poetry ${{ env.POETRY_VERSION }} + run: | + python -m pip install poetry==${{ env.POETRY_VERSION }} + poetry config virtualenvs.in-project true + poetry config installer.modern-installation false + - name: Cache Virtual Environment + uses: actions/cache@v4 + with: + path: ./.venv + key: ${{ runner.os }}-sphinx-${{ env.PYTHON_VERSION }}-venv-${{ hashFiles('**/poetry.lock') }} + - name: Install Dependencies + run: | + poetry install + - name: Configure Problem Matcher + run: echo "::add-matcher::.github/problem-matchers/sphinx.json" + # See: https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md + # See: https://github.com/python/cpython/pull/20325 + - name: Run Sphinx + run: | + poetry run make clean html + working-directory: docs + - name: Upload documentation artifact + uses: actions/upload-artifact@v4 + with: + name: hpcflow-documentation (${{ github.sha }}) + path: docs/build/html + if-no-files-found: error + # TODO: Publish the docs to an internal site diff --git a/docs/source/conf.py b/docs/source/conf.py index 8f50f5610..70c39a4e8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -234,6 +234,7 @@ def prepare_task_schema_action_info(app): "sphinx.ext.autosummary", "sphinx.ext.intersphinx", "sphinx.ext.autosectionlabel", + "sphinx.ext.todo", "sphinx_jinja", "sphinx_copybutton", "sphinx_click", diff --git a/docs/source/contribute/index.rst b/docs/source/contribute/index.rst index 66ffff915..e222df24b 100644 --- a/docs/source/contribute/index.rst +++ b/docs/source/contribute/index.rst @@ -15,8 +15,7 @@ Recommended reads Installation for development ============================ -`Install poetry `_ ----------------------------------------------------------------- +`Install poetry `_ first. Clone repo @@ -27,14 +26,14 @@ Clone the git repo (see ssh links below), and then make sure that you switch to This branch is protected, so create a feature branch before pushing to the repo. -hpcflow -......... +Checking out hpcflow +.................... :: git clone git@github.com:hpcflow/hpcflow-new.git -matflow -........ +Checking out matflow +.................... :: git clone git@github.com:hpcflow/matflow-new.git @@ -78,17 +77,17 @@ Open the virtual enviroment with:: poetry shell -hpcflow --------- -CLI -.... +Working with hpcflow +-------------------- + You can interact with the CLI by calling:: python3 hpcflow/cli/cli.py --help -matflow --------- +Working with matflow +-------------------- + link to local hpcflow ...................... To be able to work with hpcflow and immediately see the changes reflected in matflow you need to reconfigure the hpcflow dependency to point to your local copy of hpcflow. diff --git a/docs/source/user/how_to/index.rst b/docs/source/user/how_to/index.rst index 045bbc291..a29a3f5b7 100644 --- a/docs/source/user/how_to/index.rst +++ b/docs/source/user/how_to/index.rst @@ -8,6 +8,7 @@ This help snippets guide you through common quick tasks in |app_name|. Configuration Task Schemas + Template Components Workflow Templates Workflows Environments diff --git a/docs/source/user/how_to/workflows.rst b/docs/source/user/how_to/workflows.rst index 0edc5df15..beeafcee0 100644 --- a/docs/source/user/how_to/workflows.rst +++ b/docs/source/user/how_to/workflows.rst @@ -37,4 +37,4 @@ We support paths like: * `ssh://user@host/path/to/workflow.json` for remote json * `https://sandbox.zenodo.org/record/1210144/files/workflow.zip` for zenodo zarr-zip -You can convert a zarr store to a zarr-zip store using `Workflow.to_zip()`. +You can convert a zarr store to an archived zarr-zip store using `Workflow.zip()`. diff --git a/hpcflow/data/demo_data/__init__.py b/hpcflow/data/demo_data/__init__.py index c3330c6d4..31b8fa174 100644 --- a/hpcflow/data/demo_data/__init__.py +++ b/hpcflow/data/demo_data/__init__.py @@ -1 +1,8 @@ -# required for `demo_data` to be accessible as a package resource in Python 3.8/3.9 +""" +Data for the demonstration workflows. + +Note +---- +This is required for `demo_data` to be accessible as a package resource in +Python 3.8/3.9. +""" diff --git a/hpcflow/data/demo_data_manifest/__init__.py b/hpcflow/data/demo_data_manifest/__init__.py index e69de29bb..135af6c91 100644 --- a/hpcflow/data/demo_data_manifest/__init__.py +++ b/hpcflow/data/demo_data_manifest/__init__.py @@ -0,0 +1,3 @@ +""" +Manifest for demonstration data. +""" diff --git a/hpcflow/sdk/__init__.py b/hpcflow/sdk/__init__.py index 9d7f70a18..ee7239233 100644 --- a/hpcflow/sdk/__init__.py +++ b/hpcflow/sdk/__init__.py @@ -4,7 +4,8 @@ import os import sys -# classes used in the construction of a workflow: +#: Classes used in the construction of a workflow. +#: :meta hide-value: sdk_classes = { "Workflow": "hpcflow.sdk.core.workflow", "Task": "hpcflow.sdk.core.task", @@ -95,6 +96,8 @@ } # these are defined as `BaseApp` methods with an underscore prefix: +#: Functions exported by the application. +#: :meta hide-value: sdk_funcs = ( "make_workflow", "make_demo_workflow", diff --git a/hpcflow/sdk/app.py b/hpcflow/sdk/app.py index dc3a2f1eb..ff5ecdeb5 100644 --- a/hpcflow/sdk/app.py +++ b/hpcflow/sdk/app.py @@ -65,7 +65,7 @@ def rate_limit_safe_url_to_fs(app, *args, logger=None, **kwargs): - """Call fsspec's `url_to_fs` but retry on `requests.exceptions.HTTPError`s + R"""Call fsspec's ``url_to_fs`` but retry on ``requests.exceptions.HTTPError``\ s. References ---------- @@ -125,14 +125,24 @@ def get_app_attribute(name): def get_app_module_all(): + """ + The list of all symbols exported by this module. + """ return ["app"] + list(sdk_classes.keys()) + list(sdk_funcs) def get_app_module_dir(): + """ + The sorted list of all symbols exported by this module. + """ return lambda: sorted(get_app_module_all()) class Singleton(type): + """ + Metaclass that enforces that only one instance of a class can be made. + """ + _instances = {} def __call__(cls, *args, **kwargs): @@ -158,15 +168,42 @@ class BaseApp(metaclass=Singleton): Parameters ---------- + name: + The name of the application. + version: + The version of the application. module: The module name in which the app object is defined. + description: + Description of the application. + gh_org: + Name of Github organisation responsible for the application. + gh_repo: + Github repository containing the application source. + config_options: + Configuration options. + scripts_dir: + Directory for scripts. + workflows_dir: + Directory for workflows. + demo_data_dir: + Directory for demonstration data. + data_data_manifest_dir: + Directory for demonstration data manifests. + template_components: + Template components. + pytest_args: + Arguments for pytest. + package_name: + Name of package if not the application name. docs_import_conv: The convention for the app alias used in import statements in the documentation. E.g. for the `hpcflow` base app, this is `hf`. This is combined with `module` to form the complete import statement. E.g. for the `hpcflow` base app, the complete import statement is: `import hpcflow.app as hf`, where `hpcflow.app` is the `module` argument and `hf` is the `docs_import_conv` argument. - + docs_url: + URL to documentation. """ _known_subs_file_name = "known_submissions.txt" @@ -194,22 +231,38 @@ def __init__( ): SDK_logger.info(f"Generating {self.__class__.__name__} {name!r}.") + #: The name of the application. self.name = name + #: Name of package. self.package_name = package_name or name.lower() + #: The version of the application. self.version = version + #: The module name in which the app object is defined. self.module = module + #: Description of the application. self.description = description + #: Name of Github organisation responsible for the application. self.gh_org = gh_org + #: Github repository containing the application source. self.gh_repo = gh_repo + #: Configuration options. self.config_options = config_options + #: Arguments for pytest. self.pytest_args = pytest_args + #: Directory for scripts. self.scripts_dir = scripts_dir + #: Directory for workflows. self.workflows_dir = workflows_dir + #: Directory for demonstration data. self.demo_data_dir = demo_data_dir + #: Directory for demonstration data manifests. self.demo_data_manifest_dir = demo_data_manifest_dir + #: The convention for the app alias used in import statements in the documentation. self.docs_import_conv = docs_import_conv + #: URL to documentation. self.docs_url = docs_url + #: Command line interface subsystem. self.cli = make_cli(self) self._log = AppLog(self) @@ -293,14 +346,23 @@ def wrap_func(func): @property def run_time_info(self) -> RunTimeInfo: + """ + Information about the runtime. + """ return self._run_time_info @property def log(self) -> AppLog: + """ + The application log. + """ return self._log @property def timeit(self) -> bool: + """ + Whether the timing analysis system is active. + """ return TimeIt.active @timeit.setter @@ -309,6 +371,9 @@ def timeit(self, value: bool): @property def template_components(self) -> Dict[str, ObjectList]: + """ + The template component data. + """ if not self.is_template_components_loaded: self._load_template_components() return self._template_components @@ -406,6 +471,10 @@ def _load_template_components(self, *include) -> None: def load_builtin_template_component_data( cls, package ) -> Dict[str, Union[List, Dict]]: + """ + Load the template component data built into the package. + This is as opposed to the template components defined by users. + """ SDK_logger.info( f"Loading built-in template component data for package: {package!r}." ) @@ -426,74 +495,119 @@ def load_builtin_template_component_data( @property def parameters(self) -> get_app_attribute("ParametersList"): + """ + The known template parameters. + """ self._ensure_template_component("parameters") return self._parameters @property def command_files(self) -> get_app_attribute("CommandFilesList"): + """ + The known template command files. + """ self._ensure_template_component("command_files") return self._command_files @property def envs(self) -> get_app_attribute("EnvironmentsList"): + """ + The known template execution environments. + """ self._ensure_template_component("environments") return self._environments @property def scripts(self): + """ + The known template scripts. + """ self._ensure_template_component("scripts") return self._scripts @property def task_schemas(self) -> get_app_attribute("TaskSchemasList"): + """ + The known template task schemas. + """ self._ensure_template_component("task_schemas") return self._task_schemas @property def logger(self) -> Logger: + """ + The main underlying logger. + """ return self.log.logger @property def API_logger(self) -> Logger: + """ + The logger for API messages. + """ return self.logger.getChild("api") @property def CLI_logger(self) -> Logger: + """ + The logger for CLI messages. + """ return self.logger.getChild("cli") @property def config_logger(self) -> Logger: + """ + The logger for configuration messages. + """ return self.logger.getChild("config") @property def persistence_logger(self) -> Logger: + """ + The logger for persistence engine messages. + """ return self.logger.getChild("persistence") @property def submission_logger(self) -> Logger: + """ + The logger for job submission messages. + """ return self.logger.getChild("submission") @property def runtime_info_logger(self) -> Logger: + """ + The logger for runtime messages. + """ return self.logger.getChild("runtime") @property def is_config_loaded(self) -> bool: + """ + Whether the configuration is loaded. + """ return bool(self._config) @property def is_template_components_loaded(self) -> bool: - """Return True if any template component (e.g. parameters) has been loaded.""" + """Whether any template component (e.g. parameters) has been loaded.""" return bool(self._template_components) @property def config(self) -> Config: + """ + The configuration. + """ if not self.is_config_loaded: self.load_config() return self._config @property def scheduler_lookup(self): + """ + The scheduler mapping. + """ return { ("direct", "posix"): self.DirectPosix, ("direct", "nt"): self.DirectWindows, @@ -550,35 +664,42 @@ def perm_error_retry(self): @property def user_data_dir(self) -> Path: + """ + The user's data directory. + """ if self._user_data_dir is None: self._user_data_dir = Path(user_data_dir(appname=self.package_name)) return self._user_data_dir @property def user_cache_dir(self) -> Path: - """Retrieve the app cache directory.""" + """The user's cache directory.""" if self._user_cache_dir is None: self._user_cache_dir = Path(user_cache_path(appname=self.package_name)) return self._user_cache_dir @property def user_runtime_dir(self) -> Path: - """Retrieve a temporary directory.""" + """The user's temporary runtime directory.""" if self._user_runtime_dir is None: self._user_runtime_dir = self.user_data_dir.joinpath("temp") return self._user_runtime_dir @property def demo_data_cache_dir(self) -> Path: - """Retrieve a directory for example data caching.""" + """A directory for example data caching.""" if self._demo_data_cache_dir is None: self._demo_data_cache_dir = self.user_cache_dir.joinpath("demo_data") return self._demo_data_cache_dir @property def user_data_hostname_dir(self) -> Path: - """We segregate by hostname to account for the case where multiple machines might - use the same shared file system""" + """ + The directory for holding user data. + + We segregate by hostname to account for the case where multiple machines might + use the same shared file system. + """ # This might need to cover e.g. multiple login nodes, as described in the # config file: @@ -589,7 +710,7 @@ def user_data_hostname_dir(self) -> Path: @property def user_cache_hostname_dir(self) -> Path: - """Retrieve the hostname-scoped app cache directory.""" + """The hostname-scoped app cache directory.""" if self._user_cache_hostname_dir is None: machine_name = self.config.get("machine") self._user_cache_hostname_dir = self.user_cache_dir.joinpath(machine_name) @@ -718,11 +839,17 @@ def load_config( warn=True, **overrides, ) -> None: + """ + Load the user's configuration. + """ if warn and self.is_config_loaded: warnings.warn("Configuration is already loaded; reloading.") self._load_config(config_dir, config_key, **overrides) def unload_config(self): + """ + Discard any loaded configuration. + """ self._config_files = {} self._config = None @@ -762,6 +889,10 @@ def reload_config( warn=True, **overrides, ) -> None: + """ + Reload the configuration. Use if a user has updated the configuration file + outside the scope of this application. + """ if warn and not self.is_config_loaded: warnings.warn("Configuration is not loaded; loading.") self.log.remove_file_handlers() @@ -909,7 +1040,10 @@ def load_demo_workflow(self, name: str) -> get_app_attribute("WorkflowTemplate") with self.get_demo_workflow_template_file(name) as path: return self.WorkflowTemplate.from_file(path) - def template_components_from_json_like(self, json_like) -> None: + def template_components_from_json_like(self, json_like): + """ + Get template components from a (simply parsed) JSOM document. + """ cls_lookup = { "parameters": self.ParametersList, "command_files": self.CommandFilesList, @@ -944,6 +1078,9 @@ def get_parameter_task_schema_map(self) -> Dict[str, List[List]]: return param_map def get_info(self) -> Dict[str, Any]: + """ + Get miscellaneous runtime system information. + """ return { "name": self.name, "version": self.version, @@ -951,13 +1088,12 @@ def get_info(self) -> Dict[str, Any]: "is_frozen": self.run_time_info.is_frozen, } - @property - def known_subs_file_name(self): - return self._known_subs_file_name - @property def known_subs_file_path(self): - return self.user_data_hostname_dir / self.known_subs_file_name + """ + The path to the file describing known submissions. + """ + return self.user_data_hostname_dir / self._known_subs_file_name def _format_known_submissions_line( self, @@ -2242,6 +2378,9 @@ def configure_env( use_current_env=False, env_source_file=None, ): + """ + Configure an execution environment. + """ if not setup: setup = [] if not executables: @@ -2505,9 +2644,15 @@ def get_demo_data_file_path(self, file_name) -> Path: return cache_file_path def cache_demo_data_file(self, file_name) -> Path: + """ + Get the name of a cached demo data file. + """ return self.get_demo_data_file_path(file_name) def cache_all_demo_data_files(self) -> List[Path]: + """ + Get the name of all cached demo data file. + """ return [self.get_demo_data_file_path(i) for i in self.list_demo_data_files()] def copy_demo_data( diff --git a/hpcflow/sdk/cli.py b/hpcflow/sdk/cli.py index 25ae83b2c..d1ef3e052 100644 --- a/hpcflow/sdk/cli.py +++ b/hpcflow/sdk/cli.py @@ -1,3 +1,7 @@ +""" +Command line interface implementation. +""" + import json import os from typing import Dict, List @@ -38,26 +42,36 @@ rechunk_backup_opt, rechunk_chunk_size_opt, rechunk_status_opt, + _add_doc_from_help, ) from hpcflow.sdk.helper.cli import get_helper_CLI from hpcflow.sdk.log import TimeIt from hpcflow.sdk.submission.shells import ALL_SHELLS +#: Standard option string_option = click.option( "--string", is_flag=True, default=False, help="Determines if passing a file path or a string.", ) +#: Standard option workflow_ref_type_opt = click.option( "--ref-type", "-r", type=click.Choice(["assume-id", "id", "path"]), default="assume-id", + help="How to interpret a reference, as an ID, a path, or to guess.", ) +_add_doc_from_help(string_option, workflow_ref_type_opt) + + def parse_jobscript_wait_spec(jobscripts: str) -> Dict[int, List[int]]: + """ + Parse a jobscript wait specification. + """ sub_js_idx_dct = {} for sub_i in jobscripts.split(";"): sub_idx_str, js_idx_lst_str = sub_i.split(":") diff --git a/hpcflow/sdk/cli_common.py b/hpcflow/sdk/cli_common.py index b500f79e2..f626fbb89 100644 --- a/hpcflow/sdk/cli_common.py +++ b/hpcflow/sdk/cli_common.py @@ -7,10 +7,14 @@ def sub_tasks_callback(ctx, param, value): + """ + Parse subtasks. + """ if value: return [int(i) for i in value.split(",")] +#: Standard option format_option = click.option( "--format", type=click.Choice(ALL_TEMPLATE_FORMATS), @@ -20,11 +24,13 @@ def sub_tasks_callback(ctx, param, value): "particular format." ), ) +#: Standard option path_option = click.option( "--path", type=click.Path(exists=True), help="The directory path into which the new workflow will be generated.", ) +#: Standard option name_option = click.option( "--name", help=( @@ -33,6 +39,7 @@ def sub_tasks_callback(ctx, param, value): "will be used, in combination with a date-timestamp." ), ) +#: Standard option overwrite_option = click.option( "--overwrite", is_flag=True, @@ -42,12 +49,15 @@ def sub_tasks_callback(ctx, param, value): "the existing directory will be overwritten." ), ) +#: Standard option store_option = click.option( "--store", type=click.Choice(ALL_STORE_FORMATS), help="The persistent store type to use.", default=DEFAULT_STORE_FORMAT, ) + +#: Standard option ts_fmt_option = click.option( "--ts-fmt", help=( @@ -56,6 +66,7 @@ def sub_tasks_callback(ctx, param, value): "should not include a time zone name." ), ) +#: Standard option ts_name_fmt_option = click.option( "--ts-name-fmt", help=( @@ -63,6 +74,8 @@ def sub_tasks_callback(ctx, param, value): "includes a timestamp." ), ) + +#: Standard option variables_option = click.option( "-v", "--var", @@ -74,6 +87,7 @@ def sub_tasks_callback(ctx, param, value): "string. Multiple variable values can be specified." ), ) +#: Standard option js_parallelism_option = click.option( "--js-parallelism", help=( @@ -84,23 +98,27 @@ def sub_tasks_callback(ctx, param, value): ), type=click.BOOL, ) +#: Standard option wait_option = click.option( "--wait", help=("If True, this command will block until the workflow execution is complete."), is_flag=True, default=False, ) +#: Standard option add_to_known_opt = click.option( "--add-to-known/--no-add-to-known", default=True, help="If True, add this submission to the known-submissions file.", ) +#: Standard option print_idx_opt = click.option( "--print-idx", help="If True, print the submitted jobscript indices for each submission index.", is_flag=True, default=False, ) +#: Standard option tasks_opt = click.option( "--tasks", help=( @@ -109,22 +127,27 @@ def sub_tasks_callback(ctx, param, value): ), callback=sub_tasks_callback, ) +#: Standard option cancel_opt = click.option( "--cancel", help="Immediately cancel the submission. Useful for testing and benchmarking.", is_flag=True, default=False, ) +#: Standard option submit_status_opt = click.option( "--status/--no-status", help="If True, display a live status to track submission progress.", default=True, ) +#: Standard option make_status_opt = click.option( "--status/--no-status", help="If True, display a live status to track workflow creation progress.", default=True, ) + +#: Standard option zip_path_opt = click.option( "--path", default=".", @@ -134,15 +157,21 @@ def sub_tasks_callback(ctx, param, value): "path is assumed to be the full file path to the new zip file." ), ) +#: Standard option zip_overwrite_opt = click.option( "--overwrite", is_flag=True, default=False, help="If set, any existing file will be overwritten.", ) +#: Standard option zip_log_opt = click.option("--log", help="Path to a log file to use during zipping.") +#: Standard option zip_include_execute_opt = click.option("--include-execute", is_flag=True) +#: Standard option zip_include_rechunk_backups_opt = click.option("--include-rechunk-backups", is_flag=True) + +#: Standard option unzip_path_opt = click.option( "--path", default=".", @@ -152,12 +181,16 @@ def sub_tasks_callback(ctx, param, value): "Otherwise, this path will represent the new workflow directory path." ), ) +#: Standard option unzip_log_opt = click.option("--log", help="Path to a log file to use during unzipping.") + +#: Standard option rechunk_backup_opt = click.option( "--backup/--no-backup", default=True, help=("First copy a backup of the array to a directory ending in `.bak`."), ) +#: Standard option rechunk_chunk_size_opt = click.option( "--chunk-size", type=click.INT, @@ -168,8 +201,58 @@ def sub_tasks_callback(ctx, param, value): "array's shape)." ), ) +#: Standard option rechunk_status_opt = click.option( "--status/--no-status", default=True, help="If True, display a live status to track rechunking progress.", ) + + +def _add_doc_from_help(*args): + """ + Attach the ``help`` field of each of its arguments as its ``__doc__``. + Only necessary because the wrappers in Click don't do this for us. + + :meta private: + """ + # Yes, this is ugly! + from types import SimpleNamespace + + for opt in args: + ns = SimpleNamespace() + params = getattr(opt(ns), "__click_params__", []) + if params: + help = getattr(params[0], "help", "") + if help: + opt.__doc__ = f"Click option decorator: {help}" + + +_add_doc_from_help( + format_option, + path_option, + name_option, + overwrite_option, + store_option, + ts_fmt_option, + ts_name_fmt_option, + variables_option, + js_parallelism_option, + wait_option, + add_to_known_opt, + print_idx_opt, + tasks_opt, + cancel_opt, + submit_status_opt, + make_status_opt, + zip_path_opt, + zip_overwrite_opt, + zip_log_opt, + zip_include_execute_opt, + zip_include_rechunk_backups_opt, + unzip_path_opt, + unzip_log_opt, + rechunk_backup_opt, + rechunk_chunk_size_opt, + rechunk_status_opt, +) diff --git a/hpcflow/sdk/config/__init__.py b/hpcflow/sdk/config/__init__.py index 8c215c86d..9c1d2a27f 100644 --- a/hpcflow/sdk/config/__init__.py +++ b/hpcflow/sdk/config/__init__.py @@ -1 +1,5 @@ +""" +Configuration loading and manipulation. +""" + from .config import Config, ConfigFile, ConfigOptions, DEFAULT_CONFIG diff --git a/hpcflow/sdk/config/callbacks.py b/hpcflow/sdk/config/callbacks.py index b8309d384..46a19b0cc 100644 --- a/hpcflow/sdk/config/callbacks.py +++ b/hpcflow/sdk/config/callbacks.py @@ -10,7 +10,9 @@ def callback_vars(config, value): - """Substitute configuration variables.""" + """ + Callback that substitutes configuration variables. + """ def vars_repl(match_obj): var_name = match_obj.groups()[0] @@ -27,6 +29,9 @@ def vars_repl(match_obj): def callback_file_paths(config, file_path): + """ + Callback that resolves file paths. + """ if isinstance(file_path, list): return [config._resolve_path(i) for i in file_path] else: @@ -34,6 +39,9 @@ def callback_file_paths(config, file_path): def callback_bool(config, value): + """ + Callback that coerces values to boolean. + """ if not isinstance(value, bool): if value.lower() == "true": return True @@ -45,6 +53,9 @@ def callback_bool(config, value): def callback_lowercase(config, value): + """ + Callback that forces a string to lower case. + """ if isinstance(value, list): return [i.lower() for i in value] elif isinstance(value, dict): @@ -54,6 +65,9 @@ def callback_lowercase(config, value): def exists_in_schedulers(config, value): + """ + Callback that tests that a value is a supported scheduler name. + """ if value not in config.schedulers: raise ValueError( f"Cannot set default scheduler; {value!r} is not a supported scheduler " @@ -64,6 +78,9 @@ def exists_in_schedulers(config, value): def callback_supported_schedulers(config, schedulers): + """ + Callback that tests that all values are names of supported schedulers. + """ # validate against supported schedulers according to the OS - this won't validate that # a particular scheduler actually exists on this system: available = config._app.get_OS_supported_schedulers() @@ -122,6 +139,9 @@ def callback_scheduler_set_up(config, schedulers): def callback_supported_shells(config, shell_name): + """ + Callback that tests if a shell names is supported on this OS. + """ supported = get_supported_shells(os.name) if shell_name not in supported: raise UnsupportedShellError(shell=shell_name, supported=supported) @@ -146,11 +166,14 @@ def set_callback_file_paths(config, value): def check_load_data_files(config, value): - """Check data files (e.g. task schema files) can be loaded successfully. This is only + """Check data files (e.g., task schema files) can be loaded successfully. This is only done on `config.set` (and not on `config.get` or `config._validate`) because it could be expensive in the case of remote files.""" config._app.reload_template_components(warn=False) def callback_update_log_console_level(config, value): + """ + Callback to set the logging level. + """ config._app.log.update_console_level(value) diff --git a/hpcflow/sdk/config/cli.py b/hpcflow/sdk/config/cli.py index d1b4d3568..c8c71ea98 100644 --- a/hpcflow/sdk/config/cli.py +++ b/hpcflow/sdk/config/cli.py @@ -44,7 +44,10 @@ def warning_formatter(func=custom_warning_formatter): def CLI_exception_wrapper_gen(*exception_cls): - """Decorator factory""" + """ + Decorator factory that enhances the wrapped function to display a nice message on + success or failure. + """ def CLI_exception_wrapper(func): """Decorator diff --git a/hpcflow/sdk/config/config.py b/hpcflow/sdk/config/config.py index 8d24345f3..5d781e5af 100644 --- a/hpcflow/sdk/config/config.py +++ b/hpcflow/sdk/config/config.py @@ -1,3 +1,7 @@ +""" +Configuration system class. +""" + from __future__ import annotations import contextlib @@ -61,6 +65,7 @@ logger = logging.getLogger(__name__) _DEFAULT_SHELL = DEFAULT_SHELL_NAMES[os.name] +#: The default configuration descriptor. DEFAULT_CONFIG = { "invocation": {"environment_setup": None, "match": {}}, "config": { @@ -82,12 +87,17 @@ class ConfigOptions: """Application-level options for configuration""" + #: The default directory. default_directory: Union[Path, str] + #: The environment variable containing the directory name. directory_env_var: str + #: The default configuration. default_config: Optional[Dict] = field( default_factory=lambda: deepcopy(DEFAULT_CONFIG) ) + #: Any extra schemas to apply. extra_schemas: Optional[List[Schema]] = field(default_factory=lambda: []) + #: Default directory of known configurations. default_known_configs_dir: Optional[str] = None def __post_init__(self): @@ -96,7 +106,9 @@ def __post_init__(self): self._configurable_keys = cfg_keys def init_schemas(self): - # Get allowed configurable keys from config schemas: + """ + Get allowed configurable keys from config schemas. + """ cfg_schemas = [get_schema("config_schema.yaml")] + self.extra_schemas cfg_keys = [] for cfg_schema in cfg_schemas: @@ -130,18 +142,114 @@ def validate(self, data, logger, metadata=None, raise_with_metadata=True): class Config: """Application configuration as defined in one or more config files. + This class supports indexing into the collection of properties via Python dot notation. + Notes ----- On modifying/setting existing values, modifications are not automatically copied - to the configuration file; use `save()` to save to the file. Items in `overrides` + to the configuration file; use :meth:`save()` to save to the file. Items in `overrides` are not saved into the file. `schedulers` is used for specifying the available schedulers on this machine, and the - default arguments that should be used when initialising the `Scheduler` object. + default arguments that should be used when initialising the + :py:class:`Scheduler` object. `shells` is used for specifying the default arguments that should be used when - initialising the `Shell` object. - + initialising the :py:class:`Shell` object. + + Parameters + ---------- + app: + The main hpcflow application instance. + config_file: + The configuration file that contains this config. + options: + Configuration options to be applied. + logger: + Where to log messages relating to configuration. + config_key: + The name of the configuration within the configuration file. + uid: int + User ID. + callbacks: dict + Overrides for the callback system. + variables: dict[str, str] + Variables to substitute when processing the configuration. + + Attributes + ---------- + config_directory: + The directory containing the configuration file. + config_file_name: + The name of the configuration file. + config_file_path: + The full path to the configuration file. + config_file_contents: + The cached contents of the configuration file. + config_key: + The primary key to select the configuration within the configuration file. + config_schemas: + The schemas that apply to the configuration file. + host_user_id: + User ID as understood by the script. + host_user_id_file_path: + Where user ID information is stored. + invoking_user_id: + User ID that created the workflow. + machine: + Machine to submit to. + Mapped to a field in the configuration file. + user_name: + User to submit as. + Mapped to a field in the configuration file. + user_orcid: + User's ORCID. + Mapped to a field in the configuration file. + user_affiliation: + User's institutional affiliation. + Mapped to a field in the configuration file. + linux_release_file: + Where to get the description of the Linux release version data. + Mapped to a field in the configuration file. + log_file_path: + Where to log to. + Mapped to a field in the configuration file. + log_file_level: + At what level to do logging to the file. + Mapped to a field in the configuration file. + log_console_level: + At what level to do logging to the console. Usually coarser than to a file. + Mapped to a field in the configuration file. + task_schema_sources: + Where to get task schemas. + Mapped to a field in the configuration file. + parameter_sources: + Where to get parameter descriptors. + Mapped to a field in the configuration file. + command_file_sources: + Where to get command files. + Mapped to a field in the configuration file. + environment_sources: + Where to get execution environment descriptors. + Mapped to a field in the configuration file. + default_scheduler: + The name of the default scheduler. + Mapped to a field in the configuration file. + default_shell: + The name of the default shell. + Mapped to a field in the configuration file. + schedulers: + Settings for supported scheduler(s). + Mapped to a field in the configuration file. + shells: + Settings for supported shell(s). + Mapped to a field in the configuration file. + demo_data_dir: + Location of demo data. + Mapped to a field in the configuration file. + demo_data_manifest_file: + Where the manifest describing the demo data is. + Mapped to a field in the configuration file. """ def __init__( @@ -517,7 +625,16 @@ def _set(self, name, value, is_json=False, callback=True, quiet=False): print(f"value is already: {callback_val!r}") def set(self, path: str, value, is_json=False, quiet=False): - """Set the value of a configuration item.""" + """ + Set the value of a configuration item. + + Parameters + ---------- + path: + Which configuration item to set. + value: + What to set it to. + """ self._logger.debug(f"Attempting to set config item {path!r} to {value!r}.") if is_json: @@ -542,7 +659,18 @@ def set(self, path: str, value, is_json=False, quiet=False): self._set(name, root, quiet=quiet) def unset(self, name): - """Unset the value of a configuration item.""" + """ + Unset the value of a configuration item. + + Parameters + ---------- + name: str + The name of the configuration item. + + Notes + ----- + Only top level configuration items may be unset. + """ if name not in self._configurable_keys: raise ConfigNonConfigurableError(name=name) if name in self._unset_keys or not self._file.is_item_set(self._config_key, name): @@ -564,6 +692,14 @@ def get( ret_parts=False, default=None, ): + """ + Get the value of a configuration item. + + Parameters + ---------- + path: str + The name of or path to the configuration item. + """ parts = path.split(".") root = deepcopy(self._get(parts[0], callback=callback)) try: @@ -582,7 +718,16 @@ def get( return tuple(ret) def append(self, path, value, is_json=False): - """Append a value to a list-like configuration item.""" + """ + Append a value to a list-like configuration item. + + Parameters + ---------- + path: str + The name of or path to the configuration item. + value: + The value to append. + """ if is_json: value = self._parse_JSON(path, value) @@ -612,7 +757,16 @@ def append(self, path, value, is_json=False): self._set(parts[0], root) def prepend(self, path, value, is_json=False): - """Prepend a value to a list-like configuration item.""" + """ + Prepend a value to a list-like configuration item. + + Parameters + ---------- + path: str + The name of or path to the configuration item. + value: + The value to prepend. + """ if is_json: value = self._parse_JSON(path, value) @@ -638,7 +792,16 @@ def prepend(self, path, value, is_json=False): self._set(parts[0], root) def pop(self, path, index): - """Remove a value from a specified index of a list-like configuration item.""" + """ + Remove a value from a specified index of a list-like configuration item. + + Parameters + ---------- + path: str + The name of or path to the configuration item. + index: int + Where to remove the value from. 0 for the first item, -1 for the last. + """ existing, root, parts = self.get( path, @@ -674,8 +837,10 @@ def update(self, path: str, value, is_json=False): Parameters ---------- - path + path: str A dot-delimited string of the nested path to update. + value: dict + A dictionary to merge in. """ if is_json: @@ -739,12 +904,18 @@ def reset(self): self._app.reset_config() def add_scheduler(self, scheduler, **defaults): + """ + Add a scheduler. + """ if scheduler in self.get("schedulers"): print(f"Scheduler {scheduler!r} already exists.") return self.update(f"schedulers.{scheduler}.defaults", defaults) def add_shell(self, shell, **defaults): + """ + Add a shell. + """ if shell in self.get("shells"): return if shell.lower() == "wsl": @@ -753,6 +924,9 @@ def add_shell(self, shell, **defaults): self.update(f"shells.{shell}.defaults", defaults) def add_shell_WSL(self, **defaults): + """ + Add shell with WSL prefix. + """ if "WSL_executable" not in defaults: defaults["WSL_executable"] = "wsl.exe" self.add_shell("wsl", **defaults) @@ -763,13 +937,13 @@ def import_from_file(self, file_path, rename=True, make_new=False): Parameters ---------- - file_path + file_path: Local or remote path to a config import YAML file which may have top-level keys "invocation" and "config". - rename + rename: If True, the current config will be renamed to the stem of the file specified in `file_path`. Ignored if `make_new` is True. - make_new + make_new: If True, add the config items as a new config, rather than modifying the current config. The name of the new config will be the stem of the file specified in `file_path`. diff --git a/hpcflow/sdk/config/config_file.py b/hpcflow/sdk/config/config_file.py index 895991d59..7c048b4a5 100644 --- a/hpcflow/sdk/config/config_file.py +++ b/hpcflow/sdk/config/config_file.py @@ -1,3 +1,7 @@ +""" +Configuration file adapter. +""" + from __future__ import annotations import copy @@ -26,10 +30,23 @@ class ConfigFile: - """Configuration file.""" + """ + Configuration file. + + Parameters + ---------- + directory: + The directory containing the configuration file. + logger: + Where to log messages. + config_options: + Configuration options. + """ def __init__(self, directory, logger, config_options): + #: Where to log messages. self.logger = logger + #: The directory containing the configuration file. self.directory = self._resolve_config_dir( config_opt=config_options, logger=self.logger, @@ -39,9 +56,13 @@ def __init__(self, directory, logger, config_options): self._configs = [] # set by _load_file_data: + #: The path to the config file. self.path = None + #: The cached contents of the config file. self.contents = None + #: The parsed contents of the config file. self.data = None + #: The parsed contents of the config file where the alternate parser was used. self.data_rt = None self._load_file_data(config_options) @@ -105,12 +126,31 @@ def _validate(self, data): return file_schema def get_invoc_data(self, config_key): + """ + Get the invocation data for the given configuration. + + Parameters + ---------- + config_key: str + The name of the configuration within the configuration file. + """ return self.data["configs"][config_key] def get_invocation(self, config_key): + """ + Get the invocation for the given configuration. + + Parameters + ---------- + config_key: str + The name of the configuration within the configuration file. + """ return self.get_invoc_data(config_key)["invocation"] def save(self): + """ + Write the (modified) configuration to the configuration file. + """ new_data = copy.deepcopy(self.data) new_data_rt = copy.deepcopy(self.data_rt) new_contents = "" @@ -271,6 +311,9 @@ def add_default_config(self, config_options, name=None) -> str: @staticmethod def get_config_file_path(directory): + """ + Get the path to the configuration file. + """ # Try both ".yml" and ".yaml" extensions: path_yaml = directory.joinpath("config.yaml") if path_yaml.is_file(): @@ -307,11 +350,36 @@ def _load_file_data(self, config_options): def get_config_item( self, config_key, name, raise_on_missing=False, default_value=None ): + """ + Get a configuration item. + + Parameters + ---------- + config_key: str + The name of the configuration within the configuration file. + name: str + The name of the configuration item. + raise_on_missing: bool + Whether to raise an error if the config item is absent. + default_value: + The default value to use when the config item is absent + (and ``raise_on_missing`` is not specified). + """ if raise_on_missing and name not in self.get_invoc_data(config_key)["config"]: raise ValueError(f"missing from file: {name!r}") return self.get_invoc_data(config_key)["config"].get(name, default_value) def is_item_set(self, config_key, name): + """ + Determine if a configuration item is set. + + Parameters + ---------- + config_key: str + The name of the configuration within the configuration file. + name: str + The name of the configuration item. + """ try: self.get_config_item(config_key, name, raise_on_missing=True) except ValueError: @@ -319,7 +387,16 @@ def is_item_set(self, config_key, name): return True def rename_config_key(self, config_key: str, new_config_key: str): - """Change the config key of the loaded config.""" + """ + Change the config key of the loaded config. + + Parameters + ---------- + config_key: str + The old name of the configuration within the configuration file. + new_config_key: str + The new name of the configuration. + """ new_data = copy.deepcopy(self.data) new_data_rt = copy.deepcopy(self.data_rt) @@ -342,7 +419,18 @@ def update_invocation( environment_setup: Optional[str] = None, match: Optional[Dict] = None, ): - """Modify the invocation parameters of the loaded config.""" + """ + Modify the invocation parameters of the loaded config. + + Parameters + ---------- + config_key: str + The name of the configuration within the configuration file. + environment_setup: + The new value of the ``environment_setup`` key. + match: + The new values to merge into the ``match`` key. + """ new_data = copy.deepcopy(self.data) new_data_rt = copy.deepcopy(self.data_rt) diff --git a/hpcflow/sdk/config/errors.py b/hpcflow/sdk/config/errors.py index 2da28f380..e6c1bcfd9 100644 --- a/hpcflow/sdk/config/errors.py +++ b/hpcflow/sdk/config/errors.py @@ -1,3 +1,8 @@ +""" +Miscellaneous configuration-related errors. +""" + + class ConfigError(Exception): """Raised when a valid configuration can not be associated with the current invocation.""" @@ -6,6 +11,10 @@ class ConfigError(Exception): class ConfigUnknownItemError(ConfigError): + """ + Raised when the configuration contains an unknown item. + """ + def __init__(self, name, message=None): self.message = message or ( f"Specified name {name!r} is not a valid meta-data or configurable " @@ -15,6 +24,10 @@ def __init__(self, name, message=None): class ConfigUnknownOverrideError(ConfigError): + """ + Raised when the configuration override contains an unknown item. + """ + def __init__(self, name, message=None): self.message = message or ( f"Specified configuration override {name!r} is not a valid configurable item." @@ -23,22 +36,38 @@ def __init__(self, name, message=None): class ConfigNonConfigurableError(ConfigError): + """ + Raised when the configuration contains an item that can't be configured. + """ + def __init__(self, name, message=None): self.message = message or (f"Specified name {name!r} is not a configurable item.") super().__init__(self.message) class ConfigItemAlreadyUnsetError(ConfigError): + """ + Raised when the configuration tries to unset an unset item. + """ + def __init__(self, name, message=None): self.message = message or f"Configuration item {name!r} is already not set." super().__init__(self.message) class ConfigFileValidationError(ConfigError): + """ + Raised when the configuration file fails validation. + """ + pass class ConfigItemCallbackError(ConfigError): + """ + Raised when a configuration callback errors. + """ + def __init__(self, name, callback, err, message=None): self.message = message or ( f"Callback function {callback.__name__!r} for configuration item {name!r} " @@ -59,6 +88,10 @@ def __init__(self, message=None): class ConfigFileInvocationUnknownMatchKey(ConfigError): + """ + Raised when the configuration contains an invalid match key. + """ + def __init__(self, match_key, message=None): self.message = message or ( f"Specified match key ({match_key!r}) is not a valid run time info " diff --git a/hpcflow/sdk/core/__init__.py b/hpcflow/sdk/core/__init__.py index c7389448c..74cfa0abc 100644 --- a/hpcflow/sdk/core/__init__.py +++ b/hpcflow/sdk/core/__init__.py @@ -8,5 +8,7 @@ """ +#: Formats supported for templates. ALL_TEMPLATE_FORMATS = ("yaml", "json") +#: The exit code used by an EAR when it aborts. ABORT_EXIT_CODE = 64 diff --git a/hpcflow/sdk/core/actions.py b/hpcflow/sdk/core/actions.py index f61393ac6..40b81fdc1 100644 --- a/hpcflow/sdk/core/actions.py +++ b/hpcflow/sdk/core/actions.py @@ -1,3 +1,9 @@ +""" +Actions are base components of elements. +Element action runs (EARs) are the basic components of any enactment; +they may be grouped together within a jobscript for efficiency. +""" + from __future__ import annotations import copy from dataclasses import dataclass @@ -37,13 +43,23 @@ class ActionScopeType(enum.Enum): + """ + Types of action scope. + """ + + #: Scope that applies to anything. ANY = 0 + #: Scope that only applies to main scripts. MAIN = 1 + #: Scope that applies to processing steps. PROCESSING = 2 + #: Scope that applies to input file generators. INPUT_FILE_GENERATOR = 3 + #: Scope that applies to output file parsers. OUTPUT_FILE_PARSER = 4 +#: Keyword arguments permitted for particular scopes. ACTION_SCOPE_ALLOWED_KWARGS = { ActionScopeType.ANY.name: set(), ActionScopeType.MAIN.name: set(), @@ -64,30 +80,36 @@ def __new__(cls, value, symbol, colour, doc=None): member.__doc__ = doc return member + #: Not yet associated with a submission. pending = ( 0, ".", "grey46", "Not yet associated with a submission.", ) + #: Associated with a prepared submission that is not yet submitted. prepared = ( 1, ".", "grey46", "Associated with a prepared submission that is not yet submitted.", ) + #: Submitted for execution. submitted = ( 2, ".", "grey46", "Submitted for execution.", ) + #: Executing now. running = ( 3, "●", "dodger_blue1", "Executing now.", ) + #: Not attempted due to a failure of an upstream action on which this depends, + #: or a loop termination condition being satisfied. skipped = ( 4, "s", @@ -97,18 +119,21 @@ def __new__(cls, value, symbol, colour, doc=None): "or a loop termination condition being satisfied." ), ) + #: Aborted by the user; downstream actions will be attempted. aborted = ( 5, "A", "deep_pink4", "Aborted by the user; downstream actions will be attempted.", ) + #: Probably exited successfully. success = ( 6, "■", "green3", "Probably exited successfully.", ) + #: Probably failed. error = ( 7, "E", @@ -128,10 +153,57 @@ def get_non_running_submitted_states(cls): @property def rich_repr(self): + """ + The rich representation of the value. + """ return f"[{self.colour}]{self.symbol}[/{self.colour}]" class ElementActionRun: + """ + The Element Action Run (EAR) is an atomic unit of an enacted workflow, representing + one unit of work (e.g., particular submitted job to run a program) within that + overall workflow. With looping over, say, parameter spaces, there may be many EARs + per element. + + Parameters + ---------- + id_: int + The ID of the EAR. + is_pending: bool + Whether this EAR is pending. + element_action: + The particular element action that this is a run of. + index: int: + The index of the run within the collection of runs. + data_idx: dict + Used for looking up input data to the EAR. + commands_idx: list[int] + Indices of commands to apply. + start_time: datetime + Time of start of run, if the run has ever been started. + end_time: datetime + Time of end of run, if the run has ever ended. + snapshot_start: dict + Parameters for taking a snapshot of the data directory before the run. + If unspecified, no snapshot will be taken. + snapshot_end: dict + Parameters for taking a snapshot of the data directory after the run. + If unspecified, no snapshot will be taken. + submission_idx: int + What submission was this (if it has been submitted)? + success: bool + Whether this EAR succeeded (if it has run). + skip: bool + Whether this EAR was skipped. + exit_code: int + The exit code, if known. + metadata: dict + Metadata about the EAR. + run_hostname: str + Where to run the EAR (if not locally). + """ + _app_attr = "app" def __init__( @@ -189,14 +261,23 @@ def __repr__(self): @property def id_(self) -> int: + """ + The ID of the EAR. + """ return self._id @property def is_pending(self) -> bool: + """ + Whether this EAR is pending. + """ return self._is_pending @property def element_action(self): + """ + The particular element action that this is a run of. + """ return self._element_action @property @@ -206,58 +287,100 @@ def index(self): @property def action(self): + """ + The action this is a run of. + """ return self.element_action.action @property def element_iteration(self): + """ + The iteration information of this run. + """ return self.element_action.element_iteration @property def element(self): + """ + The element this is a run of. + """ return self.element_iteration.element @property def workflow(self): + """ + The workflow this is a run of. + """ return self.element_iteration.workflow @property def data_idx(self): + """ + Used for looking up input data to the EAR. + """ return self._data_idx @property def commands_idx(self): + """ + Indices of commands to apply. + """ return self._commands_idx @property def metadata(self): + """ + Metadata about the EAR. + """ return self._metadata @property def run_hostname(self): + """ + Where to run the EAR, if known/specified. + """ return self._run_hostname @property def start_time(self): + """ + When the EAR started. + """ return self._start_time @property def end_time(self): + """ + When the EAR finished. + """ return self._end_time @property def submission_idx(self): + """ + What actual submission index was this? + """ return self._submission_idx @property def success(self): + """ + Did the EAR succeed? + """ return self._success @property def skip(self): + """ + Was the EAR skipped? + """ return self._skip @property def snapshot_start(self): + """ + The snapshot of the data directory at the start of the run. + """ if self._ss_start_obj is None and self._snapshot_start: self._ss_start_obj = JSONLikeDirSnapShot( root_path=".", @@ -267,14 +390,18 @@ def snapshot_start(self): @property def snapshot_end(self): + """ + The snapshot of the data directory at the end of the run. + """ if self._ss_end_obj is None and self._snapshot_end: self._ss_end_obj = JSONLikeDirSnapShot(root_path=".", **self._snapshot_end) return self._ss_end_obj @property def dir_diff(self) -> DirectorySnapshotDiff: - """Get the changes to the EAR working directory due to the execution of this - EAR.""" + """ + The changes to the EAR working directory due to the execution of this EAR. + """ if self._ss_diff_obj is None and self.snapshot_end: self._ss_diff_obj = DirectorySnapshotDiff( self.snapshot_start, self.snapshot_end @@ -283,15 +410,23 @@ def dir_diff(self) -> DirectorySnapshotDiff: @property def exit_code(self): + """ + The exit code of the underlying program run by the EAR, if known. + """ return self._exit_code @property def task(self): + """ + The task that this EAR is part of the implementation of. + """ return self.element_action.task @property def status(self): - """Return the state of this EAR.""" + """ + The state of this EAR. + """ if self.skip: return EARStatus.skipped @@ -336,6 +471,14 @@ def get_parameter_names(self, prefix: str) -> List[str]: return self.action.get_parameter_names(prefix) def get_data_idx(self, path: str = None): + """ + Get the data index of a value in the most recent iteration. + + Parameters + ---------- + path: + Path to the parameter. + """ return self.element_iteration.get_data_idx( path, action_idx=self.element_action.action_idx, @@ -350,6 +493,20 @@ def get_parameter_sources( as_strings: bool = False, use_task_index: bool = False, ): + """ + Get the source or sources of a parameter in the most recent iteration. + + Parameters + ---------- + path: + Path to the parameter. + typ: + The parameter type. + as_strings: + Whether to return the result as human-readable strings. + use_task_index: + Whether to use the task index. + """ return self.element_iteration.get_parameter_sources( path, action_idx=self.element_action.action_idx, @@ -366,6 +523,22 @@ def get( raise_on_missing: bool = False, raise_on_unset: bool = False, ): + """ + Get a value (parameter, input, output, etc.) from the most recent iteration. + + Parameters + ---------- + path: + Path to the value. + default: + Default value to provide if value absent. + raise_on_missing: + Whether to raise an exception on an absent value. + If not, the default is returned. + raise_on_unset: + Whether to raise an exception on an explicitly unset value. + If not, the default is returned. + """ return self.element_iteration.get( path=path, action_idx=self.element_action.action_idx, @@ -436,12 +609,18 @@ def get_dependent_EARs( @property def inputs(self): + """ + The inputs to this EAR. + """ if not self._inputs: self._inputs = self.app.ElementInputs(element_action_run=self) return self._inputs @property def outputs(self): + """ + The outputs from this EAR. + """ if not self._outputs: self._outputs = self.app.ElementOutputs(element_action_run=self) return self._outputs @@ -449,24 +628,36 @@ def outputs(self): @property @TimeIt.decorator def resources(self): + """ + The resources to use with (or used by) this EAR. + """ if not self._resources: self._resources = self.app.ElementResources(**self.get_resources()) return self._resources @property def input_files(self): + """ + The input files to the controlled program. + """ if not self._input_files: self._input_files = self.app.ElementInputFiles(element_action_run=self) return self._input_files @property def output_files(self): + """ + The output files from the controlled program. + """ if not self._output_files: self._output_files = self.app.ElementOutputFiles(element_action_run=self) return self._output_files @property def env_spec(self) -> Dict[str, Any]: + """ + Environment details. + """ return self.resources.environments[self.action.get_environment_name()] @TimeIt.decorator @@ -476,9 +667,15 @@ def get_resources(self): return self.element_iteration.get_resources(self.action) def get_environment_spec(self) -> str: + """ + What environment to run in? + """ return self.action.get_environment_spec() def get_environment(self) -> app.Environment: + """ + What environment to run in? + """ return self.action.get_environment() def get_all_previous_iteration_runs(self, include_self: bool = True): @@ -502,13 +699,13 @@ def get_input_values( Parameters ---------- - inputs + inputs: If specified, a list of input parameter types to include, or a dict whose keys are input parameter types to include. For schema inputs that have `multiple=True`, the input type should be labelled. If a dict is passed, and the key "all_iterations` is present and `True`, the return for that input will be structured to include values for all previous iterations. - label_dict + label_dict: If True, arrange the values of schema inputs with multiple=True as a dict whose keys are the labels. If False, labels will be included in the top level keys. @@ -565,6 +762,9 @@ def get_input_values_direct(self, label_dict: bool = True): return self.get_input_values(inputs=inputs, label_dict=label_dict) def get_IFG_input_values(self) -> Dict[str, Any]: + """ + Get a dict of input values that are to be passed via an input file generator. + """ if not self.action._from_expand: raise RuntimeError( f"Cannot get input file generator inputs from this EAR because the " @@ -583,6 +783,10 @@ def get_IFG_input_values(self) -> Dict[str, Any]: return inputs def get_OFP_output_files(self) -> Dict[str, Union[str, List[str]]]: + """ + Get a dict of output files that are going to be parsed to generate one or more + outputs. + """ # TODO: can this return multiple files for a given FileSpec? if not self.action._from_expand: raise RuntimeError( @@ -595,6 +799,9 @@ def get_OFP_output_files(self) -> Dict[str, Union[str, List[str]]]: return out_files def get_OFP_inputs(self) -> Dict[str, Union[str, List[str]]]: + """ + Get a dict of input values that are to be passed to output file parsers. + """ if not self.action._from_expand: raise RuntimeError( f"Cannot get output file parser inputs from this from EAR because the " @@ -610,6 +817,9 @@ def get_OFP_inputs(self) -> Dict[str, Union[str, List[str]]]: return inputs def get_OFP_outputs(self) -> Dict[str, Union[str, List[str]]]: + """ + Get the outputs obtained by parsing an output file. + """ if not self.action._from_expand: raise RuntimeError( f"Cannot get output file parser outputs from this from EAR because the " @@ -621,6 +831,9 @@ def get_OFP_outputs(self) -> Dict[str, Union[str, List[str]]]: return outputs def write_source(self, js_idx: int, js_act_idx: int): + """ + Write values to files in standard formats. + """ import h5py for fmt, ins in self.action.script_data_in_grouped.items(): @@ -690,10 +903,13 @@ def compose_commands( self, jobscript: app.Jobscript, JS_action_idx: int ) -> Tuple[str, List[str], List[int]]: """ + Write the EAR's enactment to disk in preparation for submission. + Returns ------- - commands - shell_vars + commands: + List of argument words for the command that enacts the EAR. + shell_vars: Dict whose keys are command indices, and whose values are lists of tuples, where each tuple contains: (parameter name, shell variable name, "stdout"/"stderr"). @@ -735,6 +951,20 @@ def compose_commands( class ElementAction: + """ + An abstract representation of an element's action at a particular iteration and + the runs that enact that element iteration. + + Parameters + ---------- + element_iteration: + The iteration + action_idx: + The action index. + runs: + The list of run indices. + """ + _app_attr = "app" def __init__(self, element_iteration, action_idx, runs): @@ -761,18 +991,30 @@ def __repr__(self): @property def element_iteration(self): + """ + The iteration for this action. + """ return self._element_iteration @property def element(self): + """ + The element for this action. + """ return self.element_iteration.element @property def num_runs(self): + """ + The number of runs associated with this action. + """ return len(self._runs) @property def runs(self): + """ + The EARs that this action is enacted by. + """ if self._run_objs is None: self._run_objs = [ self.app.ElementActionRun( @@ -790,41 +1032,65 @@ def runs(self): @property def task(self): + """ + The task that this action is an instance of. + """ return self.element_iteration.task @property def action_idx(self): + """ + The index of the action. + """ return self._action_idx @property def action(self): + """ + The abstract task that this is a concrete model of. + """ return self.task.template.get_schema_action(self.action_idx) @property def inputs(self): + """ + The inputs to this action. + """ if not self._inputs: self._inputs = self.app.ElementInputs(element_action=self) return self._inputs @property def outputs(self): + """ + The outputs from this action. + """ if not self._outputs: self._outputs = self.app.ElementOutputs(element_action=self) return self._outputs @property def input_files(self): + """ + The input files to this action. + """ if not self._input_files: self._input_files = self.app.ElementInputFiles(element_action=self) return self._input_files @property def output_files(self): + """ + The output files from this action. + """ if not self._output_files: self._output_files = self.app.ElementOutputFiles(element_action=self) return self._output_files def get_data_idx(self, path: str = None, run_idx: int = -1): + """ + Get the data index for some path/run. + """ return self.element_iteration.get_data_idx( path, action_idx=self.action_idx, @@ -839,6 +1105,9 @@ def get_parameter_sources( as_strings: bool = False, use_task_index: bool = False, ): + """ + Get information about where parameters originated. + """ return self.element_iteration.get_parameter_sources( path, action_idx=self.action_idx, @@ -856,6 +1125,9 @@ def get( raise_on_missing: bool = False, raise_on_unset: bool = False, ): + """ + Get the value of a parameter. + """ return self.element_iteration.get( path=path, action_idx=self.action_idx, @@ -868,8 +1140,8 @@ def get( def get_parameter_names(self, prefix: str) -> List[str]: """Get parameter types associated with a given prefix. - For inputs, labels are ignored. See `Action.get_parameter_names` for more - information. + For inputs, labels are ignored. + See :py:meth:`.Action.get_parameter_names` for more information. Parameters ---------- @@ -898,7 +1170,9 @@ def __init__(self, typ: Union[app.ActionScopeType, str], **kwargs): if isinstance(typ, str): typ = getattr(self.app.ActionScopeType, typ.upper()) + #: Action scope type. self.typ = typ + #: Any provided extra keyword arguments. self.kwargs = {k: v for k, v in kwargs.items() if v is not None} bad_keys = set(kwargs.keys()) - ACTION_SCOPE_ALLOWED_KWARGS[self.typ.name] @@ -932,6 +1206,9 @@ def _parse_from_string(cls, string): return {"type": typ_str, **kwargs} def to_string(self): + """ + Render this action scope as a string. + """ kwargs_str = "" if self.kwargs: kwargs_str = "[" + ", ".join(f"{k}={v}" for k, v in self.kwargs.items()) + "]" @@ -948,27 +1225,46 @@ def from_json_like(cls, json_like, shared_data=None): @classmethod def any(cls): + """ + Any scope. + """ return cls(typ=ActionScopeType.ANY) @classmethod def main(cls): + """ + The main scope. + """ return cls(typ=ActionScopeType.MAIN) @classmethod def processing(cls): + """ + The processing scope. + """ return cls(typ=ActionScopeType.PROCESSING) @classmethod def input_file_generator(cls, file=None): + """ + The scope of an input file generator. + """ return cls(typ=ActionScopeType.INPUT_FILE_GENERATOR, file=file) @classmethod def output_file_parser(cls, output=None): + """ + The scope of an output file parser. + """ return cls(typ=ActionScopeType.OUTPUT_FILE_PARSER, output=output) @dataclass class ActionEnvironment(JSONLike): + """ + The environment that an action is enacted within. + """ + _app_attr = "app" _child_objects = ( @@ -978,7 +1274,9 @@ class ActionEnvironment(JSONLike): ), ) + #: The environment document. environment: Union[str, Dict[str, Any]] + #: The scope. scope: Optional[app.ActionScope] = None def __post_init__(self): @@ -998,8 +1296,27 @@ def __post_init__(self): class ActionRule(JSONLike): - """Class to represent a rule/condition that must be True if an action is to be - included.""" + """ + Class to represent a rule/condition that must be True if an action is to be + included. + + Parameters + ---------- + rule: ~hpcflow.app.Rule + The rule to apply. + check_exists: str + A special rule that is enabled if this named attribute is present. + check_missing: str + A special rule that is enabled if this named attribute is absent. + path: str + Where to find the attribute to check. + condition: dict | ConditionLike + A more complex condition to apply. + cast: str + The name of a class to cast the attribute to before checking. + doc: str + Documentation for this rule, if any. + """ _child_objects = (ChildObjectSpec(name="rule", class_name="Rule"),) @@ -1031,8 +1348,11 @@ def __init__( f"constructor arguments." ) + #: The rule to apply. self.rule = rule + #: The action that contains this rule. self.action = None # assigned by parent action + #: The command that is guarded by this rule. self.command = None # assigned by parent command def __eq__(self, other): @@ -1044,19 +1364,85 @@ def __eq__(self, other): @TimeIt.decorator def test(self, element_iteration: app.ElementIteration) -> bool: + """ + Test if this rule holds for a particular iteration. + + Parameter + --------- + element_iteration: + The iteration to apply this rule to. + """ return self.rule.test(element_like=element_iteration, action=self.action) @classmethod def check_exists(cls, check_exists): + """ + Make an action rule that checks if a named attribute is present. + + Parameter + --------- + check_exists: str + The path to the attribute to check for. + """ return cls(rule=app.Rule(check_exists=check_exists)) @classmethod def check_missing(cls, check_missing): + """ + Make an action rule that checks if a named attribute is absent. + + Parameter + --------- + check_missing: str + The path to the attribute to check for. + """ return cls(rule=app.Rule(check_missing=check_missing)) class Action(JSONLike): - """""" + """ + An atomic component of a workflow that will be enacted within an iteration + structure. + + Parameters + ---------- + environments: list[ActionEnvironment] + The environments in which this action can run. + commands: list[~hpcflow.app.Command] + The commands to be run by this action. + script: str + The name of the Python script to run. + script_data_in: str + Information about data input to the script. + script_data_out: str + Information about data output from the script. + script_data_files_use_opt: bool + If True, script data input and output file paths will be passed to the script + execution command line with an option like ``--input-json`` or ``--output-hdf5`` + etc. If False, the file paths will be passed on their own. For Python scripts, + options are always passed, and this parameter is overwritten to be True, + regardless of its initial value. + script_exe: str + The executable to use to run the script. + script_pass_env_spec: bool + Whether to pass the environment details to the script. + abortable: bool + Whether this action can be aborted. + input_file_generators: list[~hpcflow.app.InputFileGenerator] + Any applicable input file generators. + output_file_parsers: list[~hpcflow.app.OutputFileParser] + Any applicable output file parsers. + input_files: list[~hpcflow.app.FileSpec] + The input files to the action's commands. + output_files: list[~hpcflow.app.FileSpec] + The output files from the action's commands. + rules: list[ActionRule] + How to determine whether to run the action. + save_files: list[str] + The names of files to be explicitly saved after each step. + clean_up: list[str] + The names of files to be deleted after each step. + """ _app_attr = "app" _child_objects = ( @@ -1136,36 +1522,45 @@ def __init__( save_files: Optional[List[str]] = None, clean_up: Optional[List[str]] = None, ): - """ - Parameters - ---------- - script_data_files_use_opt - If True, script data input and output file paths will be passed to the script - execution command line with an option like `--input-json` or `--output-hdf5` - etc. If False, the file paths will be passed on their own. For Python scripts, - options are always passed, and this parameter is overwritten to be True, - regardless of its initial value. - - """ + #: The commands to be run by this action. self.commands = commands or [] + #: The name of the Python script to run. self.script = script + #: Information about data input to the script. self.script_data_in = script_data_in + #: Information about data output from the script. self.script_data_out = script_data_out + #: If True, script data input and output file paths will be passed to the script + #: execution command line with an option like `--input-json` or `--output-hdf5` + #: etc. If False, the file paths will be passed on their own. For Python scripts, + #: options are always passed, and this parameter is overwritten to be True, + #: regardless of its initial value. self.script_data_files_use_opt = ( script_data_files_use_opt if not self.script_is_python else True ) + #: The executable to use to run the script. self.script_exe = script_exe.lower() if script_exe else None + #: Whether to pass the environment details to the script. self.script_pass_env_spec = script_pass_env_spec + #: The environments in which this action can run. self.environments = environments or [ self.app.ActionEnvironment(environment="null_env") ] + #: Whether this action can be aborted. self.abortable = abortable + #: Any applicable input file generators. self.input_file_generators = input_file_generators or [] + #: Any applicable output file parsers. self.output_file_parsers = output_file_parsers or [] + #: The input files to the action's commands. self.input_files = self._resolve_input_files(input_files or []) + #: The output files from the action's commands. self.output_files = self._resolve_output_files(output_files or []) + #: How to determine whether to run the action. self.rules = rules or [] + #: The names of files to be explicitly saved after each step. self.save_files = save_files or [] + #: The names of files to be deleted after each step. self.clean_up = clean_up or [] self._task_schema = None # assigned by parent TaskSchema @@ -1174,6 +1569,9 @@ def __init__( self._set_parent_refs() def process_script_data_formats(self): + """ + Convert script data information into standard form. + """ self.script_data_in = self._process_script_data_in(self.script_data_in) self.script_data_out = self._process_script_data_out(self.script_data_out) @@ -1317,6 +1715,9 @@ def __deepcopy__(self, memo): @property def task_schema(self): + """ + The task schema that this action came from. + """ return self._task_schema def _resolve_input_files(self, input_files): @@ -1397,7 +1798,7 @@ def get_parameter_dependence(self, parameter: app.SchemaParameter): out = {"input_file_writers": writer_files, "commands": commands} return out - def get_resolved_action_env( + def _get_resolved_action_env( self, relevant_scopes: Tuple[app.ActionScopeType], input_file_generator: app.InputFileGenerator = None, @@ -1427,7 +1828,10 @@ def get_resolved_action_env( def get_input_file_generator_action_env( self, input_file_generator: app.InputFileGenerator ): - return self.get_resolved_action_env( + """ + Get the actual environment to use for an input file generator. + """ + return self._get_resolved_action_env( relevant_scopes=( ActionScopeType.ANY, ActionScopeType.PROCESSING, @@ -1437,7 +1841,10 @@ def get_input_file_generator_action_env( ) def get_output_file_parser_action_env(self, output_file_parser: app.OutputFileParser): - return self.get_resolved_action_env( + """ + Get the actual environment to use for an output file parser. + """ + return self._get_resolved_action_env( relevant_scopes=( ActionScopeType.ANY, ActionScopeType.PROCESSING, @@ -1447,15 +1854,24 @@ def get_output_file_parser_action_env(self, output_file_parser: app.OutputFilePa ) def get_commands_action_env(self): - return self.get_resolved_action_env( + """ + Get the actual environment to use for the action commands. + """ + return self._get_resolved_action_env( relevant_scopes=(ActionScopeType.ANY, ActionScopeType.MAIN), commands=self.commands, ) def get_environment_name(self) -> str: + """ + Get the name of the primary environment. + """ return self.get_environment_spec()["name"] def get_environment_spec(self) -> Dict[str, Any]: + """ + Get the specification for the primary envionment, assuming it has been expanded. + """ if not self._from_expand: raise RuntimeError( f"Cannot choose a single environment from this action because it is not " @@ -1464,6 +1880,9 @@ def get_environment_spec(self) -> Dict[str, Any]: return self.environments[0].environment def get_environment(self) -> app.Environment: + """ + Get the primary environment. + """ return self.app.envs.get(**self.get_environment_spec()) @staticmethod @@ -1487,6 +1906,9 @@ def get_script_name(cls, script: str) -> str: def get_snippet_script_str( cls, script, env_spec: Optional[Dict[str, Any]] = None ) -> str: + """ + Get the substituted script snippet path as a string. + """ if not cls.is_snippet_script(script): raise ValueError( f"Must be an app-data script name (e.g. " @@ -1508,6 +1930,9 @@ def get_snippet_script_str( def get_snippet_script_path( cls, script_path, env_spec: Optional[Dict[str, Any]] = None ) -> Path: + """ + Get the substituted script snippet path, or False if there is no snippet. + """ if not cls.is_snippet_script(script_path): return False @@ -1518,26 +1943,42 @@ def get_snippet_script_path( return Path(path) @staticmethod - def get_param_dump_file_stem(js_idx: int, js_act_idx: int): + def __get_param_dump_file_stem(js_idx: int, js_act_idx: int): return RunDirAppFiles.get_run_param_dump_file_prefix(js_idx, js_act_idx) @staticmethod - def get_param_load_file_stem(js_idx: int, js_act_idx: int): + def __get_param_load_file_stem(js_idx: int, js_act_idx: int): return RunDirAppFiles.get_run_param_load_file_prefix(js_idx, js_act_idx) def get_param_dump_file_path_JSON(self, js_idx: int, js_act_idx: int): - return Path(self.get_param_dump_file_stem(js_idx, js_act_idx) + ".json") + """ + Get the path of the JSON dump file. + """ + return Path(self.__get_param_dump_file_stem(js_idx, js_act_idx) + ".json") def get_param_dump_file_path_HDF5(self, js_idx: int, js_act_idx: int): - return Path(self.get_param_dump_file_stem(js_idx, js_act_idx) + ".h5") + """ + Get the path of the HDF56 dump file. + """ + return Path(self.__get_param_dump_file_stem(js_idx, js_act_idx) + ".h5") def get_param_load_file_path_JSON(self, js_idx: int, js_act_idx: int): - return Path(self.get_param_load_file_stem(js_idx, js_act_idx) + ".json") + """ + Get the path of the JSON load file. + """ + return Path(self.__get_param_load_file_stem(js_idx, js_act_idx) + ".json") def get_param_load_file_path_HDF5(self, js_idx: int, js_act_idx: int): - return Path(self.get_param_load_file_stem(js_idx, js_act_idx) + ".h5") + """ + Get the path of the HDF5 load file. + """ + return Path(self.__get_param_load_file_stem(js_idx, js_act_idx) + ".h5") def expand(self): + """ + Expand this action into a list of actions if necessary. + This converts input file generators and output file parsers into their own actions. + """ if self._from_expand: # already expanded return [self] @@ -1694,7 +2135,7 @@ def get_command_input_types(self, sub_parameters: bool = False) -> Tuple[str]: Parameters ---------- - sub_parameters + sub_parameters: If True, sub-parameters (i.e. dot-delimited parameter types) will be returned untouched. If False (default), only return the root parameter type and disregard the sub-parameter part. @@ -1747,7 +2188,7 @@ def get_input_types(self, sub_parameters: bool = False) -> Tuple[str]: Parameters ---------- - sub_parameters + sub_parameters: If True, sub-parameters (i.e. dot-delimited parameter types) in command line inputs will be returned untouched. If False (default), only return the root parameter type and disregard the sub-parameter part. @@ -1786,9 +2227,15 @@ def get_output_types(self) -> Tuple[str]: return tuple(set(params)) def get_input_file_labels(self): + """ + Get the labels from the input files. + """ return tuple(i.label for i in self.input_files) def get_output_file_labels(self): + """ + Get the labels from the output files. + """ return tuple(i.label for i in self.output_files) @TimeIt.decorator @@ -1919,6 +2366,10 @@ def get_possible_scopes(self) -> Tuple[app.ActionScope]: return scopes def get_precise_scope(self) -> app.ActionScope: + """ + Get the exact scope of this action. + The action must have been expanded prior to calling this. + """ if not self._from_expand: raise RuntimeError( "Precise scope cannot be unambiguously defined until the Action has been " @@ -1942,6 +2393,9 @@ def get_precise_scope(self) -> app.ActionScope: def is_input_type_required( self, typ: str, provided_files: List[app.FileSpec] ) -> bool: + """ + Determine if the given input type is required by this action. + """ # TODO: for now assume a script takes all inputs if ( self.script @@ -1966,6 +2420,9 @@ def is_input_type_required( if typ in (OFP.inputs or []): return True + # Appears to be not required + return False + @TimeIt.decorator def test_rules(self, element_iter) -> Tuple[bool, List[int]]: """Test all rules against the specified element iteration.""" diff --git a/hpcflow/sdk/core/cache.py b/hpcflow/sdk/core/cache.py index abe945774..4f7d5a9db 100644 --- a/hpcflow/sdk/core/cache.py +++ b/hpcflow/sdk/core/cache.py @@ -1,3 +1,7 @@ +""" +Dependency resolution cache. +""" + from collections import defaultdict from dataclasses import dataclass from typing import Set, Dict @@ -9,21 +13,39 @@ class DependencyCache: """Class to bulk-retrieve dependencies between elements, iterations, and runs.""" + #: What EARs (by ID) a given EAR depends on. run_dependencies: Dict[int, Set] + #: What EARs (by ID) are depending on a given EAR. run_dependents: Dict[int, Set] + #: What EARs (by ID) a given iteration depends on. iter_run_dependencies: Dict[int, Set] + #: What iterations (by ID) a given iteration depends on. iter_iter_dependencies: Dict[int, Set] + #: What iterations (by ID) a given element depends on. elem_iter_dependencies: Dict[int, Set] + #: What elements (by ID) a given element depends on. elem_elem_dependencies: Dict[int, Set] + #: What elements (by ID) are depending on a given element. elem_elem_dependents: Dict[int, Set] + #: Transitive closure of :py:attr:`elem_elem_dependents`. elem_elem_dependents_rec: Dict[int, Set] + #: The elements of the workflow that this cache was built from. elements: Dict + #: The iterations of the workflow that this cache was built from. iterations: Dict @classmethod @TimeIt.decorator def build(cls, workflow): + """ + Build a cache instance. + + Parameters + ---------- + workflow: ~hpcflow.app.Workflow + The workflow to build the cache from. + """ num_iters = workflow.num_element_iterations num_elems = workflow.num_elements num_runs = workflow.num_EARs diff --git a/hpcflow/sdk/core/command_files.py b/hpcflow/sdk/core/command_files.py index 2a2275c94..d3772a634 100644 --- a/hpcflow/sdk/core/command_files.py +++ b/hpcflow/sdk/core/command_files.py @@ -1,3 +1,7 @@ +""" +Model of files that hold commands. +""" + from __future__ import annotations import copy from dataclasses import dataclass, field @@ -15,12 +19,18 @@ @dataclass class FileSpec(JSONLike): + """ + A specification of a file handled by a workflow. + """ + _app_attr = "app" _validation_schema = "files_spec_schema.yaml" _child_objects = (ChildObjectSpec(name="name", class_name="FileNameSpec"),) + #: Label for this file specification. label: str + #: The name of the file. name: str _hash_value: Optional[str] = field(default=None, repr=False) @@ -30,6 +40,9 @@ def __post_init__(self): ) def value(self, directory="."): + """ + The path to a file, optionally resolved with respect to a particular directory. + """ return self.name.value(directory) def __eq__(self, other: object) -> bool: @@ -41,19 +54,42 @@ def __eq__(self, other: object) -> bool: @property def stem(self): + """ + The stem of the file name. + """ return self.name.stem @property def ext(self): + """ + The extension of the file name. + """ return self.name.ext class FileNameSpec(JSONLike): + """ + The name of a file handled by a workflow, or a pattern that matches multiple files. + + Parameters + ---------- + name: str + The name or pattern. + args: list + Positional arguments to use when formatting the name. + Can be omitted if the name does not contain a Python formatting pattern. + is_regex: bool + If true, the name is used as a regex to search for actual files. + """ + _app_attr = "app" def __init__(self, name, args=None, is_regex=False): + #: The name or pattern. self.name = name + #: Positional arguments to use when formatting the name. self.args = args + #: Whether the name is used as a regex to search for actual files. self.is_regex = is_regex def __eq__(self, other: object) -> bool: @@ -67,13 +103,28 @@ def __eq__(self, other: object) -> bool: @property def stem(self): + """ + The stem of the name or pattern. + """ return self.app.FileNameStem(self) @property def ext(self): + """ + The extension of the name or pattern. + """ return self.app.FileNameExt(self) def value(self, directory="."): + """ + Get the template-resolved name of the file + (or files matched if the name is a regex pattern). + + Parameters + ---------- + directory: str + Where to resolve values with respect to. + """ format_args = [i.value(directory) for i in self.args or []] value = self.name.format(*format_args) if self.is_regex: @@ -86,22 +137,60 @@ def __repr__(self): @dataclass class FileNameStem(JSONLike): + """ + The stem of a file name. + """ + + #: The file specification this is derived from. file_name: app.FileNameSpec def value(self, directory=None): + """ + Get the stem, possibly with directory specified. + """ return Path(self.file_name.value(directory)).stem @dataclass class FileNameExt(JSONLike): + """ + The extension of a file name. + """ + + #: The file specification this is derived from. file_name: app.FileNameSpec def value(self, directory=None): + """ + Get the extension. + """ return Path(self.file_name.value(directory)).suffix @dataclass class InputFileGenerator(JSONLike): + """ + Represents a script that is run to generate input files for an action. + + Parameters + ---------- + input_file: + The file to generate. + inputs: list[~hpcflow.app.Parameter] + The input parameters to the generator. + script: + The script that generates the input. + environment: + The environment in which to run the generator. + script_pass_env_spec: + Whether to pass in the environment. + abortable: + Whether the generator can be stopped early. + Quick-running scripts tend to not need this. + rules: list[~hpcflow.app.ActionRule] + User-specified rules for whether to run the generator. + """ + _app_attr = "app" _child_objects = ( @@ -127,12 +216,20 @@ class InputFileGenerator(JSONLike): ), ) + #: The file to generate. input_file: app.FileSpec + #: The input parameters to the generator. inputs: List[app.Parameter] + #: The script that generates the inputs. script: str = None + #: The environment in which to run the generator. environment: app.Environment = None + #: Whether to pass in the environment. script_pass_env_spec: Optional[bool] = False + #: Whether the generator can be stopped early. + #: Quick-running scripts tend to not need this. abortable: Optional[bool] = False + #: User-specified rules for whether to run the generator. rules: Optional[List[app.ActionRule]] = None def __post_init__(self): @@ -190,9 +287,10 @@ def compose_source(self, snip_path) -> str: return out def write_source(self, action, env_spec: Dict[str, Any]): - - # write the script if it is specified as a snippet script, otherwise we assume - # the script already exists in the working directory: + """ + Write the script if it is specified as a snippet script, otherwise we assume + the script already exists in the working directory. + """ snip_path = action.get_snippet_script_path(self.script, env_spec) if snip_path: source_str = self.compose_source(snip_path) @@ -203,13 +301,35 @@ def write_source(self, action, env_spec: Dict[str, Any]): @dataclass class OutputFileParser(JSONLike): """ + Represents a script that is run to parse output files from an action and create outputs. + Parameters ---------- - output + output_files: list[FileSpec] + The output files that this parser will parse. + output: ~hpcflow.app.Parameter The singular output parsed by this parser. Not to be confused with `outputs` (plural). - outputs + script: str + The name of the file containing the output file parser source. + environment: ~hpcflow.app.Environment + The environment to use to run the parser. + inputs: list[str] + The other inputs to the parser. + outputs: list[str] Optional multiple outputs from the upstream actions of the schema that are required to parametrise this parser. + options: dict + Miscellaneous options. + script_pass_env_spec: bool + Whether to pass the environment specifier to the script. + abortable: bool + Whether this script can be aborted. + save_files: list[str] + The files that should be saved to the persistent store for the workflow. + clean_files: list[str] + The files that should be immediately removed. + rules: list[~hpcflow.app.ActionRule] + Rules for whether to enable this parser. """ _child_objects = ( @@ -249,17 +369,32 @@ class OutputFileParser(JSONLike): ), ) + #: The output files that this parser will parse. output_files: List[app.FileSpec] + #: The singular output parsed by this parser. + #: Not to be confused with :py:attr:`outputs` (plural). output: Optional[app.Parameter] = None + #: The name of the file containing the output file parser source. script: str = None + #: The environment to use to run the parser. environment: Environment = None + #: The other inputs to the parser. inputs: List[str] = None + #: Optional multiple outputs from the upstream actions of the schema that are + #: required to parametrise this parser. + #: Not to be confused with :py:attr:`output` (plural). outputs: List[str] = None + #: Miscellaneous options. options: Dict = None + #: Whether to pass the environment specifier to the script. script_pass_env_spec: Optional[bool] = False + #: Whether this script can be aborted. abortable: Optional[bool] = False + #: The files that should be saved to the persistent store for the workflow. save_files: Union[List[str], bool] = True + #: The files that should be immediately removed. clean_up: Optional[List[str]] = None + #: Rules for whether to enable this parser. rules: Optional[List[app.ActionRule]] = None def __post_init__(self): @@ -345,6 +480,9 @@ def compose_source(self, snip_path) -> str: return out def write_source(self, action, env_spec: Dict[str, Any]): + """ + Write the actual output parser to a file so it can be enacted. + """ if self.output is None: # might be used just for saving files: return @@ -485,20 +623,32 @@ def _get_value(self, value_name=None): return val def read_contents(self): + """ + Get the actual contents of the file. + """ with self.path.open("r") as fh: return fh.read() @property def path(self): + """ + The path to the file. + """ path = self._get_value("path") return Path(path) if path else None @property def store_contents(self): + """ + Whether the file's contents are stored in the workflow's persistent store. + """ return self._get_value("store_contents") @property def contents(self): + """ + The contents of the file. + """ if self.store_contents: contents = self._get_value("contents") else: @@ -508,10 +658,16 @@ def contents(self): @property def extension(self): + """ + The extension of the file. + """ return self._get_value("extension") @property def workflow(self) -> app.Workflow: + """ + The owning workflow. + """ if self._workflow: return self._workflow elif self._element_set: @@ -519,6 +675,23 @@ def workflow(self) -> app.Workflow: class InputFile(_FileContentsSpecifier): + """ + An input file. + + Parameters + ---------- + file: + What file is this? + path: Path + Where is the (original) file? + contents: str + What is the contents of the file (if already known)? + extension: str + What is the extension of the file? + store_contents: bool + Are the file's contents to be cached in the workflow persistent store? + """ + _child_objects = ( ChildObjectSpec( name="file", @@ -536,6 +709,7 @@ def __init__( extension: Optional[str] = "", store_contents: Optional[bool] = True, ): + #: What file is this? self.file = file if not isinstance(self.file, FileSpec): self.file = self.app.command_files.get(self.file.label) @@ -571,14 +745,39 @@ def __repr__(self): @property def normalised_files_path(self): + """ + Standard name for the file within the workflow. + """ return self.file.label @property def normalised_path(self): + """ + Full workflow value path to the file. + + Note + ---- + This is not the same as the path in the filesystem. + """ return f"input_files.{self.normalised_files_path}" class InputFileGeneratorSource(_FileContentsSpecifier): + """ + The source of code for use in an input file generator. + + Parameters + ---------- + generator: + How to generate the file. + path: + Path to the file. + contents: + Contents of the file. Only used when recreating this object. + extension: + File name extension. + """ + def __init__( self, generator: app.InputFileGenerator, @@ -586,11 +785,27 @@ def __init__( contents: str = None, extension: str = "", ): + #: How to generate the file. self.generator = generator super().__init__(path, contents, extension) class OutputFileParserSource(_FileContentsSpecifier): + """ + The source of code for use in an output file parser. + + Parameters + ---------- + parser: + How to parse the file. + path: Path + Path to the file. + contents: + Contents of the file. Only used when recreating this object. + extension: + File name extension. + """ + def __init__( self, parser: app.OutputFileParser, @@ -598,5 +813,6 @@ def __init__( contents: str = None, extension: str = "", ): + #: How to parse the file. self.parser = parser super().__init__(path, contents, extension) diff --git a/hpcflow/sdk/core/commands.py b/hpcflow/sdk/core/commands.py index 42ea74a3f..5c93abae7 100644 --- a/hpcflow/sdk/core/commands.py +++ b/hpcflow/sdk/core/commands.py @@ -1,3 +1,7 @@ +""" +Model of a command run in an action. +""" + from dataclasses import dataclass, field from functools import partial from pathlib import Path @@ -15,6 +19,32 @@ @dataclass class Command(JSONLike): + """ + A command that may be run within a workflow action. + + Parameters + ---------- + command: str + The actual command. + executable: str + The executable to run, + from the set of executable managed by the environment. + arguments: list[str] + The arguments to pass in. + variables: dict[str, str] + Values that may be substituted when preparing the arguments. + stdout: str + The name of a file to write standard output to. + stderr: str + The name of a file to write standard error to. + stdin: str + The name of a file to read standard input from. + rules: list[~hpcflow.app.ActionRule] + Rules that state whether this command is eligible to run. + """ + + # TODO: What is the difference between command and executable? + _app_attr = "app" _child_objects = ( ChildObjectSpec( @@ -25,13 +55,23 @@ class Command(JSONLike): ), ) + #: The actual command. + #: Overrides :py:attr:`executable`. command: Optional[str] = None + #: The executable to run, + #: from the set of executable managed by the environment. executable: Optional[str] = None + #: The arguments to pass in. arguments: Optional[List[str]] = None + #: Values that may be substituted when preparing the arguments. variables: Optional[Dict[str, str]] = None + #: The name of a file to write standard output to. stdout: Optional[str] = None + #: The name of a file to write standard error to. stderr: Optional[str] = None + #: The name of a file to read standard input from. stdin: Optional[str] = None + #: Rules that state whether this command is eligible to run. rules: Optional[List[app.ActionRule]] = field(default_factory=lambda: []) def __repr__(self) -> str: @@ -207,6 +247,9 @@ def input_param_repl(match_obj, inp_val): return cmd_str, shell_vars def get_output_types(self): + """ + Get whether stdout and stderr are workflow parameters. + """ # note: we use "parameter" rather than "output", because it could be a schema # output or schema input. pattern = ( @@ -253,6 +296,20 @@ def _prepare_kwargs_from_string(args_str: Union[str, None], doubled_quoted_args= return kwargs def process_std_stream(self, name: str, value: str, stderr: bool): + """ + Process a description of a standard stread from a command to get how it becomes + a workflow parameter for later actions. + + Parameters + --------- + name: + The name of the output, describing how to process things. + value: + The actual value read from the stream. + stderr: + If true, this is handling the stderr stream. If false, the stdout stream. + """ + def _parse_list(lst_str: str, item_type: str = "str", delim: str = " "): return [parse_types[item_type](i) for i in lst_str.split(delim)] diff --git a/hpcflow/sdk/core/element.py b/hpcflow/sdk/core/element.py index 9bf34192f..66ce05bb2 100644 --- a/hpcflow/sdk/core/element.py +++ b/hpcflow/sdk/core/element.py @@ -1,3 +1,7 @@ +""" +Elements are components of tasks. +""" + from __future__ import annotations import copy from dataclasses import dataclass, field @@ -89,7 +93,8 @@ def _task(self): @property def prefixed_names_unlabelled(self) -> Dict[str, List[str]]: - """Get a mapping between inputs types and associated labels. + """ + A mapping between input types and associated labels. If the schema input for a given input type has `multiple=False` (even if a label is defined), the values for that input type will be an empty list. @@ -101,6 +106,9 @@ def prefixed_names_unlabelled(self) -> Dict[str, List[str]]: @property def prefixed_names_unlabelled_str(self): + """ + A description of the prefixed names. + """ return ", ".join(i for i in self.prefixed_names_unlabelled) def __repr__(self): @@ -137,6 +145,19 @@ def __iter__(self): class ElementInputs(_ElementPrefixedParameter): + """ + The inputs to an element. + + Parameters + ---------- + element_iteration: ElementIteration + Which iteration does this refer to? + element_action: ~hpcflow.app.ElementAction + Which action does this refer to? + element_action_run: ~hpcflow.app.ElementActionRun + Which EAR does this refer to? + """ + def __init__( self, element_iteration: Optional[app.ElementIteration] = None, @@ -147,6 +168,19 @@ def __init__( class ElementOutputs(_ElementPrefixedParameter): + """ + The outputs from an element. + + Parameters + ---------- + element_iteration: ElementIteration + Which iteration does this refer to? + element_action: ~hpcflow.app.ElementAction + Which action does this refer to? + element_action_run: ~hpcflow.app.ElementActionRun + Which EAR does this refer to? + """ + def __init__( self, element_iteration: Optional[app.ElementIteration] = None, @@ -157,6 +191,19 @@ def __init__( class ElementInputFiles(_ElementPrefixedParameter): + """ + The input files to an element. + + Parameters + ---------- + element_iteration: ElementIteration + Which iteration does this refer to? + element_action: ~hpcflow.app.ElementAction + Which action does this refer to? + element_action_run: ~hpcflow.app.ElementActionRun + Which EAR does this refer to? + """ + def __init__( self, element_iteration: Optional[app.ElementIteration] = None, @@ -169,6 +216,19 @@ def __init__( class ElementOutputFiles(_ElementPrefixedParameter): + """ + The output files from an element. + + Parameters + ---------- + element_iteration: ElementIteration + Which iteration does this refer to? + element_action: ~hpcflow.app.ElementAction + Which action does this refer to? + element_action_run: ~hpcflow.app.ElementActionRun + Which EAR does this refer to? + """ + def __init__( self, element_iteration: Optional[app.ElementIteration] = None, @@ -182,33 +242,108 @@ def __init__( @dataclass class ElementResources(JSONLike): + """ + The resources an element requires. + + Note + ---- + It is common to leave most of these unspecified. + Many of them have complex interactions with each other. + + Parameters + ---------- + scratch: str + Which scratch space to use. + parallel_mode: ParallelMode + Which parallel mode to use. + num_cores: int + How many cores to request. + num_cores_per_node: int + How many cores per compute node to request. + num_threads: int + How many threads to request. + num_nodes: int + How many compute nodes to request. + scheduler: str + Which scheduler to use. + shell: str + Which system shell to use. + use_job_array: bool + Whether to use array jobs. + max_array_items: int + If using array jobs, up to how many items should be in the job array. + time_limit: str + How long to run for. + scheduler_args: dict[str, Any] + Additional arguments to pass to the scheduler. + shell_args: dict[str, Any] + Additional arguments to pass to the shell. + os_name: str + Which OS to use. + environments: dict + Which execution environments to use. + SGE_parallel_env: str + Which SGE parallel environment to request. + SLURM_partition: str + Which SLURM partition to request. + SLURM_num_tasks: str + How many SLURM tasks to request. + SLURM_num_tasks_per_node: str + How many SLURM tasks per compute node to request. + SLURM_num_nodes: str + How many compute nodes to request. + SLURM_num_cpus_per_task: str + How many CPU cores to ask for per SLURM task. + """ + # TODO: how to specify e.g. high-memory requirement? + #: Which scratch space to use. scratch: Optional[str] = None + #: Which parallel mode to use. parallel_mode: Optional[ParallelMode] = None + #: How many cores to request. num_cores: Optional[int] = None + #: How many cores per compute node to request. num_cores_per_node: Optional[int] = None + #: How many threads to request. num_threads: Optional[int] = None + #: How many compute nodes to request. num_nodes: Optional[int] = None + + #: Which scheduler to use. scheduler: Optional[str] = None + #: Which system shell to use. shell: Optional[str] = None + #: Whether to use array jobs. use_job_array: Optional[bool] = None + #: If using array jobs, up to how many items should be in the job array. max_array_items: Optional[int] = None + #: How long to run for. time_limit: Optional[str] = None - + #: Additional arguments to pass to the scheduler. scheduler_args: Optional[Dict] = None + #: Additional arguments to pass to the shell. shell_args: Optional[Dict] = None + #: Which OS to use. os_name: Optional[str] = None + #: Which execution environments to use. environments: Optional[Dict] = None # SGE scheduler specific: + #: Which SGE parallel environment to request. SGE_parallel_env: str = None # SLURM scheduler specific: + #: Which SLURM partition to request. SLURM_partition: str = None + #: How many SLURM tasks to request. SLURM_num_tasks: str = None + #: How many SLURM tasks per compute node to request. SLURM_num_tasks_per_node: str = None + #: How many compute nodes to request. SLURM_num_nodes: str = None + #: How many CPU cores to ask for per SLURM task. SLURM_num_cpus_per_task: str = None def __post_init__(self): @@ -291,20 +426,32 @@ def get_env_instance_filterable_attributes() -> Tuple[str]: @staticmethod def get_default_os_name(): + """ + Get the default value for OS name. + """ return os.name @classmethod def get_default_shell(cls): + """ + Get the default value for name. + """ return cls.app.config.default_shell @classmethod def get_default_scheduler(cls, os_name, shell_name): + """ + Get the default value for scheduler. + """ if os_name == "nt" and "wsl" in shell_name: # provide a "*_posix" default scheduler on windows if shell is WSL: return "direct_posix" return cls.app.config.default_scheduler def set_defaults(self): + """ + Set defaults for unspecified values that need defaults. + """ if self.os_name is None: self.os_name = self.get_default_os_name() if self.shell is None: @@ -353,6 +500,33 @@ def validate_against_machine(self): class ElementIteration: + """ + A particular iteration of an element. + + Parameters + ---------- + id_ : int + The ID of this iteration. + is_pending: bool + Whether this iteration is pending execution. + index: int + The index of this iteration in its parent element. + element: Element + The element this is an iteration of. + data_idx: dict + The overall element iteration data index, before resolution of EARs. + EARs_initialised: bool + Whether EARs have been set up for the iteration. + EAR_IDs: dict[int, int] + Mapping from iteration number to EAR ID, where known. + EARs: list[dict] + Data about EARs. + schema_parameters: list[str] + Parameters from the schema. + loop_idx: dict[str, int] + Indexing information from the loop. + """ + _app_attr = "app" def __init__( @@ -406,46 +580,79 @@ def EARs_initialised(self): @property def element(self): + """ + The element this is an iteration of. + """ return self._element @property def index(self): + """ + The index of this iteration in its parent element. + """ return self._index @property def id_(self) -> int: + """ + The ID of this iteration. + """ return self._id @property def is_pending(self) -> bool: + """ + Whether this iteration is pending execution. + """ return self._is_pending @property def task(self): + """ + The task this is an iteration of an element for. + """ return self.element.task @property def workflow(self): + """ + The workflow this is a part of. + """ return self.element.workflow @property def loop_idx(self) -> Dict[str, int]: + """ + Indexing information from the loop. + """ return self._loop_idx @property def schema_parameters(self) -> List[str]: + """ + Parameters from the schema. + """ return self._schema_parameters @property def EAR_IDs(self) -> Dict[int, int]: + """ + Mapping from iteration number to EAR ID, where known. + """ return self._EAR_IDs @property def EAR_IDs_flat(self): + """ + The EAR IDs. + """ return [j for i in self.EAR_IDs.values() for j in i] @property def actions(self) -> Dict[app.ElementAction]: + """ + The actions of this iteration. + """ if self._action_objs is None: self._action_objs = { act_idx: self.app.ElementAction( @@ -459,30 +666,44 @@ def actions(self) -> Dict[app.ElementAction]: @property def action_runs(self) -> List[app.ElementActionRun]: - """Get a list of element action runs, where only the final run is taken for each - element action.""" + """ + A list of element action runs, where only the final run is taken for each + element action. + """ return [i.runs[-1] for i in self.actions.values()] @property def inputs(self) -> app.ElementInputs: + """ + The inputs to this element. + """ if not self._inputs: self._inputs = self.app.ElementInputs(element_iteration=self) return self._inputs @property def outputs(self) -> app.ElementOutputs: + """ + The outputs from this element. + """ if not self._outputs: self._outputs = self.app.ElementOutputs(element_iteration=self) return self._outputs @property def input_files(self) -> app.ElementInputFiles: + """ + The input files to this element. + """ if not self._input_files: self._input_files = self.app.ElementInputFiles(element_iteration=self) return self._input_files @property def output_files(self) -> app.ElementOutputFiles: + """ + The output files from this element. + """ if not self._output_files: self._output_files = self.app.ElementOutputFiles(element_iteration=self) return self._output_files @@ -521,10 +742,16 @@ def get_data_idx( run_idx: int = -1, ) -> Dict[str, int]: """ + Get the data index. + Parameters ---------- - action_idx + path: + If specified, filters the data indices to the ones relevant to this path. + action_idx: The index of the action within the schema. + run_idx: + The index of the run within the action. """ if not self.actions: @@ -563,6 +790,8 @@ def get_parameter_sources( use_task_index: bool = False, ) -> Dict[str, Union[str, Dict[str, Any]]]: """ + Get the origin of parameters. + Parameters ---------- use_task_index @@ -938,10 +1167,39 @@ def get_resources(self, action: app.Action, set_defaults: bool = False) -> Dict: def get_resources_obj( self, action: app.Action, set_defaults: bool = False ) -> app.ElementResources: + """ + Get the resources for an action (see :py:meth:`get_resources`) + as a searchable model. + """ return self.app.ElementResources(**self.get_resources(action, set_defaults)) class Element: + """ + A basic component of a workflow. Elements are enactments of tasks. + + Parameters + ---------- + id_ : int + The ID of this element. + is_pending: bool + Whether this element is pending execution. + task: ~hpcflow.app.WorkflowTask + The task this is part of the enactment of. + index: int + The index of this element. + es_idx: int + The index within the task of the element set containing this element. + seq_idx: dict[str, int] + The sequence index IDs. + src_idx: dict[str, int] + The input source indices. + iteration_IDs: list[int] + The known IDs of iterations, + iterations: list[dict] + Data for creating iteration objects. + """ + _app_attr = "app" # TODO: use slots @@ -984,14 +1242,23 @@ def __repr__(self): @property def id_(self) -> int: + """ + The ID of this element. + """ return self._id @property def is_pending(self) -> bool: + """ + Whether this element is pending execution. + """ return self._is_pending @property def task(self) -> app.WorkflowTask: + """ + The task this is part of the enactment of. + """ return self._task @property @@ -1005,22 +1272,37 @@ def index(self) -> int: @property def element_set_idx(self) -> int: + """ + The index within the task of the element set containing this element. + """ return self._es_idx @property def element_set(self): + """ + The element set containing this element. + """ return self.task.template.element_sets[self.element_set_idx] @property def sequence_idx(self) -> Dict[str, int]: + """ + The sequence index IDs. + """ return self._seq_idx @property def input_source_idx(self) -> Dict[str, int]: + """ + The input source indices. + """ return self._src_idx @property def input_sources(self) -> Dict[str, app.InputSource]: + """ + The sources of the inputs to this element. + """ return { k: self.element_set.input_sources[k.split("inputs.")[1]][v] for k, v in self.input_source_idx.items() @@ -1028,15 +1310,24 @@ def input_sources(self) -> Dict[str, app.InputSource]: @property def workflow(self) -> app.Workflow: + """ + The workflow containing this element. + """ return self.task.workflow @property def iteration_IDs(self) -> List[int]: + """ + The IDs of the iterations of this element. + """ return self._iteration_IDs @property @TimeIt.decorator - def iterations(self) -> Dict[app.ElementAction]: + def iterations(self) -> List[app.ElementIteration]: + """ + The iterations of this element. + """ # TODO: fix this if self._iteration_objs is None: self._iteration_objs = [ @@ -1051,43 +1342,72 @@ def iterations(self) -> Dict[app.ElementAction]: @property def dir_name(self): + """ + The name of the directory for containing temporary files for this element. + """ return f"e_{self.index}" @property def latest_iteration(self): + """ + The most recent iteration of this element. + """ return self.iterations[-1] @property def inputs(self) -> app.ElementInputs: + """ + The inputs to this element (or its most recent iteration). + """ return self.latest_iteration.inputs @property def outputs(self) -> app.ElementOutputs: + """ + The outputs from this element (or its most recent iteration). + """ return self.latest_iteration.outputs @property def input_files(self) -> app.ElementInputFiles: + """ + The input files to this element (or its most recent iteration). + """ return self.latest_iteration.input_files @property def output_files(self) -> app.ElementOutputFiles: + """ + The output files from this element (or its most recent iteration). + """ return self.latest_iteration.output_files @property def schema_parameters(self) -> List[str]: + """ + The schema-defined parameters to this element (or its most recent iteration). + """ return self.latest_iteration.schema_parameters @property def actions(self) -> Dict[app.ElementAction]: + """ + The actions of this element (or its most recent iteration). + """ return self.latest_iteration.actions @property def action_runs(self) -> List[app.ElementActionRun]: - """Get a list of element action runs from the latest iteration, where only the - final run is taken for each element action.""" + """ + A list of element action runs from the latest iteration, where only the + final run is taken for each element action. + """ return self.latest_iteration.action_runs def init_loop_index(self, loop_name: str): + """ + Initialise the loop index if necessary. + """ pass def to_element_set_data(self): @@ -1117,6 +1437,9 @@ def to_element_set_data(self): return inputs, resources def get_sequence_value(self, sequence_path: str) -> Any: + """ + Get the value of a sequence that applies. + """ seq = self.element_set.get_sequence_from_path(sequence_path) if not seq: raise ValueError( @@ -1286,19 +1609,44 @@ def get_deps(element): @dataclass class ElementParameter: + """ + A parameter to an :py:class:`.Element`. + + Parameters + ---------- + task: ~hpcflow.app.WorkflowTask + The task that this is part of. + path: str + The path to this parameter. + parent: Element | ~hpcflow.app.ElementAction | ~hpcflow.app.ElementActionRun | ~hpcflow.app.Parameters + The entity that owns this parameter. + element: Element + The element that this is a parameter of. + """ + _app_attr = "app" + #: The task that this is part of. task: app.WorkflowTask + #: The path to this parameter. path: str + #: The entity that owns this parameter. parent: Union[Element, app.ElementAction, app.ElementActionRun, app.Parameters] + #: The element that this is a parameter of. element: Element @property def data_idx(self): + """ + The data indices associated with this parameter. + """ return self.parent.get_data_idx(path=self.path) @property def value(self) -> Any: + """ + The value of this parameter. + """ return self.parent.get(path=self.path) def __repr__(self) -> str: @@ -1312,27 +1660,49 @@ def __eq__(self, __o: object) -> bool: @property def data_idx_is_set(self): + """ + The associated data indices for which this is set. + """ return { k: self.task.workflow.is_parameter_set(v) for k, v in self.data_idx.items() } @property def is_set(self): + """ + Whether this parameter is set. + """ return all(self.data_idx_is_set.values()) def get_size(self, **store_kwargs): + """ + Get the size of the parameter. + """ raise NotImplementedError @dataclass class ElementFilter(JSONLike): + """ + A filter for iterations. + + Parameters + ---------- + rules: list[~hpcflow.app.Rule] + The filtering rules to use. + """ + _child_objects = (ChildObjectSpec(name="rules", is_multiple=True, class_name="Rule"),) + #: The filtering rules to use. rules: List[app.Rule] = field(default_factory=list) def filter( self, element_iters: List[app.ElementIteration] ) -> List[app.ElementIteration]: + """ + Apply the filter rules to select a subsequence of iterations. + """ out = [] for i in element_iters: if all(rule_j.test(i) for rule_j in self.rules): @@ -1342,8 +1712,24 @@ def filter( @dataclass class ElementGroup(JSONLike): + """ + A grouping rule for element iterations. + + Parameters + ---------- + name: + The name of the grouping rule. + where: + A filtering rule to select which iterations to use in the group. + group_by_distinct: + If specified, the name of the property to group iterations by. + """ + + #: The name of the grouping rule. name: str + #: A filtering rule to select which iterations to use in the group. where: Optional[ElementFilter] = None + #: If specified, the name of the property to group iterations by. group_by_distinct: Optional[app.ParameterPath] = None def __post_init__(self): @@ -1352,5 +1738,18 @@ def __post_init__(self): @dataclass class ElementRepeats: + """ + A repetition rule. + + Parameters + ---------- + number: + The number of times to repeat. + where: + A filtering rule for what to repeat. + """ + + #: The number of times to repeat. number: int + #: A filtering rule for what to repeat. where: Optional[ElementFilter] = None diff --git a/hpcflow/sdk/core/environment.py b/hpcflow/sdk/core/environment.py index 4cbad9869..aee6ac031 100644 --- a/hpcflow/sdk/core/environment.py +++ b/hpcflow/sdk/core/environment.py @@ -1,3 +1,7 @@ +""" +Model of an execution environment. +""" + from __future__ import annotations from dataclasses import dataclass @@ -14,8 +18,24 @@ @dataclass class NumCores(JSONLike): + """ + A range of cores supported by an executable instance. + + Parameters + ---------- + start: + The minimum number of cores supported. + stop: + The maximum number of cores supported. + step: int + The step in the number of cores supported. Defaults to 1. + """ + + #: The minimum number of cores supported. start: int + #: The maximum number of cores supported. stop: int + #: The step in the number of cores supported. Normally 1. step: int = None def __post_init__(self): @@ -41,8 +61,24 @@ def __eq__(self, other): @dataclass class ExecutableInstance(JSONLike): + """ + A particular instance of an executable that can support some mode of operation. + + Parameters + ---------- + parallel_mode: + What parallel mode is supported by this executable instance. + num_cores: NumCores | int | dict[str, int] + The number of cores supported by this executable instance. + command: + The actual command to use for this executable instance. + """ + + #: What parallel mode is supported by this executable instance. parallel_mode: str + #: The number of cores supported by this executable instance. num_cores: Any + #: The actual command to use for this executable instance. command: str def __post_init__(self): @@ -63,10 +99,24 @@ def __eq__(self, other): @classmethod def from_spec(cls, spec): + """ + Construct an instance from a specification dictionary. + """ return cls(**spec) class Executable(JSONLike): + """ + A program managed by the environment. + + Parameters + ---------- + label: + The abstract name of the program. + instances: list[ExecutableInstance] + The concrete instances of the application that may be present. + """ + _child_objects = ( ChildObjectSpec( name="instances", @@ -76,7 +126,9 @@ class Executable(JSONLike): ) def __init__(self, label: str, instances: List[app.ExecutableInstance]): + #: The abstract name of the program. self.label = check_valid_py_identifier(label) + #: The concrete instances of the application that may be present. self.instances = instances self._executables_list = None # assigned by parent @@ -101,9 +153,28 @@ def __eq__(self, other): @property def environment(self): + """ + The environment that the executable is going to run in. + """ return self._executables_list.environment def filter_instances(self, parallel_mode=None, num_cores=None): + """ + Select the instances of the executable that are compatible with the given + requirements. + + Parameters + ---------- + parallel_mode: str + If given, the parallel mode to require. + num_cores: int + If given, the number of cores desired. + + Returns + ------- + list[ExecutableInstance]: + The known executable instances that match the requirements. + """ out = [] for i in self.instances: if parallel_mode is None or i.parallel_mode == parallel_mode: @@ -113,6 +184,22 @@ def filter_instances(self, parallel_mode=None, num_cores=None): class Environment(JSONLike): + """ + An execution environment that contains a number of executables. + + Parameters + ---------- + name: str + The name of the environment. + setup: list[str] + Commands to run to enter the environment. + specifiers: dict[str, str] + Dictionary of attributes that may be used to supply addional key/value pairs to + look up an environment by. + executables: list[Executable] + List of abstract executables in the environment. + """ + _hash_value = None _validation_schema = "environments_spec_schema.yaml" _child_objects = ( @@ -126,9 +213,14 @@ class Environment(JSONLike): def __init__( self, name, setup=None, specifiers=None, executables=None, _hash_value=None ): + #: The name of the environment. self.name = name + #: Commands to run to enter the environment. self.setup = setup + #: Dictionary of attributes that may be used to supply addional key/value pairs + #: to look up an environment by. self.specifiers = specifiers or {} + #: List of abstract executables in the environment. self.executables = ( executables if isinstance(executables, ExecutablesList) diff --git a/hpcflow/sdk/core/errors.py b/hpcflow/sdk/core/errors.py index 593fcd334..5f38677f2 100644 --- a/hpcflow/sdk/core/errors.py +++ b/hpcflow/sdk/core/errors.py @@ -1,32 +1,59 @@ +""" +Errors from the workflow system. +""" + import os from typing import Iterable, List class InputValueDuplicateSequenceAddress(ValueError): - pass + """ + An InputValue has the same sequence address twice. + """ class TaskTemplateMultipleSchemaObjectives(ValueError): - pass + """ + A TaskTemplate has multiple objectives. + """ class TaskTemplateUnexpectedInput(ValueError): - pass + """ + A TaskTemplate was given unexpected input. + """ class TaskTemplateUnexpectedSequenceInput(ValueError): - pass + """ + A TaskTemplate was given an unexpected sequence. + """ class TaskTemplateMultipleInputValues(ValueError): - pass + """ + A TaskTemplate had multiple input values bound over each other. + """ class InvalidIdentifier(ValueError): - pass + """ + A bad identifier name was given. + """ class MissingInputs(Exception): + """ + Inputs were missing. + + Parameters + ---------- + message: str + The message of the exception. + missing_inputs: list[str] + The missing inputs. + """ + # TODO: add links to doc pages for common user-exceptions? def __init__(self, message, missing_inputs) -> None: @@ -35,6 +62,17 @@ def __init__(self, message, missing_inputs) -> None: class UnrequiredInputSources(ValueError): + """ + Input sources were provided that were not required. + + Parameters + ---------- + message: str + The message of the exception. + unrequired_sources: list + The input sources that were not required. + """ + def __init__(self, message, unrequired_sources) -> None: self.unrequired_sources = unrequired_sources for src in unrequired_sources: @@ -50,129 +88,199 @@ def __init__(self, message, unrequired_sources) -> None: class ExtraInputs(Exception): + """ + Extra inputs were provided. + + Parameters + ---------- + message: str + The message of the exception. + extra_inputs: list + The extra inputs. + """ + def __init__(self, message, extra_inputs) -> None: self.extra_inputs = extra_inputs super().__init__(message) class UnavailableInputSource(ValueError): - pass + """ + An input source was not available. + """ class InapplicableInputSourceElementIters(ValueError): - pass + """ + An input source element iteration was inapplicable.""" class NoCoincidentInputSources(ValueError): - pass + """ + Could not line up input sources to make an actual valid execution. + """ class TaskTemplateInvalidNesting(ValueError): - pass + """ + Invalid nesting in a task template. + """ class TaskSchemaSpecValidationError(Exception): - pass + """ + A task schema failed to validate. + """ class WorkflowSpecValidationError(Exception): - pass + """ + A workflow failed to validate. + """ class InputSourceValidationError(Exception): - pass + """ + An input source failed to validate. + """ class EnvironmentSpecValidationError(Exception): - pass + """ + An environment specification failed to validate. + """ class ParameterSpecValidationError(Exception): - pass + """ + A parameter specification failed to validate. + """ class FileSpecValidationError(Exception): - pass + """ + A file specification failed to validate. + """ class DuplicateExecutableError(ValueError): - pass + """ + The same executable was present twice in an executable environment. + """ class MissingCompatibleActionEnvironment(Exception): - pass + """ + Could not find a compatible action environment. + """ class MissingActionEnvironment(Exception): - pass + """ + Could not find an action environment. + """ class ActionEnvironmentMissingNameError(Exception): - pass + """ + An action environment was missing its name. + """ class FromSpecMissingObjectError(Exception): - pass + """ + Missing object when deserialising from specification. + """ class TaskSchemaMissingParameterError(Exception): - pass + """ + Parameter was missing from task schema. + """ class ToJSONLikeChildReferenceError(Exception): - pass + """ + Failed to generate or reference a child object when converting to JSON. + """ class InvalidInputSourceTaskReference(Exception): - pass + """ + Invalid input source in task reference. + """ class WorkflowNotFoundError(Exception): - pass + """ + Could not find the workflow. + """ class MalformedWorkflowError(Exception): - pass + """ + Workflow was a malformed document. + """ class ValuesAlreadyPersistentError(Exception): - pass + """ + Trying to make a value persistent that already is so. + """ class MalformedParameterPathError(ValueError): - pass + """ + The path to a parameter was ill-formed. + """ class MalformedNestingOrderPath(ValueError): - pass + """ + A nesting order path was ill-formed. + """ class UnknownResourceSpecItemError(ValueError): - pass + """ + A resource specification item was not found. + """ class WorkflowParameterMissingError(AttributeError): - pass + """ + A parameter to a workflow was missing. + """ class WorkflowBatchUpdateFailedError(Exception): - pass + """ + An update to a workflow failed. + """ class WorkflowLimitsError(ValueError): - pass + """ + Workflow hit limits. + """ class UnsetParameterDataError(Exception): - pass + """ + Tried to read from an unset parameter. + """ class LoopAlreadyExistsError(Exception): - pass + """ + A particular loop (or its name) already exists. + """ class LoopTaskSubsetError(ValueError): - pass + """ + Problem constructing a subset of a task for a loop. + """ class SchedulerVersionsFailure(RuntimeError): @@ -184,6 +292,10 @@ def __init__(self, message): class JobscriptSubmissionFailure(RuntimeError): + """ + A job script could not be submitted to the scheduler. + """ + def __init__( self, message, @@ -207,13 +319,19 @@ def __init__( class SubmissionFailure(RuntimeError): + """ + A job submission failed. + """ + def __init__(self, message) -> None: self.message = message super().__init__(message) class WorkflowSubmissionFailure(RuntimeError): - pass + """ + A workflow submission failed. + """ class ResourceValidationError(ValueError): @@ -272,31 +390,45 @@ def __init__(self, scheduler, supported=None, available=None) -> None: class UnknownSGEPEError(ResourceValidationError): - pass + """ + Miscellaneous error from SGE parallel environment. + """ class IncompatibleSGEPEError(ResourceValidationError): - pass + """ + The SGE parallel environment selected is incompatible. + """ class NoCompatibleSGEPEError(ResourceValidationError): - pass + """ + No SGE parallel environment is compatible with request. + """ class IncompatibleParallelModeError(ResourceValidationError): - pass + """ + The parallel mode is incompatible. + """ class UnknownSLURMPartitionError(ResourceValidationError): - pass + """ + The requested SLURM partition isn't known. + """ class IncompatibleSLURMPartitionError(ResourceValidationError): - pass + """ + The requested SLURM partition is incompatible. + """ class IncompatibleSLURMArgumentsError(ResourceValidationError): - pass + """ + The SLURM arguments are incompatible with each other. + """ class _MissingStoreItemError(ValueError): @@ -354,80 +486,132 @@ def __init__(self, id_lst: Iterable[int]) -> None: class NotSubmitMachineError(RuntimeError): - pass + """ + The requested machine can't be submitted to. + """ class RunNotAbortableError(ValueError): - pass + """ + Cannot abort the run. + """ class NoCLIFormatMethodError(AttributeError): - pass + """ + Some CLI class lacks a format method + """ class ContainerKeyError(KeyError): + """ + A key could not be mapped in a container. + + Parameters + ---------- + path: list[str] + The path whose resolution failed. + """ + def __init__(self, path: List[str]) -> None: self.path = path super().__init__() class MayNeedObjectError(Exception): + """ + An object is needed but not present. + + Parameters + ---------- + path: list[str] + The path whose resolution failed. + """ + def __init__(self, path): self.path = path super().__init__() class NoAvailableElementSetsError(Exception): - pass + """ + No element set is available. + """ class OutputFileParserNoOutputError(ValueError): - pass + """ + There was no output for the output file parser to parse. + """ class SubmissionEnvironmentError(ValueError): - """Raised when submitting a workflow on a machine without a compatible environment.""" + """ + Raised when submitting a workflow on a machine without a compatible environment. + """ class MissingEnvironmentExecutableError(SubmissionEnvironmentError): - pass + """ + The environment does not have the requested executable at all. + """ class MissingEnvironmentExecutableInstanceError(SubmissionEnvironmentError): - pass + """ + The environment does not have a suitable instance of the requested executable. + """ class MissingEnvironmentError(SubmissionEnvironmentError): - pass + """ + There is no environment with that name. + """ class UnsupportedScriptDataFormat(ValueError): - pass + """ + That format of script data is not supported. + """ class UnknownScriptDataParameter(ValueError): - pass + """ + Unknown parameter in script data. + """ class UnknownScriptDataKey(ValueError): - pass + """ + Unknown key in script data. + """ class MissingVariableSubstitutionError(KeyError): - pass + """ + No definition available of a variable being substituted. + """ class EnvironmentPresetUnknownEnvironmentError(ValueError): - pass + """ + An environment preset could not be resolved to an execution environment. + """ class UnknownEnvironmentPresetError(ValueError): - pass + """ + An execution environment was unknown. + """ class MultipleEnvironmentsError(ValueError): - pass + """ + Multiple applicable execution environments exist. + """ class MissingElementGroup(ValueError): - pass + """ + An element group should exist but doesn't. + """ diff --git a/hpcflow/sdk/core/json_like.py b/hpcflow/sdk/core/json_like.py index a26af84e9..1c2cd5604 100644 --- a/hpcflow/sdk/core/json_like.py +++ b/hpcflow/sdk/core/json_like.py @@ -1,3 +1,7 @@ +""" +Serialization and deserialization mechanism intended to map between a complex +graph of objects and either JSON or YAML. +""" from __future__ import annotations import copy @@ -10,7 +14,7 @@ from .validation import get_schema from .errors import ToJSONLikeChildReferenceError - +#: Primitive types supported by the serialization mechanism. PRIMITIVES = ( int, float, @@ -22,6 +26,10 @@ def to_json_like(obj, shared_data=None, parent_refs=None, path=None): + """ + Convert the object to a JSON-like basic value tree. + Such trees are trivial to serialize as JSON or YAML. + """ path = path or [] if len(path) > 50: @@ -81,32 +89,54 @@ def to_json_like(obj, shared_data=None, parent_refs=None, path=None): @dataclass class ChildObjectSpec: + """ + Used to describe what the child structure of an class is so that the generic + deserializer can build the structure. + """ + + #: The name of the attribute. name: str + #: The name of the class (or class of members of a list) used to deserialize the + #: attribute. class_name: Optional[str] = None + #: The class (or class of members of a list) used to deserialize the + #: attribute. class_obj: Optional[ Type ] = None # TODO: no need for class_obj/class_name if shared data? + #: The name of the key used in the JSON document, if different from the attribute + #: name. json_like_name: Optional[str] = None + #: If true, the attribute is really a list of instances, + #: or a dictionary if :attr:`dict_key_attr` is set. is_multiple: Optional[bool] = False + #: If set, the name of an attribute of the object to use as a dictionary key. + #: Requires that :attr:`is_multiple` be set as well. dict_key_attr: Optional[str] = None + #: If set, the name of an attribute of the object to use as a dictionary value. + #: If not set but :attr:`dict_key_attr` is set, the whole object is the value. + #: Requires that :attr:`dict_key_attr` be set as well. dict_val_attr: Optional[str] = None + #: If set, the attribute of the child object that contains a reference to its parent. parent_ref: Optional[ str ] = None # TODO: do parent refs make sense when from shared? Prob not. - is_single_attribute: Optional[ - bool - ] = False # if True, obj is not represented as a dict of attr name-values, but just a value. - is_enum: Optional[ - bool - ] = False # if true, we don't invoke to/from_json_like on the data/Enum - is_dict_values: Optional[ - bool - ] = False # if True, the child object is a dict, whose values are of the specified class. The dict structure will remain. - is_dict_values_ensure_list: Optional[ - bool - ] = False # if True, values that are not lists are cast to lists and multiple child objects are instantiated for each dict value - + #: If true, the object is not represented as a dict of attr name-values, but just a value. + is_single_attribute: Optional[bool] = False + #: If true, the object is an enum member and should use special serialization rules. + is_enum: Optional[bool] = False + #: If true, the child object is a dict, whose values are of the specified class. + #: The dict structure will remain. + is_dict_values: Optional[bool] = False + #: If true, values that are not lists are cast to lists and multiple child objects + #: are instantiated for each dict value. + is_dict_values_ensure_list: Optional[bool] = False + #: What key to look values up under in the shared data cache. + #: If unspecified, the shared data cache is ignored. shared_data_name: Optional[str] = None + #: What attribute provides the value of the key into the shared data cache. + #: If unspecified, a hash of the object dictionary is used. + #: Ignored if :py:attr:`~.shared_data_name` is unspecified. shared_data_primary_key: Optional[str] = None # shared_data_secondary_keys: Optional[Tuple[str]] = None # TODO: what's the point? @@ -155,6 +185,8 @@ def __post_init__(self): class BaseJSONLike: """ + An object that has a serialization as JSON or YAML. + Parameters ---------- _class_namespace : namespace @@ -200,6 +232,21 @@ def from_json_like( json_like: Union[Dict, List], shared_data: Optional[Dict[str, ObjectList]] = None, ): + """ + Make an instance of this class from JSON (or YAML) data. + + Parameters + ---------- + json_like: + The data to deserialise. + shared_data: + Shared context data. + + Returns + ------- + The deserialised object. + """ + def _from_json_like_item(child_obj_spec, json_like_i): if not ( child_obj_spec.class_name @@ -403,12 +450,20 @@ def _get_hash_from_json_like(json_like): return get_md5_hash(json_like) def to_dict(self): + """ + Serialize this object as a dictionary. + """ if hasattr(self, "__dict__"): return dict(self.__dict__) elif hasattr(self, "__slots__"): return {k: getattr(self, k) for k in self.__slots__} def to_json_like(self, dct=None, shared_data=None, exclude=None, path=None): + """ + Serialize this object as an object structure that can be trivially converted + to JSON. Note that YAML can also be produced from the result of this method; + it just requires a different final serialization step. + """ if dct is None: dct = {k: v for k, v in self.to_dict().items() if k not in (exclude or [])} @@ -475,6 +530,9 @@ def _class_namespace(cls): return getattr(cls, cls._app_attr) def to_dict(self): + """ + Serialize this object as a dictionary. + """ out = super().to_dict() # remove parent references: diff --git a/hpcflow/sdk/core/loop.py b/hpcflow/sdk/core/loop.py index e5fc0c502..ef5a7d3ac 100644 --- a/hpcflow/sdk/core/loop.py +++ b/hpcflow/sdk/core/loop.py @@ -1,3 +1,9 @@ +""" +A looping construct for a workflow. +There are multiple types of loop, +notably looping over a set of values or until a condition holds. +""" + from __future__ import annotations import copy @@ -26,11 +32,29 @@ # @dataclass # class Loop: # parameter: Parameter -# stopping_criteria: StoppingCriterion # TODO: should be a logical combination of these (maybe provide a superclass in valida to re-use some logic there?) +# stopping_criteria: StoppingCriterion +# # TODO: should be a logical combination of these (maybe provide a superclass in valida to re-use some logic there?) # maximum_iterations: int class Loop(JSONLike): + """ + A loop in a workflow template. + + Parameters + ---------- + tasks: list[int | ~hpcflow.app.WorkflowTask] + List of task insert IDs or workflow tasks. + num_iterations: + Number of iterations to perform. + name: str + Loop name. + non_iterable_parameters: list[str] + Specify input parameters that should not iterate. + termination: v~hpcflow.app.Rule + Stopping criterion, expressed as a rule. + """ + _app_attr = "app" _child_objects = (ChildObjectSpec(name="termination", class_name="Rule"),) @@ -42,21 +66,6 @@ def __init__( non_iterable_parameters: Optional[List[str]] = None, termination: Optional[app.Rule] = None, ) -> None: - """ - - Parameters - ---------- - name - Loop name, optional - tasks - List of task insert IDs or WorkflowTask objects - non_iterable_parameters - Specify input parameters that should not iterate. - termination - Stopping criterion, expressed as a rule. - - """ - _task_insert_IDs = [] for task in tasks: if isinstance(task, WorkflowTask): @@ -98,22 +107,37 @@ def task_insert_IDs(self) -> Tuple[int]: @property def name(self): + """ + The name of the loop, if one was provided. + """ return self._name @property def num_iterations(self): + """ + The number of loop iterations to do. + """ return self._num_iterations @property def non_iterable_parameters(self): + """ + Which parameters are not iterable. + """ return self._non_iterable_parameters @property def termination(self): + """ + A termination rule for the loop, if one is provided. + """ return self._termination @property def workflow_template(self): + """ + The workflow template that contains this loop. + """ return self._workflow_template @workflow_template.setter @@ -123,6 +147,9 @@ def workflow_template(self, template: app.WorkflowTemplate): @property def task_objects(self) -> Tuple[app.WorkflowTask]: + """ + The tasks in the loop. + """ if not self.workflow_template: raise RuntimeError( "Workflow template must be assigned to retrieve task objects of the loop." @@ -169,7 +196,25 @@ def __deepcopy__(self, memo): class WorkflowLoop: - """Class to represent a Loop that is bound to a Workflow.""" + """ + Class to represent a :py:class:`.Loop` that is bound to a + :py:class:`~hpcflow.app.Workflow`. + + Parameters + ---------- + index: int + The index of this loop in the workflow. + workflow: ~hpcflow.app.Workflow + The workflow containing this loop. + template: Loop + The loop that this was generated from. + num_added_iterations: + Description of what iterations have been added. + iterable_parameters: + Description of what parameters are being iterated over. + parents: list[str] + The paths to the parent entities of this loop. + """ _app_attr = "app" @@ -229,7 +274,9 @@ def __repr__(self) -> str: @property def num_added_iterations(self): - + """ + The number of added iterations. + """ if self._pending_num_added_iterations: return self._pending_num_added_iterations else: @@ -281,53 +328,82 @@ def _accept_pending_parents(self): @property def index(self): + """ + The index of this loop within its workflow. + """ return self._index @property def task_insert_IDs(self): + """ + The insertion IDs of the tasks inside this loop. + """ return self.template.task_insert_IDs @property def task_objects(self): + """ + The tasks in this loop. + """ return self.template.task_objects @property def task_indices(self) -> Tuple[int]: - """Get the list of task indices that define the extent of the loop.""" + """ + The list of task indices that define the extent of the loop. + """ return tuple(i.index for i in self.task_objects) @property def workflow(self): + """ + The workflow containing this loop. + """ return self._workflow @property def template(self): + """ + The loop template for this loop. + """ return self._template @property def parents(self) -> List[str]: + """ + The parents of this loop. + """ return self._parents + self._pending_parents @property def name(self): + """ + The name of this loop, if one is defined. + """ return self.template.name @property def iterable_parameters(self): + """ + The parameters that are being iterated over. + """ return self._iterable_parameters @property def num_iterations(self): + """ + The number of iterations. + """ return self.template.num_iterations @property def downstream_tasks(self) -> List[app.WorkflowLoop]: - """Return tasks that are not part of the loop, and downstream from this loop.""" + """Tasks that are not part of the loop, and downstream from this loop.""" return self.workflow.tasks[self.task_objects[-1].index + 1 :] @property def upstream_tasks(self) -> List[app.WorkflowLoop]: - """Return tasks that are not part of the loop, and upstream from this loop.""" + """Tasks that are not part of the loop, and upstream from this loop.""" return self.workflow.tasks[: self.task_objects[0].index] @staticmethod @@ -367,6 +443,20 @@ def new_empty_loop( template: app.Loop, iter_loop_idx: List[Dict], ) -> Tuple[app.WorkflowLoop, List[Dict[str, int]]]: + """ + Make a new empty loop. + + Parameters + ---------- + index: int + The index of the loop to create. + workflow: ~hpcflow.app.Workflow + The workflow that will contain the loop. + template: Loop + The template for the loop. + iter_loop_idx: list[dict] + Iteration information from parent loops. + """ parent_loops = cls._get_parent_loops(index, workflow, template) parent_names = [i.name for i in parent_loops] num_added_iters = {} @@ -436,6 +526,17 @@ def get_child_loops(self) -> List[app.WorkflowLoop]: @TimeIt.decorator def add_iteration(self, parent_loop_indices=None, cache: Optional[LoopCache] = None): + """ + Add an iteration to this loop. + + Parameters + ---------- + parent_loop_indices: + Where have any parent loops got up to? + cache: + A cache used to make adding the iteration more efficient. + One will be created if it is not supplied. + """ if not cache: cache = LoopCache.build(self.workflow) parent_loops = self.get_parent_loops() diff --git a/hpcflow/sdk/core/loop_cache.py b/hpcflow/sdk/core/loop_cache.py index 569502640..7620aa9c2 100644 --- a/hpcflow/sdk/core/loop_cache.py +++ b/hpcflow/sdk/core/loop_cache.py @@ -1,3 +1,7 @@ +""" +Cache of loop statuses. +""" + from dataclasses import dataclass from collections import defaultdict from typing import Dict, List, Optional, Tuple @@ -10,40 +14,55 @@ @dataclass class LoopCache: - """Class to store a cache for use in `Workflow.add_empty_loop` and - `WorkflowLoop.add_iterations`. + """Class to store a cache for use in :py:meth:`.Workflow.add_empty_loop` and + :py:meth:`.WorkflowLoop.add_iterations`. Use :py:meth:`build` to get a new instance. - Attributes + Parameters ---------- - element_dependents + element_dependents: Keys are element IDs, values are dicts whose keys are element IDs that depend on the key element ID (via `Element.get_dependent_elements_recursively`), and whose values are dicts with keys: `group_names`, which is a tuple of the string group names associated with the dependent element's element set. - elements + elements: Keys are element IDs, values are dicts with keys: `input_statuses`, `input_sources`, and `task_insert_ID`. - zeroth_iters + zeroth_iters: Keys are element IDs, values are data associated with the zeroth iteration of that element, namely a tuple of iteration ID and `ElementIteration.data_idx`. - data_idx + data_idx: Keys are element IDs, values are data associated with all iterations of that element, namely a dict whose keys are the iteration loop index as a tuple, and whose values are data indices via `ElementIteration.get_data_idx()`. - iterations + iterations: Keys are iteration IDs, values are tuples of element ID and iteration index within that element. - task_iterations + task_iterations: Keys are task insert IDs, values are list of all iteration IDs associated with that task. """ + #: Keys are element IDs, values are dicts whose keys are element IDs that depend on + #: the key element ID (via `Element.get_dependent_elements_recursively`), and whose + #: values are dicts with keys: `group_names`, which is a tuple of the string group + #: names associated with the dependent element's element set. element_dependents: Dict[int, Dict] + #: Keys are element IDs, values are dicts with keys: `input_statuses`, + #: `input_sources`, and `task_insert_ID`. elements: Dict[int, Dict] + #: Keys are element IDs, values are data associated with the zeroth iteration of that + #: element, namely a tuple of iteration ID and `ElementIteration.data_idx`. zeroth_iters: Dict[int, Tuple] + #: Keys are element IDs, values are data associated with all iterations of that + #: element, namely a dict whose keys are the iteration loop index as a tuple, and + #: whose values are data indices via `ElementIteration.get_data_idx()`. data_idx: Dict[int, Dict] + #: Keys are iteration IDs, values are tuples of element ID and iteration index within + #: that element. iterations: Dict[int, Tuple] + #: Keys are task insert IDs, values are list of all iteration IDs associated with + #: that task. task_iterations: Dict[int, List[int]] @TimeIt.decorator @@ -53,6 +72,9 @@ def get_iter_IDs(self, loop: "app.Loop") -> List[int]: @TimeIt.decorator def get_iter_loop_indices(self, iter_IDs: List[int]) -> List[Dict[str, int]]: + """ + Retrieve the mapping from element to loop index for each given iteration. + """ iter_loop_idx = [] for i in iter_IDs: elem_id, idx = self.iterations[i] @@ -61,6 +83,9 @@ def get_iter_loop_indices(self, iter_IDs: List[int]) -> List[Dict[str, int]]: @TimeIt.decorator def update_loop_indices(self, new_loop_name: str, iter_IDs: List[int]): + """ + Set the loop indices for a named loop to the given list of iteration IDs. + """ elem_ids = {v[0] for k, v in self.iterations.items() if k in iter_IDs} for i in elem_ids: new_item = {} diff --git a/hpcflow/sdk/core/object_list.py b/hpcflow/sdk/core/object_list.py index aeda25234..49e505534 100644 --- a/hpcflow/sdk/core/object_list.py +++ b/hpcflow/sdk/core/object_list.py @@ -1,3 +1,7 @@ +""" +General model of a searchable serializable list. +""" + import copy from types import SimpleNamespace @@ -5,29 +9,25 @@ class ObjectListMultipleMatchError(ValueError): - pass + """ + Thrown when an object looked up by unique attribute ends up with multiple objects + being matched. + """ class ObjectList(JSONLike): """A list-like class that provides item access via a `get` method according to attributes or dict-keys. + Parameters + ---------- + objects : sequence + List + descriptor : str + Descriptive name for objects in the list. """ def __init__(self, objects, descriptor=None): - """ - - Parameters - ---------- - objects : sequence - List - access_attribute : str - Name of the attribute through which objects are accessed. The values must be - hashable. - descriptor : str - - """ - self._objects = list(objects) self._descriptor = descriptor or "object" self._object_is_dict = False @@ -145,6 +145,22 @@ def get(self, **kwargs): return self._validate_get(self.get_all(**kwargs), kwargs) def add_object(self, obj, index=-1, skip_duplicates=False): + """ + Add an object to this object list. + + Parameters + ---------- + obj: + The object to add. + index: + Where to add it. Omit to append. + skip_duplicates: + If true, don't add the object if it is already in the list. + + Returns + ------- + The index of the added object, or ``None`` if the object was not added. + """ if skip_duplicates and obj in self: return @@ -160,8 +176,19 @@ def add_object(self, obj, index=-1, skip_duplicates=False): class DotAccessObjectList(ObjectList): - """Provide dot-notation access via an access attribute for the case where the access - attribute uniquely identifies a single object.""" + """ + Provide dot-notation access via an access attribute for the case where the access + attribute uniquely identifies a single object. + + Parameters + ---------- + _objects: + The objects in the list. + access_attribute: + The main attribute for selection and filtering. A unique property. + descriptor: str + Descriptive name for the objects in the list. + """ # access attributes must not be named after any "public" methods, to avoid confusion! _pub_methods = ("get", "get_all", "add_object", "add_objects") @@ -233,6 +260,9 @@ def __dir__(self): ] def get(self, access_attribute_value=None, **kwargs): + """ + Get an object from this list that matches the given criteria. + """ vld_get_kwargs = kwargs if access_attribute_value: vld_get_kwargs = {self._access_attribute: access_attribute_value, **kwargs} @@ -243,6 +273,9 @@ def get(self, access_attribute_value=None, **kwargs): ) def get_all(self, access_attribute_value=None, **kwargs): + """ + Get all objects in this list that match the given criteria. + """ # use the index to narrow down the search first: if access_attribute_value: try: @@ -260,11 +293,17 @@ def get_all(self, access_attribute_value=None, **kwargs): return self._get_all_from_objs(all_objs, **kwargs) def add_object(self, obj, index=-1, skip_duplicates=False): + """ + Add an object to this list. + """ index = super().add_object(obj, index, skip_duplicates) self._update_index() return index def add_objects(self, objs, index=-1, skip_duplicates=False): + """ + Add multiple objects to the list. + """ for obj in objs: index = self.add_object(obj, index, skip_duplicates) if index is not None: @@ -273,6 +312,10 @@ def add_objects(self, objs, index=-1, skip_duplicates=False): class AppDataList(DotAccessObjectList): + """ + An application-aware object list. + """ + _app_attr = "_app" def to_dict(self): @@ -281,11 +324,20 @@ def to_dict(self): @classmethod def from_json_like(cls, json_like, shared_data=None, is_hashed: bool = False): """ + Make an instance of this class from JSON (or YAML) data. + Parameters ---------- - is_hashed + json_like: + The data to deserialise. + shared_data: + Shared context data. + is_hashed: If True, accept a dict whose keys are hashes of the dict values. + Returns + ------- + The deserialised object. """ if is_hashed: json_like = [ @@ -303,7 +355,13 @@ def _remove_object(self, index): class TaskList(AppDataList): """A list-like container for a task-like list with dot-notation access by task - unique-name.""" + unique-name. + + Parameters + ---------- + _objects: list[~hpcflow.app.Task] + The tasks in this list. + """ _child_objects = ( ChildObjectSpec( @@ -320,7 +378,13 @@ def __init__(self, _objects): class TaskTemplateList(AppDataList): """A list-like container for a task-like list with dot-notation access by task - unique-name.""" + unique-name. + + Parameters + ---------- + _objects: list[~hpcflow.app.TaskTemplate] + The task templates in this list. + """ _child_objects = ( ChildObjectSpec( @@ -337,7 +401,13 @@ def __init__(self, _objects): class TaskSchemasList(AppDataList): """A list-like container for a task schema list with dot-notation access by task - schema unique-name.""" + schema unique-name. + + Parameters + ---------- + _objects: list[~hpcflow.app.TaskSchema] + The task schemas in this list. + """ _child_objects = ( ChildObjectSpec( @@ -354,7 +424,13 @@ def __init__(self, _objects): class GroupList(AppDataList): """A list-like container for the task schema group list with dot-notation access by - group name.""" + group name. + + Parameters + ---------- + _objects: list[Group] + The groups in this list. + """ _child_objects = ( ChildObjectSpec( @@ -370,7 +446,14 @@ def __init__(self, _objects): class EnvironmentsList(AppDataList): - """A list-like container for environments with dot-notation access by name.""" + """ + A list-like container for environments with dot-notation access by name. + + Parameters + ---------- + _objects: list[~hpcflow.app.Environment] + The environments in this list. + """ _child_objects = ( ChildObjectSpec( @@ -393,9 +476,17 @@ def _get_obj_attr(self, obj, attr): class ExecutablesList(AppDataList): - """A list-like container for environment executables with dot-notation access by - executable label.""" + """ + A list-like container for environment executables with dot-notation access by + executable label. + + Parameters + ---------- + _objects: list[~hpcflow.app.Executable] + The executables in this list. + """ + #: The environment containing these executables. environment = None _child_objects = ( ChildObjectSpec( @@ -418,7 +509,14 @@ def __deepcopy__(self, memo): class ParametersList(AppDataList): - """A list-like container for parameters with dot-notation access by parameter type.""" + """ + A list-like container for parameters with dot-notation access by parameter type. + + Parameters + ---------- + _objects: list[~hpcflow.app.Parameter] + The parameters in this list. + """ _child_objects = ( ChildObjectSpec( @@ -455,7 +553,14 @@ def get_all(self, access_attribute_value=None, **kwargs): class CommandFilesList(AppDataList): - """A list-like container for command files with dot-notation access by label.""" + """ + A list-like container for command files with dot-notation access by label. + + Parameters + ---------- + _objects: list[~hpcflow.app.FileSpec] + The files in this list. + """ _child_objects = ( ChildObjectSpec( @@ -471,6 +576,15 @@ def __init__(self, _objects): class WorkflowTaskList(DotAccessObjectList): + """ + A list-like container for workflow tasks with dot-notation access by unique name. + + Parameters + ---------- + _objects: list[~hpcflow.app.WorkflowTask] + The tasks in this list. + """ + def __init__(self, _objects): super().__init__(_objects, access_attribute="unique_name", descriptor="task") @@ -491,6 +605,15 @@ def _remove_object(self, index): class WorkflowLoopList(DotAccessObjectList): + """ + A list-like container for workflow loops with dot-notation access by name. + + Parameters + ---------- + _objects: list[~hpcflow.app.WorkflowLoop] + The loops in this list. + """ + def __init__(self, _objects): super().__init__(_objects, access_attribute="name", descriptor="loop") @@ -499,6 +622,16 @@ def _remove_object(self, index): class ResourceList(ObjectList): + """ + A list-like container for resources. + Each contained resource must have a unique scope. + + Parameters + ---------- + _objects: list[~hpcflow.app.ResourceSpec] + The resource descriptions in this list. + """ + _app_attr = "_app" _child_objects = ( ChildObjectSpec( @@ -534,10 +667,16 @@ def __deepcopy__(self, memo): @property def element_set(self): + """ + The parent element set, if a child of an element set. + """ return self._element_set @property def workflow_template(self): + """ + The parent workflow template, if a child of a workflow template. + """ return self._workflow_template def to_json_like(self, dct=None, shared_data=None, exclude=None, path=None): @@ -581,6 +720,9 @@ def _ensure_non_persistent(resource_spec): return resources def get_scopes(self): + """ + Get the scopes of the contained resources. + """ return tuple(i.scope for i in self._objects) def merge_other(self, other): @@ -603,6 +745,10 @@ def merge_other(self, other): def index(obj_lst, obj): + """ + Get the index of the object in the list. + The item is checked for by object identity, not equality. + """ for idx, i in enumerate(obj_lst._objects): if obj is i: return idx diff --git a/hpcflow/sdk/core/parallel.py b/hpcflow/sdk/core/parallel.py index ab9eccba9..e8343baf0 100644 --- a/hpcflow/sdk/core/parallel.py +++ b/hpcflow/sdk/core/parallel.py @@ -1,7 +1,21 @@ +""" +Parallel modes. +""" + import enum class ParallelMode(enum.Enum): + """ + Potential parallel modes. + + Note: this is not yet implemented in any meaningful way! + + """ + + #: Use distributed-memory parallelism (e.g. MPI). DISTRIBUTED = 0 + #: Use shared-memory parallelism (e.g. OpenMP). SHARED = 1 + #: Use both distributed- and shared-memory parallelism. HYBRID = 2 diff --git a/hpcflow/sdk/core/parameters.py b/hpcflow/sdk/core/parameters.py index 5c2490bb4..9ea6e8996 100644 --- a/hpcflow/sdk/core/parameters.py +++ b/hpcflow/sdk/core/parameters.py @@ -1,3 +1,7 @@ +""" +Parameters represent information passed around within a workflow. +""" + from __future__ import annotations import copy from dataclasses import dataclass, field @@ -49,40 +53,74 @@ def string_processor(str_in): class ParameterValue: + """ + The value handler for a parameter. + + Intended to be subclassed. + """ + _typ = None _sub_parameters = {} def to_dict(self): + """ + Serialise this parameter value as a dictionary. + """ if hasattr(self, "__dict__"): return dict(self.__dict__) elif hasattr(self, "__slots__"): return {k: getattr(self, k) for k in self.__slots__} def prepare_JSON_dump(self) -> Dict: + """ + Prepare this parameter value for serialisation as JSON. + """ raise NotImplementedError def dump_to_HDF5_group(self, group): + """ + Write this parameter value to an HDF5 group. + """ raise NotImplementedError @classmethod def save_from_HDF5_group(cls, group, param_id: int, workflow): + """ + Extract a parameter value from an HDF5 group. + """ raise NotImplementedError @classmethod def save_from_JSON(cls, data, param_id: int, workflow): + """ + Extract a parameter value from JSON data. + """ raise NotImplementedError class ParameterPropagationMode(enum.Enum): + """ + How a parameter is propagated. + """ + + #: Parameter is propagated implicitly. IMPLICIT = 0 + #: Parameter is propagated explicitly. EXPLICIT = 1 + #: Parameter is never propagated. NEVER = 2 @dataclass class ParameterPath(JSONLike): + """ + Path to a parameter. + """ + # TODO: unused? + #: The path to the parameter. path: Sequence[Union[str, int, float]] + #: The task in which to look up the parameter. task: Optional[ Union[app.TaskTemplate, app.TaskSchema] ] = None # default is "current" task @@ -90,6 +128,28 @@ class ParameterPath(JSONLike): @dataclass class Parameter(JSONLike): + """ + A general parameter to a workflow task. + + Parameters + ---------- + typ: + Type code. + Used to look up the :py:class:`ParameterValue` for this parameter, + if any. + is_file: + Whether this parameter represents a file. + sub_parameters: list[SubParameter] + Any parameters packed within this one. + _value_class: type[ParameterValue] + Class that provides the implementation of this parameter's values. + Not normally directly user-managed. + _hash_value: + Hash of this class. Not normally user-managed. + _validation: + Validation schema. + """ + _validation_schema = "parameters_spec_schema.yaml" _child_objects = ( ChildObjectSpec( @@ -102,8 +162,12 @@ class Parameter(JSONLike): ), ) + #: Type code. Used to look up the :py:class:`ParameterValue` for this parameter, + #: if any. typ: str + #: Whether this parameter represents a file. is_file: bool = False + #: Any parameters packed within this one. sub_parameters: List[app.SubParameter] = field(default_factory=lambda: []) _value_class: Any = None _hash_value: Optional[str] = field(default=None, repr=False) @@ -158,17 +222,35 @@ def to_dict(self): @property def url_slug(self) -> str: + """ + Representation of this parameter as part of a URL. + """ return self.typ.lower().replace("_", "-") @dataclass class SubParameter: + """ + A parameter that is a component of another parameter. + """ + + #: How to find this within the containing paraneter. address: Address + #: The containing main parameter. parameter: app.Parameter @dataclass class SchemaParameter(JSONLike): + """ + A parameter bound in a schema. + + Parameters + ---------- + parameter: Parameter + The parameter. + """ + _app_attr = "app" _child_objects = ( @@ -189,14 +271,26 @@ def _validate(self): @property def name(self): + """ + The name of the parameter. + """ return self.parameter.name @property def typ(self): + """ + The type code of the parameter. + """ return self.parameter.typ class NullDefault(enum.Enum): + """ + Sentinel value used to distinguish an explicit null. + """ + + #: Special sentinel. + #: Used in situations where otherwise a JSON object or array would be. NULL = 0 @@ -206,13 +300,13 @@ class SchemaInput(SchemaParameter): Parameters ---------- - parameter + parameter: The parameter (i.e. type) of this schema input. - multiple + multiple: If True, expect one or more of these parameters defined in the workflow, distinguished by a string label in square brackets. For example `p1[0]` for a parameter `p1`. - labels + labels: Dict whose keys represent the string labels that distinguish multiple parameters if `multiple` is `True`. Use the key "*" to mean all labels not matching other label keys. If `multiple` is `False`, this will default to a @@ -220,10 +314,10 @@ class SchemaInput(SchemaParameter): `True`, this will default to a single-item dict with the catch-all key: `{{"*": {{}}}}`. On initialisation, remaining keyword-arguments are treated as default values for the dict values of `labels`. - default_value + default_value: The default value for this input parameter. This is itself a default value that will be applied to all `labels` values if a "default_value" key does not exist. - propagation_mode + propagation_mode: Determines how this input should propagate through the workflow. This is a default value that will be applied to all `labels` values if a "propagation_mode" key does not exist. By default, the input is allowed to be used in downstream tasks simply @@ -232,7 +326,7 @@ class SchemaInput(SchemaParameter): the downstream task `input_sources` for it to be used, and "never", meaning that the parameter must not be used in downstream tasks and will be inaccessible to those tasks. - group + group: Determines the name of the element group from which this input should be sourced. This is a default value that will be applied to all `labels` if a "group" key does not exist. @@ -270,8 +364,12 @@ def __init__( except ValueError: parameter = self.app.Parameter(parameter) + #: The parameter (i.e. type) of this schema input. self.parameter = parameter + #: Whether to expect more than of these parameters defined in the workflow. self.multiple = multiple + #: Dict whose keys represent the string labels that distinguish multiple + #: parameters if `multiple` is `True`. self.labels = labels if self.labels is None: @@ -395,6 +493,9 @@ def __deepcopy__(self, memo): @property def default_value(self): + """ + The default value of the input. + """ if not self.multiple: if "default_value" in self.single_labelled_data: return self.single_labelled_data["default_value"] @@ -403,28 +504,46 @@ def default_value(self): @property def task_schema(self): + """ + The schema containing this input. + """ return self._task_schema @property def all_labelled_types(self): + """ + The types of the input labels. + """ return list(f"{self.typ}{f'[{i}]' if i else ''}" for i in self.labels) @property def single_label(self): + """ + The label of this input, assuming it is not mulitple. + """ if not self.multiple: return next(iter(self.labels)) @property def single_labelled_type(self): + """ + The type code of this input, assuming it is not mulitple. + """ if not self.multiple: return next(iter(self.labelled_info()))["labelled_type"] @property def single_labelled_data(self): + """ + The value of this input, assuming it is not mulitple. + """ if not self.multiple: return self.labels[self.single_label] def labelled_info(self): + """ + Get descriptors for all the labels associated with this input. + """ for k, v in self.labels.items(): label = f"[{k}]" if k else "" dct = { @@ -458,6 +577,9 @@ def _validate(self): @property def input_or_output(self): + """ + Whether this is an input or output. Always ``input``. + """ return "input" @@ -465,11 +587,16 @@ def input_or_output(self): class SchemaOutput(SchemaParameter): """A Parameter as outputted from particular task.""" + #: The basic parameter this supplies. parameter: Parameter + #: How this output propagates. propagation_mode: ParameterPropagationMode = ParameterPropagationMode.IMPLICIT @property def input_or_output(self): + """ + Whether this is an input or output. Always ``output``. + """ return "output" def __repr__(self) -> str: @@ -483,6 +610,11 @@ def __repr__(self) -> str: @dataclass class BuiltinSchemaParameter: + """ + A parameter of a built-in schema. + """ + + # TODO: Is this used anywhere? # builtin inputs (resources,parameter_perturbations,method,implementation # builtin outputs (time, memory use, node/hostname etc) # - builtin parameters do not propagate to other tasks (since all tasks define the same @@ -493,6 +625,23 @@ class BuiltinSchemaParameter: class ValueSequence(JSONLike): + """ + A sequence of values. + + Parameters + ---------- + path: + The path to this sequence. + values: + The values in this sequence. + nesting_order: int + A nesting order for this sequence. Can be used to compose sequences together. + label: str + A label for this sequence. + value_class_method: str + Name of a method used to generate sequence values. Not normally used directly. + """ + def __init__( self, path: str, @@ -504,9 +653,13 @@ def __init__( label = str(label) if label is not None else "" path, label = self._validate_parameter_path(path, label) + #: The path to this sequence. self.path = path + #: The label of this sequence. self.label = label + #: The nesting order for this sequence. self.nesting_order = nesting_order + #: Name of a method used to generate sequence values. self.value_class_method = value_class_method if values is not None: @@ -603,30 +756,48 @@ def from_json_like(cls, json_like, shared_data=None): @property def parameter(self): + """ + The parameter this sequence supplies. + """ return self._parameter @property def path_split(self): + """ + The components of ths path. + """ if self._path_split is None: self._path_split = self.path.split(".") return self._path_split @property def path_type(self): + """ + The type of path this is. + """ return self.path_split[0] @property def input_type(self): + """ + The type of input sequence this is, if it is one. + """ if self.path_type == "inputs": return self.path_split[1].replace(self._label_fmt, "") @property def input_path(self): + """ + The path of the input sequence this is, if it is one. + """ if self.path_type == "inputs": return ".".join(self.path_split[2:]) @property def resource_scope(self): + """ + The scope of the resources this is, if it is one. + """ if self.path_type == "resources": return self.path_split[1] @@ -641,6 +812,9 @@ def _label_fmt(self): @property def labelled_type(self): + """ + The labelled type of input sequence this is, if it is one. + """ if self.input_type: return f"{self.input_type}{self._label_fmt}" @@ -750,12 +924,17 @@ def to_dict(self): @property def normalised_path(self): + """ + The path to this sequence. + """ return self.path @property def normalised_inputs_path(self): - """Return the normalised path without the "inputs" prefix, if the sequence is an - inputs sequence, else return None.""" + """ + The normalised path without the "inputs" prefix, if the sequence is an + inputs sequence, else return None. + """ if self.input_type: if self.input_path: @@ -802,6 +981,9 @@ def make_persistent( @property def workflow(self): + """ + The workflow containing this sequence. + """ if self._workflow: return self._workflow elif self._element_set: @@ -809,6 +991,9 @@ def workflow(self): @property def values(self): + """ + The values in this sequence. + """ if self._values_group_idx is not None: vals = [] for idx, pg_idx_i in enumerate(self._values_group_idx): @@ -885,6 +1070,9 @@ def from_linear_space( label=None, **kwargs, ): + """ + Build a sequence from a NumPy linear space. + """ # TODO: save persistently as an array? args = {"start": start, "stop": stop, "num": num, **kwargs} values = cls._values_from_linear_space(**args) @@ -905,6 +1093,9 @@ def from_geometric_space( label=None, **kwargs, ): + """ + Build a sequence from a NumPy geometric space. + """ args = {"start": start, "stop": stop, "num": num, "endpoint": endpoint, **kwargs} values = cls._values_from_geometric_space(**args) obj = cls(values=values, path=path, nesting_order=nesting_order, label=label) @@ -925,6 +1116,9 @@ def from_log_space( label=None, **kwargs, ): + """ + Build a sequence from a NumPy logarithmic space. + """ args = { "start": start, "stop": stop, @@ -950,6 +1144,9 @@ def from_range( label=None, **kwargs, ): + """ + Build a sequence from a range. + """ # TODO: save persistently as an array? args = {"start": start, "stop": stop, "step": step, **kwargs} if isinstance(step, int): @@ -982,6 +1179,9 @@ def from_file( label=None, **kwargs, ): + """ + Build a sequence from a simple file. + """ args = {"file_path": file_path, **kwargs} values = cls._values_from_file(**args) obj = cls( @@ -1009,6 +1209,8 @@ def from_rectangle( **kwargs, ): """ + Build a sequence to cover a rectangle. + Parameters ---------- coord: @@ -1044,6 +1246,9 @@ def from_random_uniform( label=None, **kwargs, ): + """ + Build a sequence from a uniform random number generator. + """ args = {"low": low, "high": high, "num": num, "seed": seed, **kwargs} values = cls._values_from_random_uniform(**args) obj = cls(values=values, path=path, nesting_order=nesting_order, label=label) @@ -1111,6 +1316,9 @@ def make_persistent( @property def workflow(self): + """ + The workflow containing this input value. + """ if self._workflow: return self._workflow elif self._element_set: @@ -1120,6 +1328,9 @@ def workflow(self): @property def value(self): + """ + The value itself. + """ if self._value_group_idx is not None: val = self.workflow.get_parameter_data(self._value_group_idx) if self._value_is_obj and self.parameter._value_class: @@ -1132,31 +1343,44 @@ def value(self): @dataclass class ValuePerturbation(AbstractInputValue): + """ + A perturbation applied to a value. + """ + + #: The name of this perturbation. name: str + #: The path to the value(s) to perturb. path: Optional[Sequence[Union[str, int, float]]] = None + #: The multiplicative factor to apply. multiplicative_factor: Optional[Numeric] = 1 + #: The additive factor to apply. additive_factor: Optional[Numeric] = 0 @classmethod def from_spec(cls, spec): + """ + Construct an instance from a specification dictionary. + """ return cls(**spec) class InputValue(AbstractInputValue): """ + An input value to a task. + Parameters ---------- - parameter - Parameter whose value is to be specified - label + parameter: Parameter | SchemaInput | str + Parameter whose value is to be specified. + label: str Optional identifier to be used where the associated `SchemaInput` accepts multiple parameters of the specified type. This will be cast to a string. - value + value: Any The input parameter value. - value_class_method + value_class_method: How to obtain the real value. A class method that can be invoked with the `value` attribute as keyword arguments. - path + path: str Dot-delimited path within the parameter's nested data structure for which `value` should be set. @@ -1188,9 +1412,16 @@ def __init__( elif isinstance(parameter, SchemaInput): parameter = parameter.parameter + #: Parameter whose value is to be specified. self.parameter = parameter + #: Identifier to be used where the associated `SchemaInput` accepts multiple + #: parameters of the specified type. self.label = str(label) if label is not None else "" + #: Dot-delimited path within the parameter's nested data structure for which + #: `value` should be set. self.path = (path.strip(".") if path else None) or None + #: A class method that can be invoked with the `value` attribute as keyword + #: arguments. self.value_class_method = value_class_method self._value = _process_demo_data_strings(self.app, value) @@ -1293,15 +1524,24 @@ def _json_like_constructor(cls, json_like): @property def labelled_type(self): + """ + The labelled type of this input value. + """ label = f"[{self.label}]" if self.label else "" return f"{self.parameter.typ}{label}" @property def normalised_inputs_path(self): + """ + The normalised input path without the ``inputs.`` prefix. + """ return f"{self.labelled_type}{f'.{self.path}' if self.path else ''}" @property def normalised_path(self): + """ + The full normalised input path. + """ return f"inputs.{self.normalised_inputs_path}" def make_persistent(self, workflow: Any, source: Dict) -> Tuple[str, List[int], bool]: @@ -1347,8 +1587,55 @@ class ResourceSpec(JSONLike): `os_name` is used for retrieving a default shell name and for retrieving the correct `Shell` class; when using WSL, it should still be `nt` (i.e. Windows). + Parameters + ---------- + scope: + Which scope does this apply to. + scratch: str + Which scratch space to use. + parallel_mode: ParallelMode + Which parallel mode to use. + num_cores: int + How many cores to request. + num_cores_per_node: int + How many cores per compute node to request. + num_threads: int + How many threads to request. + num_nodes: int + How many compute nodes to request. + scheduler: str + Which scheduler to use. + shell: str + Which system shell to use. + use_job_array: bool + Whether to use array jobs. + max_array_items: int + If using array jobs, up to how many items should be in the job array. + time_limit: str + How long to run for. + scheduler_args: dict[str, Any] + Additional arguments to pass to the scheduler. + shell_args: dict[str, Any] + Additional arguments to pass to the shell. + os_name: str + Which OS to use. + environments: dict + Which execution environments to use. + SGE_parallel_env: str + Which SGE parallel environment to request. + SLURM_partition: str + Which SLURM partition to request. + SLURM_num_tasks: str + How many SLURM tasks to request. + SLURM_num_tasks_per_node: str + How many SLURM tasks per compute node to request. + SLURM_num_nodes: str + How many compute nodes to request. + SLURM_num_cpus_per_task: str + How many CPU cores to ask for per SLURM task. """ + #: The names of parameters that may be used when making an instance of this class. ALLOWED_PARAMETERS = { "scratch", "parallel_mode", @@ -1407,6 +1694,7 @@ def __init__( SLURM_num_nodes: Optional[str] = None, SLURM_num_cpus_per_task: Optional[str] = None, ): + #: Which scope does this apply to. self.scope = scope or self.app.ActionScope.any() if not isinstance(self.scope, self.app.ActionScope): self.scope = self.app.ActionScope.from_json_like(self.scope) @@ -1498,10 +1786,16 @@ def _json_like_constructor(cls, json_like): @property def normalised_resources_path(self): + """ + Standard name of this resource spec. + """ return self.scope.to_string() @property def normalised_path(self): + """ + Full name of this resource spec. + """ return f"resources.{self.normalised_resources_path}" def to_dict(self): @@ -1596,31 +1890,55 @@ def _setter_persistent_check(self): @property def scratch(self): - # TODO: currently unused, except in tests + """ + Which scratch space to use. + + Todo + ---- + Currently unused, except in tests. + """ return self._get_value("scratch") @property def parallel_mode(self): + """ + Which parallel mode to use. + """ return self._get_value("parallel_mode") @property def num_cores(self): + """ + How many cores to request. + """ return self._get_value("num_cores") @property def num_cores_per_node(self): + """ + How many cores per compute node to request. + """ return self._get_value("num_cores_per_node") @property def num_nodes(self): + """ + How many compute nodes to request. + """ return self._get_value("num_nodes") @property def num_threads(self): + """ + How many threads to request. + """ return self._get_value("num_threads") @property def scheduler(self): + """ + Which scheduler to use. + """ return self._get_value("scheduler") @scheduler.setter @@ -1631,6 +1949,9 @@ def scheduler(self, value): @property def shell(self): + """ + Which system shell to use. + """ return self._get_value("shell") @shell.setter @@ -1641,54 +1962,93 @@ def shell(self, value): @property def use_job_array(self): + """ + Whether to use array jobs. + """ return self._get_value("use_job_array") @property def max_array_items(self): + """ + If using array jobs, up to how many items should be in the job array. + """ return self._get_value("max_array_items") @property def time_limit(self): + """ + How long to run for. + """ return self._get_value("time_limit") @property def scheduler_args(self): + """ + Additional arguments to pass to the scheduler. + """ return self._get_value("scheduler_args") @property def shell_args(self): + """ + Additional arguments to pass to the shell. + """ return self._get_value("shell_args") @property def os_name(self): + """ + Which OS to use. + """ return self._get_value("os_name") @property def environments(self): + """ + Which execution environments to use. + """ return self._get_value("environments") @property def SGE_parallel_env(self): + """ + Which SGE parallel environment to request. + """ return self._get_value("SGE_parallel_env") @property def SLURM_partition(self): + """ + Which SLURM partition to request. + """ return self._get_value("SLURM_partition") @property def SLURM_num_tasks(self): + """ + How many SLURM tasks to request. + """ return self._get_value("SLURM_num_tasks") @property def SLURM_num_tasks_per_node(self): + """ + How many SLURM tasks per compute node to request. + """ return self._get_value("SLURM_num_tasks_per_node") @property def SLURM_num_nodes(self): + """ + How many compute nodes to request. + """ return self._get_value("SLURM_num_nodes") @property def SLURM_num_cpus_per_task(self): + """ + How many CPU cores to ask for per SLURM task. + """ return self._get_value("SLURM_num_cpus_per_task") @os_name.setter @@ -1698,6 +2058,9 @@ def os_name(self, value): @property def workflow(self): + """ + The workflow owning this resource spec. + """ if self._workflow: return self._workflow @@ -1718,27 +2081,69 @@ def workflow(self): @property def element_set(self): + """ + The element set that will use this resource spec. + """ return self._resource_list.element_set @property def workflow_template(self): + """ + The workflow template that will use this resource spec. + """ return self._resource_list.workflow_template class InputSourceType(enum.Enum): + """ + The types if input sources. + """ + + #: Input source is an import. IMPORT = 0 + #: Input source is local. LOCAL = 1 + #: Input source is a default. DEFAULT = 2 + #: Input source is a task. TASK = 3 class TaskSourceType(enum.Enum): + """ + The types of task-based input sources. + """ + + #: Input source is a task input. INPUT = 0 + #: Input source is a task output. OUTPUT = 1 + #: Input source is unspecified. ANY = 2 class InputSource(JSONLike): + """ + An input source to a workflow task. + + Parameters + ---------- + source_type: InputSourceType + Type of the input source. + import_ref: + Where the input comes from when the type is `IMPORT`. + task_ref: + Which task is this an input for? Used when the type is `TASK`. + task_source_type: TaskSourceType + Type of task source. + element_iters: + Which element iterations does this apply to? + path: + Path to where this input goes. + where: ~hpcflow.app.Rule | list[~hpcflow.app.Rule] | ~hpcflow.app.ElementFilter + Filtering rules. + """ + _child_objects = ( ChildObjectSpec( name="source_type", @@ -1769,12 +2174,19 @@ def __init__( rules[idx] = app.Rule(**i) where = app.ElementFilter(rules=rules) + #: Type of the input source. self.source_type = self._validate_source_type(source_type) + #: Where the input comes from when the type is `IMPORT`. self.import_ref = import_ref + #: Which task is this an input for? Used when the type is `TASK`. self.task_ref = task_ref + #: Type of task source. self.task_source_type = self._validate_task_source_type(task_source_type) + #: Which element iterations does this apply to? self.element_iters = element_iters + #: Filtering rules. self.where = where + #: Path to where this input goes. self.path = path if self.source_type is InputSourceType.TASK: @@ -1852,6 +2264,9 @@ def is_in(self, other_input_sources: List[app.InputSource]) -> Union[None, int]: return None def to_string(self): + """ + Render this input source as a string. + """ out = [self.source_type.name.lower()] if self.source_type is InputSourceType.TASK: out += [str(self.task_ref), self.task_source_type.name.lower()] @@ -1893,20 +2308,26 @@ def _validate_task_source_type(cls, task_src_type): @classmethod def from_string(cls, str_defn): - return cls(**cls._parse_from_string(str_defn)) - - @classmethod - def _parse_from_string(cls, str_defn): """Parse a dot-delimited string definition of an InputSource. - Examples: - - task.[task_ref].input - - task.[task_ref].output - - local - - default - - import.[import_ref] + Parameter + --------- + str_defn: + The string to parse. + + Examples + -------- + task.[task_ref].input + task.[task_ref].output + local + default + import.[import_ref] """ + return cls(**cls._parse_from_string(str_defn)) + + @classmethod + def _parse_from_string(cls, str_defn): parts = str_defn.split(".") source_type = cls._validate_source_type(parts[0]) task_ref = None @@ -1957,6 +2378,18 @@ def from_json_like(cls, json_like, shared_data=None): @classmethod def import_(cls, import_ref, element_iters=None, where=None): + """ + Make an instnace of an input source that is an import. + + Parameters + ---------- + import_ref: + Import reference. + element_iters: + Originating element iterations. + where: + Filtering rule. + """ return cls( source_type=cls.app.InputSourceType.IMPORT, import_ref=import_ref, @@ -1966,14 +2399,34 @@ def import_(cls, import_ref, element_iters=None, where=None): @classmethod def local(cls): + """ + Make an instnace of an input source that is local. + """ return cls(source_type=cls.app.InputSourceType.LOCAL) @classmethod def default(cls): + """ + Make an instnace of an input source that is default. + """ return cls(source_type=cls.app.InputSourceType.DEFAULT) @classmethod def task(cls, task_ref, task_source_type=None, element_iters=None, where=None): + """ + Make an instnace of an input source that is a task. + + Parameters + ---------- + task_ref: + Source task reference. + task_source_type: + Type of task source. + element_iters: + Originating element iterations. + where: + Filtering rule. + """ if not task_source_type: task_source_type = cls.app.TaskSourceType.OUTPUT return cls( diff --git a/hpcflow/sdk/core/rule.py b/hpcflow/sdk/core/rule.py index 85666226e..cfc15386c 100644 --- a/hpcflow/sdk/core/rule.py +++ b/hpcflow/sdk/core/rule.py @@ -1,3 +1,7 @@ +""" +Rules apply conditions to workflow elements or loops. +""" + from __future__ import annotations from typing import Dict, Optional, Union @@ -11,7 +15,27 @@ class Rule(JSONLike): - """Class to represent a testable condition on an element iteration or run.""" + """ + Class to represent a testable condition on an element iteration or run. + + Exactly one of ``check_exists``, ``check_missing`` and ``condition`` must be provided. + + Parameters + ---------- + check_exists: str + If set, check this attribute exists. + check_missing: str + If set, check this attribute does *not* exist. + path: str + Where to look up the attribute to check. + If not specified, determined by context. + condition: ConditionLike + A general condition to check (or kwargs used to generate one). + cast: str + If set, a cast to apply prior to running the general check. + doc: str + Optional descriptive text. + """ def __init__( self, @@ -31,11 +55,17 @@ def __init__( if isinstance(condition, dict): condition = ConditionLike.from_json_like(condition) + #: If set, this rule checks this attribute exists. self.check_exists = check_exists + #: If set, this rule checks this attribute does *not* exist. self.check_missing = check_missing + #: Where to look up the attribute to check (if not determined by context). self.path = path + #: A general condition for this rule to check. self.condition = condition + #: If set, a cast to apply prior to running the general check. self.cast = cast + #: Optional descriptive text. self.doc = doc def __repr__(self): diff --git a/hpcflow/sdk/core/run_dir_files.py b/hpcflow/sdk/core/run_dir_files.py index aa554c2de..b83ca15ed 100644 --- a/hpcflow/sdk/core/run_dir_files.py +++ b/hpcflow/sdk/core/run_dir_files.py @@ -1,3 +1,7 @@ +""" +Model of files in the run directory. +""" + import re from hpcflow.sdk.core.utils import JSONLikeDirSnapShot @@ -8,7 +12,7 @@ class RunDirAppFiles: _app_attr = "app" - CMD_FILES_RE_PATTERN = r"js_\d+_act_\d+\.?\w*" + _CMD_FILES_RE_PATTERN = r"js_\d+_act_\d+\.?\w*" @classmethod def get_log_file_name(cls): @@ -22,10 +26,16 @@ def get_std_file_name(cls): @staticmethod def get_run_file_prefix(js_idx: int, js_action_idx: int): + """ + Get the common prefix for files associated with a run. + """ return f"js_{js_idx}_act_{js_action_idx}" @classmethod def get_commands_file_name(cls, js_idx: int, js_action_idx: int, shell): + """ + Get the name of the file containing commands. + """ return cls.get_run_file_prefix(js_idx, js_action_idx) + shell.JS_EXT @classmethod @@ -56,7 +66,7 @@ def take_snapshot(cls): if ( k == cls.get_log_file_name() or k == cls.get_std_file_name() - or re.match(cls.CMD_FILES_RE_PATTERN, k) + or re.match(cls._CMD_FILES_RE_PATTERN, k) ): ss_js["data"].pop(k) diff --git a/hpcflow/sdk/core/task.py b/hpcflow/sdk/core/task.py index 6f9d9a27c..2c8107cb7 100644 --- a/hpcflow/sdk/core/task.py +++ b/hpcflow/sdk/core/task.py @@ -1,3 +1,7 @@ +""" +Tasks are components of workflows. +""" + from __future__ import annotations from collections import defaultdict import copy @@ -66,18 +70,59 @@ class InputStatus: """ + #: True if a default value is available. has_default: bool + #: True if the input is required by one or more actions. An input may not be required + #: if it is only used in the generation of inputs files, and those input files are + #: passed to the element set directly. is_required: bool + #: True if the input is locally provided in the element set. is_provided: bool @property def is_extra(self): - """Return True if the input is provided but not required.""" + """True if the input is provided but not required.""" return self.is_provided and not self.is_required class ElementSet(JSONLike): - """Class to represent a parametrisation of a new set of elements.""" + """Class to represent a parameterisation of a new set of elements. + + Parameters + ---------- + inputs: list[~hpcflow.app.InputValue] + Inputs to the set of elements. + input_files: list[~hpcflow.app.InputFile] + Input files to the set of elements. + sequences: list[~hpcflow.app.ValueSequence] + Input value sequences to parameterise over. + resources: ~hpcflow.app.ResourceList + Resources to use for the set of elements. + repeats: list[dict] + Description of how to repeat the set of elements. + groups: list[~hpcflow.app.ElementGroup] + Groupings in the set of elements. + input_sources: dict[str, ~hpcflow.app.InputSource] + Input source descriptors. + nesting_order: dict[str, int] + How to handle nesting of iterations. + env_preset: str + Which environment preset to use. Don't use at same time as ``environments``. + environments: dict + Environment descriptors to use. Don't use at same time as ``env_preset``. + sourceable_elem_iters: list[int] + If specified, a list of global element iteration indices from which inputs for + the new elements associated with this element set may be sourced. If not + specified, all workflow element iterations are considered sourceable. + allow_non_coincident_task_sources: bool + If True, if more than one parameter is sourced from the same task, then allow + these sources to come from distinct element sub-sets. If False (default), + only the intersection of element sub-sets for all parameters are included. + merge_envs: bool + If True, merge ``environments`` into ``resources`` using the "any" scope. If + False, ``environments`` are ignored. This is required on first initialisation, + but not on subsequent re-initialisation from a persistent workflow. + """ _child_objects = ( ChildObjectSpec( @@ -137,35 +182,34 @@ def __init__( allow_non_coincident_task_sources: Optional[bool] = False, merge_envs: Optional[bool] = True, ): - """ - Parameters - ---------- - sourceable_elem_iters - If specified, a list of global element iteration indices from which inputs for - the new elements associated with this element set may be sourced. If not - specified, all workflow element iterations are considered sourceable. - allow_non_coincident_task_sources - If True, if more than one parameter is sourced from the same task, then allow - these sources to come from distinct element sub-sets. If False (default), - only the intersection of element sub-sets for all parameters are included. - merge_envs - If True, merge `environments` into `resources` using the "any" scope. If - False, `environments` are ignored. This is required on first initialisation, - but not on subsequent re-initialisation from a persistent workflow. - """ - + #: Inputs to the set of elements. self.inputs = inputs or [] + #: Input files to the set of elements. self.input_files = input_files or [] + #: Description of how to repeat the set of elements. self.repeats = repeats or [] + #: Groupings in the set of elements. self.groups = groups or [] + #: Resources to use for the set of elements. self.resources = self.app.ResourceList.normalise(resources) + #: Input value sequences to parameterise over. self.sequences = sequences or [] + #: Input source descriptors. self.input_sources = input_sources or {} + #: How to handle nesting of iterations. self.nesting_order = nesting_order or {} + #: Which environment preset to use. self.env_preset = env_preset + #: Environment descriptors to use. self.environments = environments + #: List of global element iteration indices from which inputs for + #: the new elements associated with this element set may be sourced. + #: If ``None``, all iterations are valid. self.sourceable_elem_iters = sourceable_elem_iters + #: Whether to allow sources to come from distinct element sub-sets. self.allow_non_coincident_task_sources = allow_non_coincident_task_sources + #: Whether to merge ``environments`` into ``resources`` using the "any" scope + #: on first initialisation. self.merge_envs = merge_envs self._validate() @@ -235,6 +279,9 @@ def to_dict(self): @property def task_template(self): + """ + The abstract task this was derived from. + """ return self._task_template @task_template.setter @@ -244,11 +291,14 @@ def task_template(self, value): @property def input_types(self): + """ + The input types of the inputs to this element set. + """ return [i.labelled_type for i in self.inputs] @property def element_local_idx_range(self): - """Used to retrieve elements belonging to this element set.""" + """Indices of elements belonging to this element set.""" return tuple(self._element_local_idx_range) def _validate(self): @@ -376,6 +426,9 @@ def ensure_element_sets( element_sets=None, sourceable_elem_iters=None, ): + """ + Make an instance after validating some argument combinations. + """ args = ( inputs, input_files, @@ -417,18 +470,30 @@ def ensure_element_sets( @property def defined_input_types(self): + """ + The input types to this element set. + """ return self._defined_input_types @property def undefined_input_types(self): + """ + The input types to the abstract task that aren't related to this element set. + """ return self.task_template.all_schema_input_types - self.defined_input_types def get_sequence_from_path(self, sequence_path): - for i in self.sequences: - if i.path == sequence_path: - return i + """ + Get the value sequence for the given path, if it exists. + """ + for seq in self.sequences: + if seq.path == sequence_path: + return seq def get_defined_parameter_types(self): + """ + Get the parameter types of this element set. + """ out = [] for inp in self.inputs: if not inp.is_sub_value: @@ -439,6 +504,9 @@ def get_defined_parameter_types(self): return out def get_defined_sub_parameter_types(self): + """ + Get the sub-parameter types of this element set. + """ out = [] for inp in self.inputs: if inp.is_sub_value: @@ -449,35 +517,48 @@ def get_defined_sub_parameter_types(self): return out def get_locally_defined_inputs(self): + """ + Get the input types that this element set defines. + """ return self.get_defined_parameter_types() + self.get_defined_sub_parameter_types() - def get_sequence_by_path(self, path): - for seq in self.sequences: - if seq.path == path: - return seq - @property def index(self): + """ + The index of this element set in its' template task's collection of sets. + """ for idx, element_set in enumerate(self.task_template.element_sets): if element_set is self: return idx @property def task(self): + """ + The concrete task corresponding to this element set. + """ return self.task_template.workflow_template.workflow.tasks[ self.task_template.index ] @property def elements(self): + """ + The elements in this element set. + """ return self.task.elements[slice(*self.element_local_idx_range)] @property def element_iterations(self): + """ + The iterations in this element set. + """ return [j for i in self.elements for j in i.iterations] @property def elem_iter_IDs(self): + """ + The IDs of the iterations in this element set. + """ return [i.id_ for i in self.element_iterations] def get_task_dependencies(self, as_objects=False): @@ -510,8 +591,18 @@ def is_input_type_provided(self, labelled_path: str) -> bool: class OutputLabel(JSONLike): - """Class to represent schema input labels that should be applied to a subset of task - outputs""" + """ + Schema input labels that should be applied to a subset of task outputs. + + Parameters + ---------- + parameter: + Name of a parameter. + label: + Label to apply to the parameter. + where: ~hpcflow.app.ElementFilter + Optional filtering rule + """ _child_objects = ( ChildObjectSpec( @@ -526,22 +617,50 @@ def __init__( label: str, where: Optional[List[app.ElementFilter]] = None, ) -> None: + #: Name of a parameter. self.parameter = parameter + #: Label to apply to the parameter. self.label = label + #: Filtering rule. self.where = where class Task(JSONLike): - """Parametrisation of an isolated task for which a subset of input values are given + """ + Parametrisation of an isolated task for which a subset of input values are given "locally". The remaining input values are expected to be satisfied by other tasks/imports in the workflow. Parameters ---------- - schema - A `TaskSchema` object or a list of `TaskSchema` objects. - inputs + schema: ~hpcflow.app.TaskSchema | list[~hpcflow.app.TaskSchema] + A (list of) `TaskSchema` object(s) and/or a (list of) strings that are task + schema names that uniquely identify a task schema. If strings are provided, + the `TaskSchema` object will be fetched from the known task schemas loaded by + the app configuration. + repeats: list[dict] + groups: list[~hpcflow.app.ElementGroup] + resources: dict + inputs: list[~hpcflow.app.InputValue] A list of `InputValue` objects. + input_files: list[~hpcflow.app.InputFile] + sequences: list[~hpcflow.app.ValueSequence] + input_sources: dict[str, ~hpcflow.app.InputSource] + nesting_order: list + env_preset: str + environments: dict[str, dict] + allow_non_coincident_task_sources: bool + If True, if more than one parameter is sourced from the same task, then allow + these sources to come from distinct element sub-sets. If False (default), + only the intersection of element sub-sets for all parameters are included. + element_sets: list[ElementSet] + output_labels: list[OutputLabel] + sourceable_elem_iters: list[int] + merge_envs: bool + If True, merge environment presets (set via the element set `env_preset` key) + into `resources` using the "any" scope. If False, these presets are ignored. + This is required on first initialisation, but not on subsequent + re-initialisation from a persistent workflow. """ _child_objects = ( @@ -585,25 +704,6 @@ def __init__( sourceable_elem_iters: Optional[List[int]] = None, merge_envs: Optional[bool] = True, ): - """ - Parameters - ---------- - schema - A (list of) `TaskSchema` object(s) and/or a (list of) strings that are task - schema names that uniquely identify a task schema. If strings are provided, - the `TaskSchema` object will be fetched from the known task schemas loaded by - the app configuration. - allow_non_coincident_task_sources - If True, if more than one parameter is sourced from the same task, then allow - these sources to come from distinct element sub-sets. If False (default), - only the intersection of element sub-sets for all parameters are included. - merge_envs - If True, merge environment presets (set via the element set `env_preset` key) - into `resources` using the "any" scope. If False, these presets are ignored. - This is required on first initialisation, but not on subsequent - re-initialisation from a persistent workflow. - """ - # TODO: allow init via specifying objective and/or method and/or implementation # (lists of) strs e.g.: Task( # objective='simulate_VE_loading', @@ -652,6 +752,8 @@ def __init__( sourceable_elem_iters=sourceable_elem_iters, ) self._output_labels = output_labels or [] + #: Whether to merge ``environments`` into ``resources`` using the "any" scope + #: on first initialisation. self.merge_envs = merge_envs # appended to when new element sets are added and reset on dump to disk: @@ -660,6 +762,7 @@ def __init__( self._validate() self._name = self._get_name() + #: The template workflow that this task is within. self.workflow_template = None # assigned by parent WorkflowTemplate self._insert_ID = None self._dir_name = None @@ -800,6 +903,9 @@ def to_dict(self): } def set_sequence_parameters(self, element_set): + """ + Set up parameters parsed by value sequences. + """ # set ValueSequence Parameter objects: for seq in element_set.sequences: if seq.input_type: @@ -882,6 +988,11 @@ def _prepare_persistent_outputs(self, workflow, local_element_idx_range): return output_data_indices def prepare_element_resolution(self, element_set, input_data_indices): + """ + Set up the resolution of details of elements + (especially multiplicities and how iterations are nested) + within an element set. + """ multiplicities = [] for path_i, inp_idx_i in input_data_indices.items(): multiplicities.append( @@ -917,6 +1028,9 @@ def prepare_element_resolution(self, element_set, input_data_indices): @property def index(self): + """ + The index of this task within the workflow's tasks. + """ if self.workflow_template: return self.workflow_template.tasks.index(self) else: @@ -924,6 +1038,9 @@ def index(self): @property def output_labels(self): + """ + The labels on the outputs of the task. + """ return self._output_labels @property @@ -1113,11 +1230,14 @@ def get_available_task_input_sources( @property def schemas(self) -> List[app.TaskSchema]: + """ + All the task schemas. + """ return self._schemas @property def schema(self) -> app.TaskSchema: - """Returns the single task schema, if only one, else raises.""" + """The single task schema, if only one, else raises.""" if len(self._schemas) == 1: return self._schemas[0] else: @@ -1128,52 +1248,81 @@ def schema(self) -> app.TaskSchema: @property def element_sets(self): + """ + The element sets. + """ return self._element_sets + self._pending_element_sets @property def num_element_sets(self): + """ + The number of element sets. + """ return len(self.element_sets) @property def insert_ID(self): + """ + Insertion ID. + """ return self._insert_ID @property def dir_name(self): - "Artefact directory name." + """ + Artefact directory name. + """ return self._dir_name @property def name(self): + """ + Task name. + """ return self._name @property def objective(self): + """ + The goal of this task. + """ return self.schemas[0].objective @property def all_schema_inputs(self) -> Tuple[app.SchemaInput]: + """ + The inputs to this task's schemas. + """ return tuple(inp_j for schema_i in self.schemas for inp_j in schema_i.inputs) @property def all_schema_outputs(self) -> Tuple[app.SchemaOutput]: + """ + The outputs from this task's schemas. + """ return tuple(inp_j for schema_i in self.schemas for inp_j in schema_i.outputs) @property def all_schema_input_types(self): - """Get the set of all schema input types (over all specified schemas).""" + """The set of all schema input types (over all specified schemas).""" return {inp_j for schema_i in self.schemas for inp_j in schema_i.input_types} @property def all_schema_input_normalised_paths(self): + """ + Normalised paths for all schema input types. + """ return {f"inputs.{i}" for i in self.all_schema_input_types} @property def all_schema_output_types(self): - """Get the set of all schema output types (over all specified schemas).""" + """The set of all schema output types (over all specified schemas).""" return {out_j for schema_i in self.schemas for out_j in schema_i.output_types} def get_schema_action(self, idx): + """ + Get the schema action at the given index. + """ _idx = 0 for schema in self.schemas: for action in schema.actions: @@ -1183,6 +1332,9 @@ def get_schema_action(self, idx): raise ValueError(f"No action in task {self.name!r} with index {idx!r}.") def all_schema_actions(self) -> Iterator[Tuple[int, app.Action]]: + """ + Get all the schema actions and their indices. + """ idx = 0 for schema in self.schemas: for action in schema.actions: @@ -1191,6 +1343,9 @@ def all_schema_actions(self) -> Iterator[Tuple[int, app.Action]]: @property def num_all_schema_actions(self) -> int: + """ + The total number of schema actions. + """ num = 0 for schema in self.schemas: for _ in schema.actions: @@ -1199,6 +1354,9 @@ def num_all_schema_actions(self) -> int: @property def all_sourced_normalised_paths(self): + """ + All the sourced normalised paths, including of sub-values. + """ sourced_input_types = [] for elem_set in self.element_sets: for inp in elem_set.inputs: @@ -1280,14 +1438,23 @@ def non_universal_input_types(self): @property def defined_input_types(self): + """ + The input types defined by this task. + """ return self._defined_input_types @property def undefined_input_types(self): + """ + The schema's input types that this task doesn't define. + """ return self.all_schema_input_types - self.defined_input_types @property def undefined_inputs(self): + """ + The task's inputs that are undefined. + """ return [ inp_j for schema_i in self.schemas @@ -1323,6 +1490,9 @@ def provides_parameters(self) -> Tuple[Tuple[str, str]]: def add_group( self, name: str, where: app.ElementFilter, group_by_distinct: app.ParameterPath ): + """ + Add an element group to this task. + """ group = ElementGroup(name=name, where=where, group_by_distinct=group_by_distinct) self.groups.add_object(group) @@ -1346,7 +1516,20 @@ def _get_single_label_lookup(self, prefix="") -> Dict[str, str]: class WorkflowTask: - """Class to represent a Task that is bound to a Workflow.""" + """ + Represents a :py:class:`Task` that is bound to a :py:class:`Workflow`. + + Parameters + ---------- + workflow: + The workflow that the task is bound to. + template: + The task template that this binds. + index: + Where in the workflow's list of tasks is this one. + element_IDs: + The IDs of the elements of this task. + """ _app_attr = "app" @@ -1379,6 +1562,18 @@ def _accept_pending_element_IDs(self): @classmethod def new_empty_task(cls, workflow: app.Workflow, template: app.Task, index: int): + """ + Make a new instance without any elements set up yet. + + Parameters + ---------- + workflow: + The workflow that the task is bound to. + template: + The task template that this binds. + index: + Where in the workflow's list of tasks is this one. + """ obj = cls( workflow=workflow, template=template, @@ -1389,61 +1584,103 @@ def new_empty_task(cls, workflow: app.Workflow, template: app.Task, index: int): @property def workflow(self): + """ + The workflow this task is bound to. + """ return self._workflow @property def template(self): + """ + The template for this task. + """ return self._template @property def index(self): + """ + The index of this task within its workflow. + """ return self._index @property def element_IDs(self): + """ + The IDs of elements associated with this task. + """ return self._element_IDs + self._pending_element_IDs @property def num_elements(self): + """ + The number of elements associated with this task. + """ return len(self.element_IDs) @property def num_actions(self): + """ + The number of actions in this task. + """ return self.template.num_all_schema_actions @property def name(self): + """ + The name of this task based on its template. + """ return self.template.name @property def unique_name(self): + """ + The unique name for this task specifically. + """ return self.workflow.get_task_unique_names()[self.index] @property def insert_ID(self): + """ + The insertion ID of the template task. + """ return self.template.insert_ID @property def dir_name(self): + """ + The name of the directory for the task's temporary files. + """ return self.template.dir_name @property def num_element_sets(self): + """ + The number of element sets associated with this task. + """ return self.template.num_element_sets @property @TimeIt.decorator def elements(self): + """ + The elements associated with this task. + """ if self._elements is None: self._elements = self.app.Elements(self) return self._elements def get_dir_name(self, loop_idx: Dict[str, int] = None) -> str: + """ + Get the directory name for a particular iteration. + """ if not loop_idx: return self.dir_name return self.dir_name + "_" + "_".join((f"{k}-{v}" for k, v in loop_idx.items())) def get_all_element_iterations(self) -> Dict[int, app.ElementIteration]: + """ + Get the iterations known by the task's elements. + """ return {j.id_: j for i in self.elements for j in i.iterations} def _make_new_elements_persistent( @@ -1633,7 +1870,7 @@ def _make_new_elements_persistent( source_idx[key] = [inp_src_idx] * len(grp_idx) if key in sequence_idx: sequence_idx.pop(key) - seq = element_set.get_sequence_by_path(key) + seq = element_set.get_sequence_from_path(key) elif inp_src.source_type is InputSourceType.DEFAULT: grp_idx = [def_val._value_group_idx] @@ -1950,6 +2187,9 @@ def generate_new_elements( sequence_indices, source_indices, ): + """ + Create information about new elements in this task. + """ new_elements = [] element_sequence_indices = {} element_src_indices = {} @@ -1984,12 +2224,12 @@ def generate_new_elements( @property def upstream_tasks(self): - """Get all workflow tasks that are upstream from this task.""" + """All workflow tasks that are upstream from this task.""" return [task for task in self.workflow.tasks[: self.index]] @property def downstream_tasks(self): - """Get all workflow tasks that are downstream from this task.""" + """All workflow tasks that are downstream from this task.""" return [task for task in self.workflow.tasks[self.index + 1 :]] @staticmethod @@ -2246,6 +2486,22 @@ def add_elements( propagate_to=None, return_indices=False, ): + """ + Add elements to this task. + + Parameters + ---------- + sourceable_elem_iters : list of int, optional + If specified, a list of global element iteration indices from which inputs + may be sourced. If not specified, all workflow element iterations are + considered sourceable. + propagate_to : dict[str, ElementPropagation] + Propagate the new elements downstream to the specified tasks. + return_indices : bool + If True, return the list of indices of the newly added elements. False by + default. + + """ propagate_to = self.app.ElementPropagation._prepare_propagate_to_dict( propagate_to, self.workflow ) @@ -2281,21 +2537,7 @@ def _add_elements( propagate_to: Dict[str, app.ElementPropagation] = None, return_indices: bool = False, ): - """Add more elements to this task. - - Parameters - ---------- - sourceable_elem_iters : list of int, optional - If specified, a list of global element iteration indices from which inputs - may be sourced. If not specified, all workflow element iterations are - considered sourceable. - propagate_to : dict of [str, ElementPropagation] - Propagate the new elements downstream to the specified tasks. - return_indices : bool, optional - If True, return the list of indices of the newly added elements. False by - default. - - """ + """Add more elements to this task.""" if base_element is not None: if base_element.task is not self: @@ -2467,13 +2709,22 @@ def get_dependent_tasks( @property def inputs(self): + """ + Inputs to this task. + """ return self.app.TaskInputParameters(self) @property def outputs(self): + """ + Outputs from this task. + """ return self.app.TaskOutputParameters(self) def get(self, path, raise_on_missing=False, default=None): + """ + Get a parameter known to this task by its path. + """ return self.app.Parameters( self, path=path, @@ -2866,6 +3117,15 @@ def _merge_relevant_data( class Elements: + """ + The elements of a task. Iterable. + + Parameters + ---------- + task: + The task this will be the elements of. + """ + __slots__ = ("_task",) def __init__(self, task: app.WorkflowTask): @@ -2881,6 +3141,9 @@ def __repr__(self) -> str: @property def task(self): + """ + The task this is the elements of. + """ return self._task @TimeIt.decorator @@ -2924,13 +3187,38 @@ def __getitem__( @dataclass class Parameters: + """ + The parameters of a (workflow-bound) task. Iterable. + + Parameters + ---------- + task: WorkflowTask + The task these are the parameters of. + path: str + The path to the parameter or parameters. + return_element_parameters: bool + Whether to return element parameters. + raise_on_missing: bool + Whether to raise an exception on a missing parameter. + raise_on_unset: bool + Whether to raise an exception on an unset parameter. + default: + A default value to use when the parameter is absent. + """ + _app_attr = "_app" + #: The task these are the parameters of. task: app.WorkflowTask + #: The path to the parameter or parameters. path: str + #: Whether to return element parameters. return_element_parameters: bool + #: Whether to raise an exception on a missing parameter. raise_on_missing: Optional[bool] = False + #: Whether to raise an exception on an unset parameter. raise_on_unset: Optional[bool] = False + #: A default value to use when the parameter is absent. default: Optional[Any] = None @TimeIt.decorator @@ -2990,10 +3278,19 @@ def __getitem__( @dataclass class TaskInputParameters: - """For retrieving schema input parameters across all elements.""" + """ + For retrieving schema input parameters across all elements. + Treat as an unmodifiable namespace. + + Parameters + ---------- + task: + The task that this represents the input parameters of. + """ _app_attr = "_app" + #: The task that this represents the input parameters of. task: app.WorkflowTask def __getattr__(self, name): @@ -3016,10 +3313,19 @@ def _get_input_names(self): @dataclass class TaskOutputParameters: - """For retrieving schema output parameters across all elements.""" + """ + For retrieving schema output parameters across all elements. + Treat as an unmodifiable namespace. + + Parameters + ---------- + task: + The task that this represents the output parameters of. + """ _app_attr = "_app" + #: The task that this represents the output parameters of. task: app.WorkflowTask def __getattr__(self, name): @@ -3042,17 +3348,38 @@ def _get_output_names(self): @dataclass class ElementPropagation: - """Class to represent how a newly added element set should propagate to a given - downstream task.""" + """ + Class to represent how a newly added element set should propagate to a given + downstream task. + + Parameters + ---------- + task: + The task this is propagating to. + nesting_order: + The nesting order information. + input_sources: + The input source information. + """ _app_attr = "app" + #: The task this is propagating to. task: app.Task + #: The nesting order information. nesting_order: Optional[Dict] = None + #: The input source information. input_sources: Optional[Dict] = None @property def element_set(self): + """ + The element set that this propagates from. + + Note + ---- + Temporary property. May be moved or reinterpreted. + """ # TEMP property; for now just use the first element set as the base: return self.task.template.element_sets[0] diff --git a/hpcflow/sdk/core/task_schema.py b/hpcflow/sdk/core/task_schema.py index cee5e9f2d..588cdcd74 100644 --- a/hpcflow/sdk/core/task_schema.py +++ b/hpcflow/sdk/core/task_schema.py @@ -1,3 +1,7 @@ +""" +Abstract task, prior to instantiation. +""" + from contextlib import contextmanager import copy from dataclasses import dataclass @@ -22,6 +26,15 @@ @dataclass class TaskObjective(JSONLike): + """ + A thing that a task is attempting to achieve. + + Parameter + --------- + name: str + The name of the objective. A valid Python identifier. + """ + _child_objects = ( ChildObjectSpec( name="name", @@ -29,6 +42,7 @@ class TaskObjective(JSONLike): ), ) + #: The name of the objective. A valid Python identifier. name: str def __post_init__(self): @@ -41,21 +55,28 @@ class TaskSchema(JSONLike): Parameters ---------- - objective + objective: This is a string representing the objective of the task schema. - actions + actions: A list of Action objects whose commands are to be executed by the task. - method + method: An optional string to label the task schema by its method. - implementation + implementation: An optional string to label the task schema by its implementation. - inputs + inputs: A list of SchemaInput objects that define the inputs to the task. - outputs + outputs: A list of SchemaOutput objects that define the outputs of the task. - web_doc - True if this object should be included in the Sphinx documentation (in the case - of built-in task schemas). True by default. + version: + The version of this task schema. + parameter_class_modules: + Where to find implementations of parameter value handlers. + web_doc: + True if this object should be included in the Sphinx documentation + (normally only relevant for built-in task schemas). True by default. + environment_presets: + Information about default execution environments. Can be overridden in specific + cases in the concrete tasks. """ _validation_schema = "task_schema_spec_schema.yaml" @@ -93,14 +114,24 @@ def __init__( environment_presets: Optional[Dict[str, Dict[str, Dict[str, Any]]]] = None, _hash_value: Optional[str] = None, ): + #: This is a string representing the objective of the task schema. self.objective = objective + #: A list of Action objects whose commands are to be executed by the task. self.actions = actions or [] + #: An optional string to label the task schema by its method. self.method = method + #: An optional string to label the task schema by its implementation. self.implementation = implementation + #: A list of SchemaInput objects that define the inputs to the task. self.inputs = inputs or [] + #: A list of SchemaOutput objects that define the outputs of the task. self.outputs = outputs or [] + #: Where to find implementations of parameter value handlers. self.parameter_class_modules = parameter_class_modules or [] + #: Whether this object should be included in the Sphinx documentation + #: (normally only relevant for built-in task schemas). self.web_doc = web_doc + #: Information about default execution environments. self.environment_presets = environment_presets self._hash_value = _hash_value @@ -112,6 +143,7 @@ def __init__( self._validate() self.actions = self._expand_actions() + #: The version of this task schema. self.version = version self._task_template = None # assigned by parent Task @@ -309,6 +341,10 @@ def info(self): return self._show_info() def get_info_html(self) -> str: + """ + Describe the task schema as an HTML document. + """ + def _format_parameter_type(param): param_typ_fmt = param.typ if param.typ in param_types: @@ -586,6 +622,9 @@ def __deepcopy__(self, memo): @classmethod @contextmanager def ignore_invalid_actions(cls): + """ + A context manager within which invalid actions will be ignored. + """ try: cls._validate_actions = False yield @@ -709,6 +748,10 @@ def _update_parameter_value_classes(self): out.parameter._set_value_class() def make_persistent(self, workflow: app.Workflow, source: Dict) -> List[int]: + """ + Convert this task schema to persistent form within the context of the given + workflow. + """ new_refs = [] for input_i in self.inputs: for lab_info in input_i.labelled_info(): @@ -721,6 +764,9 @@ def make_persistent(self, workflow: app.Workflow, source: Dict) -> List[int]: @property def name(self): + """ + The name of this schema. + """ out = ( f"{self.objective.name}" f"{f'_{self.method}' if self.method else ''}" @@ -730,14 +776,23 @@ def name(self): @property def input_types(self): + """ + The input types to the schema. + """ return tuple(j for i in self.inputs for j in i.all_labelled_types) @property def output_types(self): + """ + The output types from the schema. + """ return tuple(i.typ for i in self.outputs) @property def provides_parameters(self) -> Tuple[Tuple[str, str]]: + """ + The parameters that this schema provides. + """ out = [] for schema_inp in self.inputs: for labelled_info in schema_inp.labelled_info(): @@ -753,6 +808,9 @@ def provides_parameters(self) -> Tuple[Tuple[str, str]]: @property def task_template(self): + """ + The template that this schema is contained in. + """ return self._task_template @classmethod @@ -770,6 +828,9 @@ def get_parameter_dependence(self, parameter: app.SchemaParameter): return out def get_key(self): + """ + Get the hashable value that represents this schema. + """ return (str(self.objective), self.method, self.implementation) def _get_single_label_lookup(self, prefix="") -> Dict[str, str]: diff --git a/hpcflow/sdk/core/test_utils.py b/hpcflow/sdk/core/test_utils.py index 960a010ee..5a403f936 100644 --- a/hpcflow/sdk/core/test_utils.py +++ b/hpcflow/sdk/core/test_utils.py @@ -1,3 +1,7 @@ +""" +Utilities for making data to use in testing. +""" + from dataclasses import dataclass from importlib import resources from pathlib import Path @@ -7,6 +11,9 @@ def make_schemas(ins_outs, ret_list=False): + """ + Construct a collection of schemas. + """ out = [] for idx, info in enumerate(ins_outs): if len(info) == 2: @@ -57,6 +64,9 @@ def make_schemas(ins_outs, ret_list=False): def make_parameters(num): + """ + Construct a sequence of parameters. + """ return [hf.Parameter(f"p{i + 1}") for i in range(num)] @@ -64,6 +74,9 @@ def make_actions( ins_outs: List[Tuple[Union[Tuple, str], str]], env="env1", ) -> List[hf.Action]: + """ + Construct a collection of actions. + """ act_env = hf.ActionEnvironment(environment=env) actions = [] for ins_outs_i in ins_outs: @@ -98,6 +111,9 @@ def make_tasks( input_sources=None, groups=None, ): + """ + Construct a sequence of tasks. + """ local_inputs = local_inputs or {} local_sequences = local_sequences or {} local_resources = local_resources or {} @@ -148,6 +164,9 @@ def make_workflow( overwrite=False, store="zarr", ): + """ + Construct a workflow. + """ tasks = make_tasks( schemas_spec, local_inputs=local_inputs, @@ -202,6 +221,10 @@ def make_test_data_YAML_workflow_template(workflow_name, **kwargs): @dataclass class P1_sub_parameter_cls(ParameterValue): + """ + Parameter value handler: ``p1_sub`` + """ + _typ = "p1_sub" e: int @@ -222,6 +245,10 @@ def dump_to_HDF5_group(self, group): @dataclass class P1_sub_parameter_cls_2(ParameterValue): + """ + Parameter value handler: ``p1_sub_2`` + """ + _typ = "p1_sub_2" f: int @@ -229,6 +256,14 @@ class P1_sub_parameter_cls_2(ParameterValue): @dataclass class P1_parameter_cls(ParameterValue): + """ + Parameter value handler: ``p1c`` + + Note + ---- + This is a composite value handler. + """ + _typ = "p1c" _sub_parameters = {"sub_param": "p1_sub", "sub_param_2": "p1_sub_2"} diff --git a/hpcflow/sdk/core/utils.py b/hpcflow/sdk/core/utils.py index e5a4b7d05..7ea8b0b77 100644 --- a/hpcflow/sdk/core/utils.py +++ b/hpcflow/sdk/core/utils.py @@ -1,3 +1,7 @@ +""" +Miscellaneous utilities. +""" + import copy import enum from functools import wraps @@ -45,12 +49,18 @@ def wrapper(self, *args, **kwargs): def make_workflow_id(): + """ + Generate a random ID for a workflow. + """ length = 12 chars = string.ascii_letters + "0123456789" return "".join(random.choices(chars, k=length)) def get_time_stamp(): + """ + Get the current time in standard string form. + """ return datetime.now(timezone.utc).astimezone().strftime("%Y.%m.%d_%H:%M:%S_%z") @@ -181,6 +191,11 @@ def swap_nested_dict_keys(dct, inner_key): def get_in_container(cont, path, cast_indices=False, allow_getattr=False): + """ + Follow a path (sequence of indices of appropriate type) into a container to obtain + a "leaf" value. Containers can be lists, tuples, dicts, + or any class (with `getattr()`) if ``allow_getattr`` is True. + """ cur_data = cont err_msg = ( "Data at path {path_comps!r} is not a sequence, but is of type " @@ -221,6 +236,11 @@ def get_in_container(cont, path, cast_indices=False, allow_getattr=False): def set_in_container(cont, path, value, ensure_path=False, cast_indices=False): + """ + Follow a path (sequence of indices of appropriate type) into a container to update + a "leaf" value. Containers can be lists, tuples or dicts. + The "branch" holding the leaf to update must be modifiable. + """ if ensure_path: num_path = len(path) for idx in range(1, num_path): @@ -315,6 +335,10 @@ def search_dir_files_by_regex(pattern, group=0, directory=".") -> List[str]: class classproperty(object): + """ + Simple class property decorator. + """ + def __init__(self, f): self.f = f @@ -323,6 +347,11 @@ def __get__(self, obj, owner): class PrettyPrinter(object): + """ + A class that produces a nice readable version of itself with ``str()``. + Intended to be subclassed. + """ + def __str__(self): lines = [self.__class__.__name__ + ":"] for key, val in vars(self).items(): @@ -331,6 +360,11 @@ def __str__(self): class Singleton(type): + """ + Metaclass that enforces that only one instance can exist of the classes to which it + is applied. + """ + _instances = {} def __call__(cls, *args, **kwargs): @@ -347,6 +381,10 @@ def __call__(cls, *args, **kwargs): def capitalise_first_letter(chars): + """ + Convert the first character of a string to upper case (if that makes sense). + The rest of the string is unchanged. + """ return chars[0].upper() + chars[1:] @@ -374,6 +412,18 @@ def wrap(*args, **kwargs): @TimeIt.decorator def substitute_string_vars(string, variables: Dict[str, str] = None): + """ + Scan ``string`` and substitute sequences like ``<>`` with the value + looked up in the supplied dictionary (with ``ABC`` as the key). + + Default values for the substitution can be supplied like: + ``<>`` + + Examples + -------- + >>> substitute_string_vars("abc <> ghi", {"def": "123"}) + "abc 123 def" + """ variables = variables or {} def var_repl(match_obj): @@ -410,7 +460,7 @@ def var_repl(match_obj): @TimeIt.decorator def read_YAML_str(yaml_str, typ="safe", variables: Dict[str, str] = None): - """Load a YAML string.""" + """Load a YAML string. This will produce basic objects.""" if variables is not False and "< int: def list_to_dict(lst, exclude=None): + """ + Convert a list of dicts to a dict of lists. + """ # TODD: test exclude = exclude or [] dct = {k: [] for k in lst[0].keys() if k not in exclude} @@ -617,7 +689,7 @@ def flatten(lst): """Flatten an arbitrarily (but of uniform depth) nested list and return shape information to enable un-flattening. - Un-flattening can be performed with the `reshape` function. + Un-flattening can be performed with the :py:func:`reshape` function. lst List to be flattened. Each element must contain all lists or otherwise all items @@ -657,6 +729,10 @@ def _get_max_depth(lst): def reshape(lst, lens): + """ + Reverse the destructuring of the :py:func:`flatten` function. + """ + def _reshape(lst, lens): lens_acc = [0] + list(accumulate(lens)) lst_rs = [lst[lens_acc[idx] : lens_acc[idx + 1]] for idx in range(len(lens))] @@ -669,15 +745,30 @@ def _reshape(lst, lens): def is_fsspec_url(url: str) -> bool: + """ + Test if a URL appears to be one that can be understood by fsspec. + """ return bool(re.match(r"(?:[a-z0-9]+:{1,2})+\/\/", url)) class JSONLikeDirSnapShot(DirectorySnapshot): - """Overridden DirectorySnapshot from watchdog to allow saving and loading from JSON.""" + """ + Overridden DirectorySnapshot from watchdog to allow saving and loading from JSON. + """ def __init__(self, root_path=None, data=None): - """Create an empty snapshot or load from JSON-like data.""" + """Create an empty snapshot or load from JSON-like data. + + Parameters + ---------- + root_path: str + Where to take the snapshot based at. + data: dict + Serialised snapshot to reload from. + See :py:meth:`to_json_like`. + """ + #: Where to take the snapshot based at. self.root_path = root_path self._stat_info = {} self._inode_to_path = {} @@ -874,10 +965,16 @@ def dict_values_process_flat(d, callable): def nth_key(dct, n): + """ + Given a dict in some order, get the n'th key of that dict. + """ it = iter(dct) next(islice(it, n, n), None) return next(it) def nth_value(dct, n): + """ + Given a dict in some order, get the n'th value of that dict. + """ return dct[nth_key(dct, n)] diff --git a/hpcflow/sdk/core/validation.py b/hpcflow/sdk/core/validation.py index 939ce6dce..7177f2b61 100644 --- a/hpcflow/sdk/core/validation.py +++ b/hpcflow/sdk/core/validation.py @@ -1,10 +1,22 @@ +""" +Schema management. +""" + from importlib import resources from valida import Schema def get_schema(filename): - """Get a valida `Schema` object from the embedded data directory.""" + """ + Get a valida `Schema` object from the embedded data directory. + + Parameter + --------- + schema: str + The name of the schema file within the resources package + (:py:mod:`hpcflow.sdk.data`). + """ package = "hpcflow.sdk.data" try: fh = resources.files(package).joinpath(filename).open("rt") diff --git a/hpcflow/sdk/core/workflow.py b/hpcflow/sdk/core/workflow.py index b7e0aa7b7..35a45f875 100644 --- a/hpcflow/sdk/core/workflow.py +++ b/hpcflow/sdk/core/workflow.py @@ -1,3 +1,7 @@ +""" +Main workflow model. +""" + from __future__ import annotations from collections import defaultdict from contextlib import contextmanager @@ -90,20 +94,32 @@ class WorkflowTemplate(JSONLike): Parameters ---------- - name + name: A string name for the workflow. By default this name will be used in combination with a date-time stamp when generating a persistent workflow from the template. - tasks + tasks: list[~hpcflow.app.Task] A list of Task objects to include in the workflow. - loops + loops: list[~hpcflow.app.Loop] A list of Loop objects to include in the workflow. - resources + workflow: + The associated concrete workflow. + resources: dict[str, dict] | list[~hpcflow.app.ResourceSpec] | ~hpcflow.app.ResourceList Template-level resources to apply to all tasks as default values. This can be a dict that maps action scopes to resources (e.g. `{{"any": {{"num_cores": 2}}}}`) or a list of `ResourceSpec` objects, or a `ResourceList` object. - merge_resources + environments: + The execution environments to use. + env_presets: + The environment presets to use. + source_file: + The file this was derived from. + store_kwargs: + Additional arguments to pass to the persistent data store constructor. + merge_resources: If True, merge template-level `resources` into element set resources. If False, template-level resources are ignored. + merge_envs: + Whether to merge the environemtns into task resources. """ _app_attr = "app" @@ -129,17 +145,29 @@ class WorkflowTemplate(JSONLike): ), ) + #: A string name for the workflow. name: str + #: Documentation information. doc: Optional[Union[List[str], str]] = field(repr=False, default=None) + #: A list of Task objects to include in the workflow. tasks: Optional[List[app.Task]] = field(default_factory=lambda: []) + #: A list of Loop objects to include in the workflow. loops: Optional[List[app.Loop]] = field(default_factory=lambda: []) + #: The associated concrete workflow. workflow: Optional[app.Workflow] = None + #: Template-level resources to apply to all tasks as default values. resources: Optional[Dict[str, Dict]] = None + #: The execution environments to use. environments: Optional[Dict[str, Dict[str, Any]]] = None + #: The environment presets to use. env_presets: Optional[Union[str, List[str]]] = None + #: The file this was derived from. source_file: Optional[str] = field(default=None, compare=False) + #: Additional arguments to pass to the persistent data store constructor. store_kwargs: Optional[Dict] = field(default_factory=lambda: {}) + #: Whether to merge template-level `resources` into element set resources. merge_resources: Optional[bool] = True + #: Whether to merge the environemtns into task resources. merge_envs: Optional[bool] = True def __post_init__(self): @@ -455,6 +483,8 @@ def _add_empty_loop(self, loop: app.Loop) -> None: def resolve_fsspec(path: PathLike, **kwargs) -> Tuple[Any, str, str]: """ + Decide how to handle a particular virtual path. + Parameters ---------- kwargs @@ -486,6 +516,23 @@ def resolve_fsspec(path: PathLike, **kwargs) -> Tuple[Any, str, str]: class Workflow: + """ + A concrete workflow. + + Parameters + ---------- + workflow_ref: + Either the path to a persistent workflow, or an integer that will interpreted + as the local ID of a workflow submission, as reported by the app `show` + command. + store_fmt: + The format of persistent store to use. Used to select the store manager class. + fs_kwargs: + Additional arguments to pass when resolving a virtual workflow reference. + kwargs: + For compatibility during pre-stable development phase. + """ + _app_attr = "app" _default_ts_fmt = r"%Y-%m-%d %H:%M:%S.%f" _default_ts_name_fmt = r"%Y-%m-%d_%H%M%S" @@ -499,17 +546,6 @@ def __init__( fs_kwargs: Optional[Dict] = None, **kwargs, ): - """ - Parameters - ---------- - workflow_ref - Either the path to a persistent workflow, or an integer that will interpreted - as the local ID of a workflow submission, as reported by the app `show` - command. - kwargs - For compatibility during pre-stable development phase. - """ - if isinstance(workflow_ref, int): path = self.app._get_workflow_path_from_local_ID(workflow_ref) else: @@ -546,15 +582,19 @@ def reload(self): @property def name(self): - """The workflow name may be different from the template name, as it includes the - creation date-timestamp if generated.""" + """ + The name of the workflow. + + The workflow name may be different from the template name, as it includes the + creation date-timestamp if generated. + """ if not self._name: self._name = self._store.get_name() return self._name @property def url(self): - """Get an fsspec URL for this workflow.""" + """An fsspec URL for this workflow.""" if self._store.fs.protocol == "zip": return self._store.fs.of.path elif self._store.fs.protocol == "file": @@ -564,10 +604,16 @@ def url(self): @property def store_format(self): + """ + The format of the workflow's persistent store. + """ return self._store._name @property def num_tasks(self) -> int: + """ + The number of tasks in the workflow. + """ return len(self.tasks) @classmethod @@ -588,28 +634,28 @@ def from_template( Parameters ---------- - template + template: The WorkflowTemplate object to make persistent. - path + path: The directory in which the workflow will be generated. The current directory if not specified. - name + name: The name of the workflow. If specified, the workflow directory will be `path` joined with `name`. If not specified the `WorkflowTemplate` name will be used, in combination with a date-timestamp. - overwrite + overwrite: If True and the workflow directory (`path` + `name`) already exists, the existing directory will be overwritten. - store + store: The persistent store to use for this workflow. - ts_fmt + ts_fmt: The datetime format to use for storing datetimes. Datetimes are always stored in UTC (because Numpy does not store time zone info), so this should not include a time zone name. - ts_name_fmt + ts_name_fmt: The datetime format to use when generating the workflow name, where it includes a timestamp. - store_kwargs + store_kwargs: Keyword arguments to pass to the store's `write_empty_workflow` method. """ if status: @@ -673,30 +719,30 @@ def from_YAML_file( Parameters ---------- - YAML_path + YAML_path: The path to a workflow template in the YAML file format. - path + path: The directory in which the workflow will be generated. The current directory if not specified. - name + name: The name of the workflow. If specified, the workflow directory will be `path` joined with `name`. If not specified the `WorkflowTemplate` name will be used, in combination with a date-timestamp. - overwrite + overwrite: If True and the workflow directory (`path` + `name`) already exists, the existing directory will be overwritten. - store + store: The persistent store to use for this workflow. - ts_fmt + ts_fmt: The datetime format to use for storing datetimes. Datetimes are always stored in UTC (because Numpy does not store time zone info), so this should not include a time zone name. - ts_name_fmt + ts_name_fmt: The datetime format to use when generating the workflow name, where it includes a timestamp. - store_kwargs + store_kwargs: Keyword arguments to pass to the store's `write_empty_workflow` method. - variables + variables: String variables to substitute in the file given by `YAML_path`. """ template = cls.app.WorkflowTemplate.from_YAML_file( @@ -731,30 +777,30 @@ def from_YAML_string( Parameters ---------- - YAML_str + YAML_str: The YAML string containing a workflow template parametrisation. - path + path: The directory in which the workflow will be generated. The current directory if not specified. - name + name: The name of the workflow. If specified, the workflow directory will be `path` joined with `name`. If not specified the `WorkflowTemplate` name will be used, in combination with a date-timestamp. - overwrite + overwrite: If True and the workflow directory (`path` + `name`) already exists, the existing directory will be overwritten. - store + store: The persistent store to use for this workflow. - ts_fmt + ts_fmt: The datetime format to use for storing datetimes. Datetimes are always stored in UTC (because Numpy does not store time zone info), so this should not include a time zone name. - ts_name_fmt + ts_name_fmt: The datetime format to use when generating the workflow name, where it includes a timestamp. - store_kwargs + store_kwargs: Keyword arguments to pass to the store's `write_empty_workflow` method. - variables + variables: String variables to substitute in the string `YAML_str`. """ template = cls.app.WorkflowTemplate.from_YAML_string( @@ -790,30 +836,30 @@ def from_JSON_file( Parameters ---------- - JSON_path + JSON_path: The path to a workflow template in the JSON file format. - path + path: The directory in which the workflow will be generated. The current directory if not specified. - name + name: The name of the workflow. If specified, the workflow directory will be `path` joined with `name`. If not specified the `WorkflowTemplate` name will be used, in combination with a date-timestamp. - overwrite + overwrite: If True and the workflow directory (`path` + `name`) already exists, the existing directory will be overwritten. - store + store: The persistent store to use for this workflow. - ts_fmt + ts_fmt: The datetime format to use for storing datetimes. Datetimes are always stored in UTC (because Numpy does not store time zone info), so this should not include a time zone name. - ts_name_fmt + ts_name_fmt: The datetime format to use when generating the workflow name, where it includes a timestamp. - store_kwargs + store_kwargs: Keyword arguments to pass to the store's `write_empty_workflow` method. - variables + variables: String variables to substitute in the file given by `JSON_path`. """ template = cls.app.WorkflowTemplate.from_JSON_file( @@ -850,30 +896,30 @@ def from_JSON_string( Parameters ---------- - JSON_str + JSON_str: The JSON string containing a workflow template parametrisation. - path + path: The directory in which the workflow will be generated. The current directory if not specified. - name + name: The name of the workflow. If specified, the workflow directory will be `path` joined with `name`. If not specified the `WorkflowTemplate` name will be used, in combination with a date-timestamp. - overwrite + overwrite: If True and the workflow directory (`path` + `name`) already exists, the existing directory will be overwritten. - store + store: The persistent store to use for this workflow. - ts_fmt + ts_fmt: The datetime format to use for storing datetimes. Datetimes are always stored in UTC (because Numpy does not store time zone info), so this should not include a time zone name. - ts_name_fmt + ts_name_fmt: The datetime format to use when generating the workflow name, where it includes a timestamp. - store_kwargs + store_kwargs: Keyword arguments to pass to the store's `write_empty_workflow` method. - variables + variables: String variables to substitute in the string `JSON_str`. """ template = cls.app.WorkflowTemplate.from_JSON_string( @@ -912,34 +958,34 @@ def from_file( Parameters ---------- - template_path + template_path: The path to a template file in YAML or JSON format, and with a ".yml", ".yaml", or ".json" extension. - template_format + template_format: If specified, one of "json" or "yaml". This forces parsing from a particular format regardless of the file extension. - path + path: The directory in which the workflow will be generated. The current directory if not specified. - name + name: The name of the workflow. If specified, the workflow directory will be `path` joined with `name`. If not specified the `WorkflowTemplate` name will be used, in combination with a date-timestamp. - overwrite + overwrite: If True and the workflow directory (`path` + `name`) already exists, the existing directory will be overwritten. - store + store: The persistent store to use for this workflow. - ts_fmt + ts_fmt: The datetime format to use for storing datetimes. Datetimes are always stored in UTC (because Numpy does not store time zone info), so this should not include a time zone name. - ts_name_fmt + ts_name_fmt: The datetime format to use when generating the workflow name, where it includes a timestamp. - store_kwargs + store_kwargs: Keyword arguments to pass to the store's `write_empty_workflow` method. - variables + variables: String variables to substitute in the file given by `template_path`. """ try: @@ -984,37 +1030,37 @@ def from_template_data( Parameters ---------- - template_name + template_name: Name of the new workflow template, from which the new workflow will be generated. - tasks + tasks: List of Task objects to add to the new workflow. - loops + loops: List of Loop objects to add to the new workflow. - resources + resources: Mapping of action scopes to resource requirements, to be applied to all element sets in the workflow. `resources` specified in an element set take precedence of those defined here for the whole workflow. - path + path: The directory in which the workflow will be generated. The current directory if not specified. - workflow_name + workflow_name: The name of the workflow. If specified, the workflow directory will be `path` joined with `name`. If not specified `template_name` will be used, in combination with a date-timestamp. - overwrite + overwrite: If True and the workflow directory (`path` + `name`) already exists, the existing directory will be overwritten. - store + store: The persistent store to use for this workflow. - ts_fmt + ts_fmt: The datetime format to use for storing datetimes. Datetimes are always stored in UTC (because Numpy does not store time zone info), so this should not include a time zone name. - ts_name_fmt + ts_name_fmt: The datetime format to use when generating the workflow name, where it includes a timestamp. - store_kwargs + store_kwargs: Keyword arguments to pass to the store's `write_empty_workflow` method. """ template = cls.app.WorkflowTemplate( @@ -1080,6 +1126,9 @@ def _add_task(self, task: app.Task, new_index: Optional[int] = None) -> None: new_wk_task._add_elements(element_sets=task.element_sets) def add_task(self, task: app.Task, new_index: Optional[int] = None) -> None: + """ + Add a task to this workflow. + """ with self._store.cached_load(): with self.batch_update(): self._add_task(task, new_index=new_index) @@ -1186,6 +1235,9 @@ def add_loop(self, loop: app.Loop) -> None: @property def creation_info(self): + """ + The creation descriptor for the workflow. + """ if not self._creation_info: info = self._store.get_creation_info() info["create_time"] = ( @@ -1198,22 +1250,34 @@ def creation_info(self): @property def id_(self): + """ + The ID of this workflow. + """ return self.creation_info["id"] @property def ts_fmt(self): + """ + The timestamp format. + """ if not self._ts_fmt: self._ts_fmt = self._store.get_ts_fmt() return self._ts_fmt @property def ts_name_fmt(self): + """ + The timestamp format for names. + """ if not self._ts_name_fmt: self._ts_name_fmt = self._store.get_ts_name_fmt() return self._ts_name_fmt @property def template_components(self) -> Dict: + """ + The template components used for this workflow. + """ if self._template_components is None: with self._store.cached_load(): tc_js = self._store.get_template_components() @@ -1222,6 +1286,9 @@ def template_components(self) -> Dict: @property def template(self) -> app.WorkflowTemplate: + """ + The template that this workflow was made from. + """ if self._template is None: with self._store.cached_load(): temp_js = self._store.get_template() @@ -1240,6 +1307,9 @@ def template(self) -> app.WorkflowTemplate: @property def tasks(self) -> app.WorkflowTaskList: + """ + The tasks in this workflow. + """ if self._tasks is None: with self._store.cached_load(): all_tasks = self._store.get_tasks() @@ -1258,6 +1328,9 @@ def tasks(self) -> app.WorkflowTaskList: @property def loops(self) -> app.WorkflowLoopList: + """ + The loops in this workflow. + """ if self._loops is None: with self._store.cached_load(): wk_loops = [] @@ -1279,6 +1352,9 @@ def loops(self) -> app.WorkflowLoopList: @property def submissions(self) -> List[app.Submission]: + """ + The job submissions done by this workflow. + """ if self._submissions is None: self.app.persistence_logger.debug("loading workflow submissions") with self._store.cached_load(): @@ -1293,42 +1369,66 @@ def submissions(self) -> List[app.Submission]: @property def num_added_tasks(self) -> int: + """ + The total number of added tasks. + """ return self._store._get_num_total_added_tasks() @TimeIt.decorator def get_store_EARs(self, id_lst: Iterable[int]) -> List[AnySEAR]: + """ + Get the persistent element action runs. + """ return self._store.get_EARs(id_lst) @TimeIt.decorator def get_store_element_iterations( self, id_lst: Iterable[int] ) -> List[AnySElementIter]: + """ + Get the persistent element iterations. + """ return self._store.get_element_iterations(id_lst) @TimeIt.decorator def get_store_elements(self, id_lst: Iterable[int]) -> List[AnySElement]: + """ + Get the persistent elements. + """ return self._store.get_elements(id_lst) @TimeIt.decorator def get_store_tasks(self, id_lst: Iterable[int]) -> List[AnySTask]: + """ + Get the persistent tasks. + """ return self._store.get_tasks_by_IDs(id_lst) def get_element_iteration_IDs_from_EAR_IDs(self, id_lst: Iterable[int]) -> List[int]: + """ + Get the element iteration IDs of EARs. + """ return [i.elem_iter_ID for i in self.get_store_EARs(id_lst)] def get_element_IDs_from_EAR_IDs(self, id_lst: Iterable[int]) -> List[int]: + """ + Get the element IDs of EARs. + """ iter_IDs = self.get_element_iteration_IDs_from_EAR_IDs(id_lst) return [i.element_ID for i in self.get_store_element_iterations(iter_IDs)] def get_task_IDs_from_element_IDs(self, id_lst: Iterable[int]) -> List[int]: + """ + Get the task IDs of elements. + """ return [i.task_ID for i in self.get_store_elements(id_lst)] def get_EAR_IDs_of_tasks(self, id_lst: int) -> List[int]: - """Get EAR IDs belonging to multiple tasks""" + """Get EAR IDs belonging to multiple tasks.""" return [i.id_ for i in self.get_EARs_of_tasks(id_lst)] def get_EARs_of_tasks(self, id_lst: Iterable[int]) -> List[app.ElementActionRun]: - """Get EARs belonging to multiple tasks""" + """Get EARs belonging to multiple task.s""" EARs = [] for i in id_lst: task = self.tasks.get(insert_ID=i) @@ -1341,7 +1441,7 @@ def get_EARs_of_tasks(self, id_lst: Iterable[int]) -> List[app.ElementActionRun] def get_element_iterations_of_tasks( self, id_lst: Iterable[int] ) -> List[app.ElementIteration]: - """Get element iterations belonging to multiple tasks""" + """Get element iterations belonging to multiple tasks.""" iters = [] for i in id_lst: task = self.tasks.get(insert_ID=i) @@ -1431,7 +1531,7 @@ def get_element_iterations_from_IDs( @TimeIt.decorator def get_EARs_from_IDs(self, id_lst: Iterable[int]) -> List[app.ElementActionRun]: - """Return element action run objects from a list of IDs.""" + """Get element action run objects from a list of IDs.""" self.app.persistence_logger.debug(f"get_EARs_from_IDs: id_lst={id_lst!r}") store_EARs = self._store.get_EARs(id_lst) @@ -1490,14 +1590,23 @@ def get_EARs_from_IDs(self, id_lst: Iterable[int]) -> List[app.ElementActionRun] @TimeIt.decorator def get_all_elements(self) -> List[app.Element]: + """ + Get all elements in the workflow. + """ return self.get_elements_from_IDs(range(self.num_elements)) @TimeIt.decorator def get_all_element_iterations(self) -> List[app.ElementIteration]: + """ + Get all iterations in the workflow. + """ return self.get_element_iterations_from_IDs(range(self.num_element_iterations)) @TimeIt.decorator def get_all_EARs(self) -> List[app.ElementActionRun]: + """ + Get all runs in the workflow. + """ return self.get_EARs_from_IDs(range(self.num_EARs)) @contextmanager @@ -1705,6 +1814,8 @@ def zip( include_rechunk_backups=False, ) -> str: """ + Convert the workflow to a zipped form. + Parameters ---------- path: @@ -1722,6 +1833,8 @@ def zip( def unzip(self, path=".", log=None) -> str: """ + Convert the workflow to an unzipped form. + Parameters ---------- path: @@ -1736,6 +1849,9 @@ def copy(self, path=None) -> str: return self._store.copy(path) def delete(self): + """ + Delete the persistent data. + """ self._store.delete() def _delete_no_confirm(self): @@ -1744,22 +1860,37 @@ def _delete_no_confirm(self): def get_parameters( self, id_lst: Iterable[int], **kwargs: Dict ) -> List[AnySParameter]: + """ + Get parameters known to the workflow. + """ return self._store.get_parameters(id_lst, **kwargs) @TimeIt.decorator def get_parameter_sources(self, id_lst: Iterable[int]) -> List[Dict]: + """ + Get parameter sources known to the workflow. + """ return self._store.get_parameter_sources(id_lst) @TimeIt.decorator def get_parameter_set_statuses(self, id_lst: Iterable[int]) -> List[bool]: + """ + Get whether some parameters are set. + """ return self._store.get_parameter_set_statuses(id_lst) @TimeIt.decorator def get_parameter(self, index: int, **kwargs: Dict) -> AnySParameter: + """ + Get a single parameter. + """ return self.get_parameters([index], **kwargs)[0] @TimeIt.decorator def get_parameter_data(self, index: int, **kwargs: Dict) -> Any: + """ + Get the data relating to a parameter. + """ param = self.get_parameter(index, **kwargs) if param.data is not None: return param.data @@ -1768,22 +1899,28 @@ def get_parameter_data(self, index: int, **kwargs: Dict) -> Any: @TimeIt.decorator def get_parameter_source(self, index: int) -> Dict: + """ + Get the source of a particular parameter. + """ return self.get_parameter_sources([index])[0] @TimeIt.decorator def is_parameter_set(self, index: int) -> bool: + """ + Test if a particular parameter is set. + """ return self.get_parameter_set_statuses([index])[0] @TimeIt.decorator def get_all_parameters(self, **kwargs: Dict) -> List[AnySParameter]: - """Retrieve all store parameters.""" + """Retrieve all persistent parameters.""" num_params = self._store._get_num_total_parameters() id_lst = list(range(num_params)) return self._store.get_parameters(id_lst, **kwargs) @TimeIt.decorator def get_all_parameter_sources(self, **kwargs: Dict) -> List[Dict]: - """Retrieve all store parameters.""" + """Retrieve all persistent parameters sources.""" num_params = self._store._get_num_total_parameters() id_lst = list(range(num_params)) return self._store.get_parameter_sources(id_lst, **kwargs) @@ -1797,6 +1934,9 @@ def get_all_parameter_data(self, **kwargs: Dict) -> Dict[int, Any]: def check_parameters_exist( self, id_lst: Union[int, List[int]] ) -> Union[bool, List[bool]]: + """ + Check if parameters exist. + """ is_multi = True if isinstance(id_lst, int): is_multi = False @@ -1858,7 +1998,7 @@ def get_task_unique_names( Parameters ---------- - map_to_insert_ID : bool, optional + map_to_insert_ID : bool If True, return a dict whose values are task insert IDs, otherwise return a list. @@ -1922,48 +2062,81 @@ def _reject_pending(self) -> None: @property def num_tasks(self): + """ + The total number of tasks. + """ return self._store._get_num_total_tasks() @property def num_submissions(self): + """ + The total number of job submissions. + """ return self._store._get_num_total_submissions() @property def num_elements(self): + """ + The total number of elements. + """ return self._store._get_num_total_elements() @property def num_element_iterations(self): + """ + The total number of element iterations. + """ return self._store._get_num_total_elem_iters() @property @TimeIt.decorator def num_EARs(self): + """ + The total number of element action runs. + """ return self._store._get_num_total_EARs() @property def num_loops(self) -> int: + """ + The total number of loops. + """ return self._store._get_num_total_loops() @property def artifacts_path(self): + """ + Path to artifacts of the workflow (temporary files, etc). + """ # TODO: allow customisation of artifacts path at submission and resources level return Path(self.path) / "artifacts" @property def input_files_path(self): + """ + Path to input files for the workflow. + """ return self.artifacts_path / self._input_files_dir_name @property def submissions_path(self): + """ + Path to submission data for ths workflow. + """ return self.artifacts_path / "submissions" @property def task_artifacts_path(self): + """ + Path to artifacts of tasks. + """ return self.artifacts_path / "tasks" @property def execution_path(self): + """ + Path to working directory path for executing. + """ return Path(self.path) / self._exec_dir_name @TimeIt.decorator @@ -1972,6 +2145,9 @@ def get_task_elements( task: app.Task, idx_lst: Optional[List[int]] = None, ) -> List[app.Element]: + """ + Get the elements of a task. + """ return [ self.app.Element(task=task, **{k: v for k, v in i.items() if k != "task_ID"}) for i in self._store.get_task_elements(task.insert_ID, idx_lst) @@ -2107,8 +2283,10 @@ def set_EAR_end( self._store.set_EAR_end(EAR_ID, exit_code, success) def set_EAR_skip(self, EAR_ID: int) -> None: - """Record that an EAR is to be skipped due to an upstream failure or loop - termination condition being met.""" + """ + Record that an EAR is to be skipped due to an upstream failure or loop + termination condition being met. + """ with self._store.cached_load(): with self.batch_update(): self._store.set_EAR_skip(EAR_ID) @@ -2122,6 +2300,9 @@ def get_EAR_skipped(self, EAR_ID: int) -> None: def set_parameter_value( self, param_id: int, value: Any, commit: bool = False ) -> None: + """ + Set the value of a parameter. + """ with self._store.cached_load(): with self.batch_update(): self._store.set_parameter_value(param_id, value) @@ -2131,18 +2312,28 @@ def set_parameter_value( self._store._pending.commit_all() def set_EARs_initialised(self, iter_ID: int): - """Set `ElementIteration.EARs_initialised` to True for the specified iteration.""" + """ + Set :py:attr:`~hpcflow.app.ElementIteration.EARs_initialised` to True for the + specified iteration. + """ with self._store.cached_load(): with self.batch_update(): self._store.set_EARs_initialised(iter_ID) def elements(self) -> Iterator[app.Element]: + """ + Get the elements of the workflow's tasks. + """ for task in self.tasks: for element in task.elements[:]: yield element @TimeIt.decorator def get_iteration_task_pathway(self, ret_iter_IDs=False, ret_data_idx=False): + """ + Get the iteration task pathway. + """ + # FIXME: I don't understand this concept, alas. pathway = [] for task in self.tasks: pathway.append((task.insert_ID, {})) @@ -2574,6 +2765,9 @@ def cancel(self, hard=False): def add_submission( self, tasks: Optional[List[int]] = None, JS_parallelism: Optional[bool] = None ) -> app.Submission: + """ + Add a job submission to this workflow. + """ with self._store.cached_load(): with self.batch_update(): return self._add_submission(tasks, JS_parallelism) @@ -2617,6 +2811,9 @@ def _add_submission( def resolve_jobscripts( self, tasks: Optional[List[int]] = None ) -> List[app.Jobscript]: + """ + Resolve this workflow to a set of job scripts to run. + """ js, element_deps = self._resolve_singular_jobscripts(tasks) js_deps = resolve_jobscript_dependencies(js, element_deps) @@ -2824,6 +3021,9 @@ def save_parameter( value: Any, EAR_ID: int, ): + """ + Save a parameter where an EAR can find it. + """ self.app.logger.info(f"save parameter {name!r} for EAR_ID {EAR_ID}.") self.app.logger.debug(f"save parameter {name!r} value is {value!r}.") with self._store.cached_load(): @@ -2833,6 +3033,10 @@ def save_parameter( self.set_parameter_value(param_id, value) def show_all_EAR_statuses(self): + """ + Print a description of the status of every element action run in + the workflow. + """ print( f"{'task':8s} {'element':8s} {'iteration':8s} {'action':8s} " f"{'run':8s} {'sub.':8s} {'exitcode':8s} {'success':8s} {'skip':8s}" @@ -2893,6 +3097,9 @@ def _resolve_input_source_task_reference( ) def get_all_submission_run_IDs(self) -> List[int]: + """ + Get the run IDs of all submissions. + """ self.app.persistence_logger.debug("Workflow.get_all_submission_run_IDs") id_lst = [] for sub in self.submissions: @@ -2918,6 +3125,9 @@ def check_loop_termination(self, loop_name: str, run_ID: int) -> bool: self.set_EAR_skip(run_ID) def get_loop_map(self, id_lst: Optional[List[int]] = None): + """ + Get a description of what is going on with looping. + """ # TODO: test this works across multiple jobscripts self.app.persistence_logger.debug("Workflow.get_loop_map") if id_lst is None: @@ -2962,6 +3172,9 @@ def rechunk_runs( backup: Optional[bool] = True, status: Optional[bool] = True, ): + """ + Reorganise the stored data chunks for EARs to be more efficient. + """ self._store.rechunk_runs(chunk_size=chunk_size, backup=backup, status=status) def rechunk_parameter_base( @@ -2970,6 +3183,9 @@ def rechunk_parameter_base( backup: Optional[bool] = True, status: Optional[bool] = True, ): + """ + Reorganise the stored data chunks for parameterss to be more efficient. + """ self._store.rechunk_parameter_base( chunk_size=chunk_size, backup=backup, status=status ) @@ -2980,13 +3196,17 @@ def rechunk( backup: Optional[bool] = True, status: Optional[bool] = True, ): - """Rechunk metadata/runs and parameters/base arrays.""" + """ + Rechunk metadata/runs and parameters/base arrays, making them more efficient. + """ self.rechunk_runs(chunk_size=chunk_size, backup=backup, status=status) self.rechunk_parameter_base(chunk_size=chunk_size, backup=backup, status=status) @dataclass class WorkflowBlueprint: - """Pre-built workflow templates that are simpler to parametrise (e.g. fitting workflows).""" + """Pre-built workflow templates that are simpler to parameterise. + (For example, fitting workflows.)""" + #: The template inside this blueprint. workflow_template: WorkflowTemplate diff --git a/hpcflow/sdk/core/zarr_io.py b/hpcflow/sdk/core/zarr_io.py index bb4d20278..e364ff685 100644 --- a/hpcflow/sdk/core/zarr_io.py +++ b/hpcflow/sdk/core/zarr_io.py @@ -1,3 +1,7 @@ +""" +Utilities for working with Zarr. +""" + from typing import Any, Dict, Union import zarr @@ -68,6 +72,9 @@ def _zarr_encode(obj, zarr_group, path=None, encoded=None): def zarr_encode(data, zarr_group, is_pending_add, is_set): + """ + Encode data into a zarr group. + """ data, encoded = _zarr_encode(data, zarr_group) zarr_group.attrs["encoded"] = encoded zarr_group.attrs["data"] = data @@ -176,6 +183,9 @@ def zarr_decode( path=None, dataset_copy=False, ): + """ + Decode data from a zarr group. + """ if param_data is None: return None @@ -204,19 +214,32 @@ def zarr_decode( class ZarrEncodable: + """ + Base class of data that can be converted to and from zarr form. + """ + _typ = None def to_dict(self): + """ + Convert this object to a dict. + """ if hasattr(self, "__dict__"): return dict(self.__dict__) elif hasattr(self, "__slots__"): return {k: getattr(self, k) for k in self.__slots__} def to_zarr(self, zarr_group): + """ + Save this object into the given zarr group. + """ data = self.to_dict() zarr_encode(data, zarr_group) @classmethod def from_zarr(cls, zarr_group, dataset_copy=False): + """ + Read an instance of this class from the given zarr group. + """ data = zarr_decode(zarr_group, dataset_copy=dataset_copy) return cls(**data) diff --git a/hpcflow/sdk/data/__init__.py b/hpcflow/sdk/data/__init__.py index e69de29bb..4e0594ef8 100644 --- a/hpcflow/sdk/data/__init__.py +++ b/hpcflow/sdk/data/__init__.py @@ -0,0 +1,13 @@ +""" +YAML schemas. + +Contents: + +* ``config_file_schema.yaml`` – Schema for configuration selection files. +* ``config_schema.yaml`` – Schema for configuration files. +* ``environments_spec_schema.yaml`` – Schema for execution environment definition files. +* ``files_spec_schema.yaml`` – Schema for input/output specification files. +* ``parameters_spec_schema.yaml`` – Schema for parameter specification files. +* ``task_schema_spec_schema.yaml`` – Schema for task template specification files. +* ``workflow_spec_schema.yaml`` – Schema for workflow files. +""" diff --git a/hpcflow/sdk/demo/__init__.py b/hpcflow/sdk/demo/__init__.py index e69de29bb..f715c0e0d 100644 --- a/hpcflow/sdk/demo/__init__.py +++ b/hpcflow/sdk/demo/__init__.py @@ -0,0 +1,3 @@ +""" +Demonstration code. +""" diff --git a/hpcflow/sdk/helper/__init__.py b/hpcflow/sdk/helper/__init__.py index e69de29bb..40fc2857f 100644 --- a/hpcflow/sdk/helper/__init__.py +++ b/hpcflow/sdk/helper/__init__.py @@ -0,0 +1,3 @@ +""" +Helpers for the CLI. +""" diff --git a/hpcflow/sdk/helper/cli.py b/hpcflow/sdk/helper/cli.py index 177971467..53efbf206 100644 --- a/hpcflow/sdk/helper/cli.py +++ b/hpcflow/sdk/helper/cli.py @@ -1,3 +1,7 @@ +""" +Common Click command line options related to the helper. +""" + from datetime import timedelta import click @@ -18,7 +22,9 @@ get_helper_PID, get_helper_uptime, ) +from ..cli_common import _add_doc_from_help +#: Helper option: ``--timeout`` timeout_option = click.option( "--timeout", type=click.FLOAT, @@ -26,6 +32,7 @@ show_default=True, help="Helper timeout in seconds.", ) +#: Helper option: ``--timeout-check-interval`` timeout_check_interval_option = click.option( "--timeout-check-interval", type=click.FLOAT, @@ -33,6 +40,7 @@ show_default=True, help="Interval between testing if the timeout has been exceeded in seconds.", ) +#: Helper option: ``--watch interval`` watch_interval_option = click.option( "--watch-interval", type=click.FLOAT, @@ -43,6 +51,7 @@ "seconds." ), ) +_add_doc_from_help(timeout_option, timeout_check_interval_option, watch_interval_option) def get_helper_CLI(app): diff --git a/hpcflow/sdk/helper/helper.py b/hpcflow/sdk/helper/helper.py index 359875256..0b257151c 100644 --- a/hpcflow/sdk/helper/helper.py +++ b/hpcflow/sdk/helper/helper.py @@ -1,3 +1,7 @@ +""" +Implementation of a helper process used to monitor jobs. +""" + from datetime import datetime, timedelta import logging from logging.handlers import RotatingFileHandler @@ -71,6 +75,9 @@ def start_helper( watch_interval=DEFAULT_WATCH_INTERVAL, logger=None, ): + """ + Start the helper process. + """ PID_file = get_PID_file_path(app) if PID_file.is_file(): with PID_file.open("rt") as fp: @@ -135,11 +142,17 @@ def restart_helper( timeout_check_interval=DEFAULT_TIMEOUT_CHECK, watch_interval=DEFAULT_WATCH_INTERVAL, ): + """ + Restart the helper process. + """ logger = stop_helper(app, return_logger=True) start_helper(app, timeout, timeout_check_interval, watch_interval, logger=logger) def get_helper_PID(app): + """ + Get the process ID of the helper process. + """ PID_file = get_PID_file_path(app) if not PID_file.is_file(): print("Helper not running!") @@ -151,6 +164,9 @@ def get_helper_PID(app): def stop_helper(app, return_logger=False): + """ + Stop the helper process. + """ logger = get_helper_logger(app) pid_info = get_helper_PID(app) if pid_info: @@ -168,6 +184,9 @@ def stop_helper(app, return_logger=False): def clear_helper(app): + """ + Stop the helper or remove any stale information relating to it. + """ try: stop_helper(app) except psutil.NoSuchProcess: @@ -179,6 +198,9 @@ def clear_helper(app): def get_helper_uptime(app): + """ + Get the amount of time that the helper has been running. + """ pid_info = get_helper_PID(app) if pid_info: proc = psutil.Process(pid_info[0]) @@ -188,6 +210,9 @@ def get_helper_uptime(app): def get_helper_logger(app): + """ + Get the logger for helper-related messages. + """ log_path = get_helper_log_path(app) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -225,6 +250,9 @@ def run_helper( timeout_check_interval=DEFAULT_TIMEOUT_CHECK, watch_interval=DEFAULT_WATCH_INTERVAL, ): + """ + Run the helper core. + """ # TODO: when writing to watch_workflows from a workflow, copy, modify and then rename # this will be atomic - so there will be only one event fired. # Also return a local run ID (the position in the file) to be used in jobscript naming diff --git a/hpcflow/sdk/helper/watcher.py b/hpcflow/sdk/helper/watcher.py index 89accc61b..1568a6892 100644 --- a/hpcflow/sdk/helper/watcher.py +++ b/hpcflow/sdk/helper/watcher.py @@ -1,3 +1,7 @@ +""" +File-system watcher classes. +""" + from datetime import timedelta from pathlib import Path from watchdog.observers.polling import PollingObserver @@ -5,6 +9,10 @@ class MonitorController: + """ + Controller for tracking watch files. + """ + def __init__(self, workflow_dirs_file_path, watch_interval, logger): if isinstance(watch_interval, timedelta): @@ -46,6 +54,9 @@ def __init__(self, workflow_dirs_file_path, watch_interval, logger): @staticmethod def parse_watch_workflows_file(path, logger): + """ + Parse the file describing what workflows to watch. + """ # TODO: and parse element IDs as well; and record which are set/unset. with Path(path).open("rt") as fp: lns = fp.readlines() @@ -69,20 +80,33 @@ def parse_watch_workflows_file(path, logger): return wks def on_modified(self, event): + """ + Callback when files are modified. + """ self.logger.info(f"Watch file modified: {event.src_path}") wks = self.parse_watch_workflows_file(event.src_path, logger=self.logger) self.workflow_monitor.update_workflow_paths(wks) def join(self): + """ + Join the worker thread. + """ self.observer.join() def stop(self): + """ + Stop this monitor. + """ self.observer.stop() self.observer.join() # wait for it to stop! self.workflow_monitor.stop() class WorkflowMonitor: + """ + Workflow monitor. + """ + def __init__(self, workflow_paths, watch_interval, logger): if isinstance(watch_interval, timedelta): @@ -106,15 +130,24 @@ def _monitor_workflow_paths(self): self.observer.start() def on_modified(self, event): + """ + Triggered on a workflow being modified. + """ self.logger.info(f"Workflow modified: {event.src_path}") def update_workflow_paths(self, new_paths): + """ + Change the set of paths to monitored workflows. + """ self.logger.info(f"Updating watched workflows.") self.stop() self.workflow_paths = new_paths self._monitor_workflow_paths() def stop(self): + """ + Stop this monitor. + """ if self.observer: self.observer.stop() self.observer.join() # wait for it to stop! diff --git a/hpcflow/sdk/log.py b/hpcflow/sdk/log.py index a5b03c7ad..4f54fe18b 100644 --- a/hpcflow/sdk/log.py +++ b/hpcflow/sdk/log.py @@ -1,3 +1,7 @@ +""" +Interface to the standard logger, and performance logging utility. +""" + from functools import wraps import logging from pathlib import Path @@ -7,17 +11,31 @@ class TimeIt: + """ + Method execution time instrumentation. + """ + #: Whether the instrumentation is active. active = False + #: Where to log to. file_path = None + #: The details be tracked. timers = defaultdict(list) + #: Traces of the stack. trace = [] + #: Trace indices. trace_idx = [] + #: Preceding traces. trace_prev = [] + #: Preceding trace indices. trace_idx_prev = [] @classmethod def decorator(cls, func): + """ + Decorator for a method that is to have its execution time monitored. + """ + @wraps(func) def wrapper(*args, **kwargs): @@ -51,6 +69,9 @@ def wrapper(*args, **kwargs): @classmethod def summarise(cls): + """ + Produce a machine-readable summary of method execution time statistics. + """ stats = { k: { "number": len(v), @@ -81,6 +102,10 @@ def summarise(cls): @classmethod def summarise_string(cls): + """ + Produce a human-readable summary of method execution time statistics. + """ + def _format_nodes(node, depth=0, depth_final=None): if depth_final is None: depth_final = [] @@ -126,13 +151,22 @@ def _format_nodes(node, depth=0, depth_final=None): class AppLog: + """ + Application log control. + """ + + #: Default logging level for the console. DEFAULT_LOG_CONSOLE_LEVEL = "WARNING" + #: Default logging level for log files. DEFAULT_LOG_FILE_LEVEL = "INFO" def __init__(self, app, log_console_level=None): + #: The application context. self.app = app + #: The base logger for the application. self.logger = logging.getLogger(app.package_name) self.logger.setLevel(logging.DEBUG) + #: The handler for directing logging messages to the console. self.console_handler = self._add_console_logger( level=log_console_level or AppLog.DEFAULT_LOG_CONSOLE_LEVEL ) @@ -146,10 +180,16 @@ def _add_console_logger(self, level, fmt=None): return handler def update_console_level(self, new_level): + """ + Set the logging level for console messages. + """ if new_level: self.console_handler.setLevel(new_level.upper()) def add_file_logger(self, path, level=None, fmt=None, max_bytes=None): + """ + Add a log file. + """ fmt = fmt or f"%(asctime)s %(levelname)s %(name)s: %(message)s" level = level or AppLog.DEFAULT_LOG_FILE_LEVEL max_bytes = max_bytes or int(10e6) diff --git a/hpcflow/sdk/persistence/__init__.py b/hpcflow/sdk/persistence/__init__.py index 4733957b7..ed658110b 100644 --- a/hpcflow/sdk/persistence/__init__.py +++ b/hpcflow/sdk/persistence/__init__.py @@ -1,3 +1,7 @@ +""" +Workflow persistence subsystem. +""" + import copy from pathlib import Path import random @@ -11,21 +15,27 @@ from hpcflow.sdk.persistence.json import JSONPersistentStore from hpcflow.sdk.persistence.zarr import ZarrPersistentStore, ZarrZipPersistentStore -ALL_STORE_CLS = { +_ALL_STORE_CLS = { "zarr": ZarrPersistentStore, "zip": ZarrZipPersistentStore, "json": JSONPersistentStore, # "json-single": JSONPersistentStore, # TODO } +#: The name of the default persistence store. DEFAULT_STORE_FORMAT = "zarr" -ALL_STORE_FORMATS = tuple(ALL_STORE_CLS.keys()) +#: The persistence formats supported. +ALL_STORE_FORMATS = tuple(_ALL_STORE_CLS.keys()) +#: The persistence formats supported for creation. ALL_CREATE_STORE_FORMATS = tuple( - k for k, v in ALL_STORE_CLS.items() if v._features.create + k for k, v in _ALL_STORE_CLS.items() if v._features.create ) def store_cls_from_str(store_format: str) -> Type[PersistentStore]: + """ + Get the class that implements the persistence store from its name. + """ try: - return ALL_STORE_CLS[store_format] + return _ALL_STORE_CLS[store_format] except KeyError: raise ValueError(f"Store format {store_format!r} not known.") diff --git a/hpcflow/sdk/persistence/base.py b/hpcflow/sdk/persistence/base.py index a5aee2f16..1a6d6793e 100644 --- a/hpcflow/sdk/persistence/base.py +++ b/hpcflow/sdk/persistence/base.py @@ -1,4 +1,8 @@ -# Store* classes represent the element-metadata in the store, in a store-agnostic way +""" +Base persistence models. + +Store* classes represent the element-metadata in the store, in a store-agnostic way. +""" from __future__ import annotations from abc import ABC @@ -50,46 +54,81 @@ def update_param_source_dict(source, update): + """ + Combine two dicts into a new dict that is ordered on its keys. + """ return dict(sorted({**source, **update}.items())) @dataclass class PersistentStoreFeatures: - """Class to represent the features provided by a persistent store. + """ + Represents the features provided by a persistent store. Parameters ---------- - create + create: If True, a new workflow can be created using this store. - edit + edit: If True, the workflow can be modified. - jobscript_parallelism + jobscript_parallelism: If True, the store supports workflows running multiple independent jobscripts simultaneously. - EAR_parallelism + EAR_parallelism: If True, the store supports workflows running multiple EARs simultaneously. - schedulers - If True, the store supports submitting workflows to a scheduler - submission + schedulers: + If True, the store supports submitting workflows to a scheduler. + submission: If True, the store supports submission. If False, the store can be considered to be an archive, which would need transforming to another store type before submission. """ + #: Whether a new workflow can be created using this store. create: bool = False + #: Whether the workflow can be modified. edit: bool = False + #: Whetherthe store supports workflows running multiple independent jobscripts + #: simultaneously. jobscript_parallelism: bool = False + #: Whether the store supports workflows running multiple EARs simultaneously. EAR_parallelism: bool = False + #: Whether the store supports submitting workflows to a scheduler. schedulers: bool = False + #: Whether the store supports submission. If not, the store can be considered to + #: be an archive, which would need transforming to another store type before + #: submission. submission: bool = False @dataclass class StoreTask: + """ + Represents a task in a persistent store. + + Parameters + ---------- + id_: + The ID of the task. + index: + The index of the task within its workflow. + is_pending: + Whether the task has changes not yet persisted. + element_IDs: + The IDs of elements in the task. + task_template: + Description of the template for the task. + """ + + #: The ID of the task. id_: int + #: The index of the task within its workflow. index: int + #: Whether the task has changes not yet persisted. is_pending: bool + #: The IDs of elements in the task. element_IDs: List[int] + #: Description of the template for the task. task_template: Optional[Dict] = None def encode(self) -> Tuple[int, Dict, Dict]: @@ -124,21 +163,43 @@ def append_element_IDs(self: AnySTask, pend_IDs: List[int]) -> AnySTask: @dataclass class StoreElement: """ + Represents an element in a persistent store. + Parameters ---------- - index + id_: + The ID of the element. + is_pending: + Whether the element has changes not yet persisted. + index: Index of the element within its parent task. - iteration_IDs + es_idx: + Index of the element set containing this element. + seq_idx: + Value sequence index map. + src_idx: + Data source index map. + task_ID: + ID of the task that contains this element. + iteration_IDs: IDs of element-iterations that belong to this element. """ + #: The ID of the element. id_: int + #: Whether the element has changes not yet persisted. is_pending: bool + #: Index of the element within its parent task. index: int + #: Index of the element set containing this element. es_idx: int + #: Value sequence index map. seq_idx: Dict[str, int] + #: Data source index map. src_idx: Dict[str, int] + #: ID of the task that contains this element. task_ID: int + #: IDs of element-iterations that belong to this element. iteration_IDs: List[int] def encode(self) -> Dict: @@ -185,24 +246,45 @@ def append_iteration_IDs(self: AnySElement, pend_IDs: List[int]) -> AnySElement: @dataclass class StoreElementIter: """ + Represents an element iteration in a persistent store. + Parameters ---------- - data_idx + id_: + The ID of this element iteration. + is_pending: + Whether the element iteration has changes not yet persisted. + element_ID: + Which element is an iteration for. + EARs_initialised: + Whether EARs have been initialised for this element iteration. + EAR_IDs: + Maps task schema action indices to EARs by ID. + data_idx: Overall data index for the element-iteration, which maps parameter names to parameter data indices. - EAR_IDs - Maps task schema action indices to EARs by ID. - schema_parameters + schema_parameters: List of parameters defined by the associated task schema. + loop_idx: + What loops are being handled here and where they're up to. """ + #: The ID of this element iteration. id_: int + #: Whether the element iteration has changes not yet persisted. is_pending: bool + #: Which element is an iteration for. element_ID: int + #: Whether EARs have been initialised for this element iteration. EARs_initialised: bool + #: Maps task schema action indices to EARs by ID. EAR_IDs: Dict[int, List[int]] + #: Overall data index for the element-iteration, which maps parameter names to + #: parameter data indices. data_idx: Dict[str, int] + #: List of parameters defined by the associated task schema. schema_parameters: List[str] + #: What loops are being handled here and where they're up to. loop_idx: Dict[str, int] = field(default_factory=dict) def encode(self) -> Dict: @@ -296,31 +378,75 @@ def set_EARs_initialised(self: AnySElementIter) -> AnySElementIter: @dataclass class StoreEAR: """ + Represents an element action run in a persistent store. + Parameters ---------- - data_idx + id_: + The ID of this element action run. + is_pending: + Whether the element action run has changes not yet persisted. + elem_iter_ID: + What element iteration owns this EAR. + action_idx: + The task schema action associated with this EAR. + commands_idx: + The indices of the commands in the EAR. + data_idx: Maps parameter names within this EAR to parameter data indices. - metadata + submission_idx: + Which submission contained this EAR, if known. + skip: + Whether to skip this EAR. + success: + Whether this EAR was successful, if known. + start_time: + When this EAR started, if known. + end_time: + When this EAR finished, if known. + snapshot_start: + Snapshot of files at EAR start, if recorded. + snapshot_end: + Snapshot of files at EAR end, if recorded. + exit_code: + The exit code of the underlying executable, if known. + metadata: Metadata concerning e.g. the state of the EAR. - action_idx - The task schema action associated with this EAR. + run_hostname: + Where this EAR was submitted to run, if known. """ + #: The ID of this element action run. id_: int + #: Whether the element action run has changes not yet persisted. is_pending: bool + #: What element iteration owns this EAR. elem_iter_ID: int + #: The task schema action associated with this EAR. action_idx: int + #: The indices of the commands in the EAR. commands_idx: List[int] + #: Maps parameter names within this EAR to parameter data indices. data_idx: Dict[str, int] + #: Which submission contained this EAR, if known. submission_idx: Optional[int] = None + #: Whether to skip this EAR. skip: Optional[bool] = False + #: Whether this EAR was successful, if known. success: Optional[bool] = None + #: When this EAR started, if known. start_time: Optional[datetime] = None + #: When this EAR finished, if known. end_time: Optional[datetime] = None + #: Snapshot of files at EAR start, if recorded. snapshot_start: Optional[Dict] = None + #: Snapshot of files at EAR end, if recorded. snapshot_end: Optional[Dict] = None + #: The exit code of the underlying executable, if known. exit_code: Optional[int] = None + #: Metadata concerning e.g. the state of the EAR. metadata: Dict[str, Any] = None + #: Where this EAR was submitted to run, if known. run_hostname: Optional[str] = None @staticmethod @@ -434,11 +560,36 @@ def update( @dataclass class StoreParameter: + """ + Represents a parameter in a persistent store. + + Parameters + ---------- + id_: + The ID of this parameter. + is_pending: + Whether the parameter has changes not yet persisted. + is_set: + Whether the parameter is set. + data: + Description of the value of the parameter. + file: + Description of the file this parameter represents. + source: + Description of where this parameter originated. + """ + + #: The ID of this parameter. id_: int + #: Whether the parameter has changes not yet persisted. is_pending: bool + #: Whether the parameter is set. is_set: bool + #: Description of the value of the parameter. data: Any + #: Description of the file this parameter represents. file: Dict + #: Description of where this parameter originated. source: Dict _encoders = {} @@ -647,6 +798,21 @@ def update_source(self, src: Dict) -> None: class PersistentStore(ABC): + """ + An abstract class representing a persistent workflow store. + + Parameters + ---------- + app: App + The main hpcflow core. + workflow: ~hpcflow.app.Workflow + The workflow being persisted. + path: pathlib.Path + Where to hold the store. + fs: fsspec.AbstractFileSystem + Optionally, information about how to access the store. + """ + _store_task_cls = StoreTask _store_elem_cls = StoreElement _store_iter_cls = StoreElementIter @@ -672,14 +838,23 @@ def __init__(self, app, workflow, path, fs=None) -> None: @property def logger(self): + """ + The logger to use. + """ return self.app.persistence_logger @property def ts_fmt(self) -> str: + """ + The format for timestamps. + """ return self.workflow.ts_fmt @property def has_pending(self): + """ + Whether there are any pending changes. + """ return bool(self._pending) @property @@ -689,6 +864,9 @@ def is_submittable(self): @property def use_cache(self): + """ + Whether to use a cache. + """ return self._use_cache @property @@ -914,6 +1092,9 @@ def save(self): self._pending.commit_all() def add_template_components(self, temp_comps: Dict, save: bool = True) -> None: + """ + Add template components to the workflow. + """ all_tc = self.get_template_components() for name, dat in temp_comps.items(): if name in all_tc: @@ -976,6 +1157,9 @@ def add_submission(self, sub_idx: int, sub_js: Dict, save: bool = True): self.save() def add_element_set(self, task_id: int, es_js: Dict, save: bool = True): + """ + Add an element set to a task. + """ self._pending.add_element_sets[task_id].append(es_js) if save: self.save() @@ -1058,6 +1242,9 @@ def add_EAR( def add_submission_part( self, sub_idx: int, dt_str: str, submitted_js_idx: List[int], save: bool = True ): + """ + Add a submission part. + """ self._pending.add_submission_parts[sub_idx][dt_str] = submitted_js_idx if save: self.save() @@ -1066,11 +1253,17 @@ def add_submission_part( def set_EAR_submission_index( self, EAR_ID: int, sub_idx: int, save: bool = True ) -> None: + """ + Set the submission index for an element action run. + """ self._pending.set_EAR_submission_indices[EAR_ID] = sub_idx if save: self.save() def set_EAR_start(self, EAR_ID: int, save: bool = True) -> datetime: + """ + Mark an element action run as started. + """ dt = datetime.utcnow() ss_js = self.app.RunDirAppFiles.take_snapshot() run_hostname = socket.gethostname() @@ -1082,6 +1275,9 @@ def set_EAR_start(self, EAR_ID: int, save: bool = True) -> datetime: def set_EAR_end( self, EAR_ID: int, exit_code: int, success: bool, save: bool = True ) -> datetime: + """ + Mark an element action run as finished. + """ # TODO: save output files dt = datetime.utcnow() ss_js = self.app.RunDirAppFiles.take_snapshot() @@ -1091,11 +1287,17 @@ def set_EAR_end( return dt def set_EAR_skip(self, EAR_ID: int, save: bool = True) -> None: + """ + Mark an element action run as skipped. + """ self._pending.set_EAR_skips.append(EAR_ID) if save: self.save() def set_EARs_initialised(self, iter_ID: int, save: bool = True) -> None: + """ + Mark an element action run as initialised. + """ self._pending.set_EARs_initialised.append(iter_ID) if save: self.save() @@ -1116,6 +1318,9 @@ def set_jobscript_metadata( process_ID: Optional[int] = None, save: bool = True, ): + """ + Set the metadata for a job script. + """ if version_info: self._pending.set_js_metadata[sub_idx][js_idx]["version_info"] = version_info if submit_time: @@ -1229,6 +1434,9 @@ def set_file( clean_up: bool = False, save: bool = True, ): + """ + Set details of a file, including whether it is associated with a parameter. + """ self.logger.debug(f"Setting new file") file_param_dat = self._prepare_set_file( store_contents=store_contents, @@ -1255,6 +1463,9 @@ def add_file( filename: str = None, save: bool = True, ): + """ + Add a file that will be associated with a parameter. + """ self.logger.debug(f"Adding new file") file_param_dat = self._prepare_set_file( store_contents=store_contents, @@ -1291,14 +1502,23 @@ def _append_files(self, files: Dict[int, Dict]): fp.write(dat["contents"]) def add_set_parameter(self, data: Any, source: Dict, save: bool = True) -> int: + """ + Add a parameter that is set to a value. + """ return self._add_parameter(data=data, is_set=True, source=source, save=save) def add_unset_parameter(self, source: Dict, save: bool = True) -> int: + """ + Add a parameter that is not set to any value. + """ return self._add_parameter(data=None, is_set=False, source=source, save=save) def set_parameter_value( self, param_id: int, value: Any, is_file: bool = False, save: bool = True ): + """ + Set the value of a parameter. + """ self.logger.debug( f"Setting store parameter ID {param_id} value with type: {type(value)!r})." ) @@ -1310,6 +1530,9 @@ def set_parameter_value( def update_param_source( self, param_sources: Dict[int, Dict], save: bool = True ) -> None: + """ + Set the source of a parameter. + """ self.logger.debug(f"Updating parameter sources with {param_sources!r}.") self._pending.update_param_sources.update(param_sources) if save: @@ -1318,6 +1541,9 @@ def update_param_source( def update_loop_num_iters( self, index: int, num_added_iters: int, save: bool = True ) -> None: + """ + Add iterations to a loop. + """ self.logger.debug( f"Updating loop {index!r} num added iterations to {num_added_iters!r}." ) @@ -1333,6 +1559,9 @@ def update_loop_parents( parents: List[str], save: bool = True, ) -> None: + """ + Set the parents of a loop. + """ self.logger.debug( f"Updating loop {index!r} parents to {parents!r}, and num added iterations " f"to {num_added_iters}." @@ -1355,6 +1584,9 @@ def get_template_components(self) -> Dict: return tc def get_template(self) -> Dict: + """ + Get the workflow template. + """ return self._get_persistent_template() def _get_task_id_to_idx_map(self) -> Dict[int, int]: @@ -1362,6 +1594,9 @@ def _get_task_id_to_idx_map(self) -> Dict[int, int]: @TimeIt.decorator def get_task(self, task_idx: int) -> AnySTask: + """ + Get a task. + """ return self.get_tasks()[task_idx] def _process_retrieved_tasks(self, tasks: List[AnySTask]) -> List[AnySTask]: @@ -1394,6 +1629,9 @@ def _process_retrieved_loops(self, loops: Dict[int, Dict]) -> Dict[int, Dict]: return loops_new def get_tasks_by_IDs(self, id_lst: Iterable[int]) -> List[AnySTask]: + """ + Get tasks with the given IDs. + """ # separate pending and persistent IDs: id_set = set(id_lst) all_pending = set(self._pending.add_tasks) @@ -1461,6 +1699,9 @@ def get_submissions(self) -> Dict[int, Dict]: @TimeIt.decorator def get_submissions_by_ID(self, id_lst: Iterable[int]) -> Dict[int, Dict]: + """ + Get submissions with the given IDs. + """ # separate pending and persistent IDs: id_set = set(id_lst) all_pending = set(self._pending.add_submissions) @@ -1477,6 +1718,9 @@ def get_submissions_by_ID(self, id_lst: Iterable[int]) -> Dict[int, Dict]: @TimeIt.decorator def get_elements(self, id_lst: Iterable[int]) -> List[AnySElement]: + """ + Get elements with the given IDs. + """ self.logger.debug(f"PersistentStore.get_elements: id_lst={id_lst!r}") # separate pending and persistent IDs: @@ -1504,6 +1748,9 @@ def get_elements(self, id_lst: Iterable[int]) -> List[AnySElement]: @TimeIt.decorator def get_element_iterations(self, id_lst: Iterable[int]) -> List[AnySElementIter]: + """ + Get element iterations with the given IDs. + """ self.logger.debug(f"PersistentStore.get_element_iterations: id_lst={id_lst!r}") # separate pending and persistent IDs: @@ -1540,6 +1787,9 @@ def get_element_iterations(self, id_lst: Iterable[int]) -> List[AnySElementIter] @TimeIt.decorator def get_EARs(self, id_lst: Iterable[int]) -> List[AnySEAR]: + """ + Get element action runs with the given IDs. + """ self.logger.debug(f"PersistentStore.get_EARs: id_lst={id_lst!r}") # separate pending and persistent IDs: @@ -1624,6 +1874,9 @@ def _get_cached_persistent_parameters(self, id_lst: Iterable[int]): return self._get_cached_persistent_items(id_lst, self.parameter_cache) def get_EAR_skipped(self, EAR_ID: int) -> bool: + """ + Whether the element action run with the given ID was skipped. + """ self.logger.debug(f"PersistentStore.get_EAR_skipped: EAR_ID={EAR_ID!r}") return self.get_EARs([EAR_ID])[0].skip @@ -1634,11 +1887,17 @@ def get_parameters( **kwargs: Dict, ) -> List[AnySParameter]: """ + Get parameters with the given IDs. + Parameters ---------- - kwargs : - dataset_copy : bool - For Zarr stores only. If True, copy arrays as NumPy arrays. + id_lst: + The IDs of the parameters to get. + + Keyword Arguments + ----------------- + dataset_copy: bool + For Zarr stores only. If True, copy arrays as NumPy arrays. """ # separate pending and persistent IDs: id_set = set(id_lst) @@ -1656,6 +1915,9 @@ def get_parameters( @TimeIt.decorator def get_parameter_set_statuses(self, id_lst: Iterable[int]) -> List[bool]: + """ + Get whether the parameters with the given IDs are set. + """ # separate pending and persistent IDs: id_set = set(id_lst) all_pending = set(self._pending.add_parameters) @@ -1670,6 +1932,9 @@ def get_parameter_set_statuses(self, id_lst: Iterable[int]) -> List[bool]: @TimeIt.decorator def get_parameter_sources(self, id_lst: Iterable[int]) -> List[Dict]: + """ + Get the sources of the parameters with the given IDs. + """ # separate pending and persistent IDs: id_set = set(id_lst) all_pending = set(self._pending.add_parameters) @@ -1698,7 +1963,8 @@ def get_task_elements( task_id, idx_lst: Optional[Iterable[int]] = None, ) -> List[Dict]: - """Get element data by an indices within a given task. + """ + Get element data by an indices within a given task. Element iterations and EARs belonging to the elements are included. diff --git a/hpcflow/sdk/persistence/json.py b/hpcflow/sdk/persistence/json.py index a05fd6167..093437979 100644 --- a/hpcflow/sdk/persistence/json.py +++ b/hpcflow/sdk/persistence/json.py @@ -1,3 +1,7 @@ +""" +Persistence model based on writing JSON documents. +""" + from __future__ import annotations from contextlib import contextmanager import copy @@ -30,6 +34,10 @@ class JSONPersistentStore(PersistentStore): + """ + A store that writes JSON files for all its state serialization. + """ + _name = "json" _features = PersistentStoreFeatures( create=True, @@ -90,6 +98,9 @@ def cached_load(self) -> Iterator[Dict]: yield md def remove_replaced_dir(self) -> None: + """ + Remove the directory containing replaced workflow details. + """ with self.using_resource("metadata", "update") as md: if "replaced_workflow" in md: self.remove_path(md["replaced_workflow"], self.fs) @@ -97,6 +108,9 @@ def remove_replaced_dir(self) -> None: md["replaced_workflow"] = None def reinstate_replaced_dir(self) -> None: + """ + Reinstate the directory containing replaced workflow details. + """ with self.using_resource("metadata", "read") as md: if "replaced_workflow" in md: self.logger.debug( @@ -128,6 +142,9 @@ def write_empty_workflow( ts_fmt: str, ts_name_fmt: str, ) -> None: + """ + Write an empty persistent workflow. + """ fs.mkdir(wk_path) submissions = [] parameters = { @@ -526,17 +543,29 @@ def _get_persistent_parameter_IDs(self) -> List[int]: return list(int(i) for i in params["data"].keys()) def get_ts_fmt(self): + """ + Get the format for timestamps. + """ with self.using_resource("metadata", action="read") as md: return md["ts_fmt"] def get_ts_name_fmt(self): + """ + Get the format for timestamps to use in names. + """ with self.using_resource("metadata", action="read") as md: return md["ts_name_fmt"] def get_creation_info(self): + """ + Get information about the creation of the workflow. + """ with self.using_resource("metadata", action="read") as md: return copy.deepcopy(md["creation_info"]) def get_name(self): + """ + Get the name of the workflow. + """ with self.using_resource("metadata", action="read") as md: return md["name"] diff --git a/hpcflow/sdk/persistence/pending.py b/hpcflow/sdk/persistence/pending.py index 6f8557c8e..0a7dafbcf 100644 --- a/hpcflow/sdk/persistence/pending.py +++ b/hpcflow/sdk/persistence/pending.py @@ -1,8 +1,12 @@ +""" +Class to hold the state that is waiting to be committed to disk. +""" + from __future__ import annotations from collections import defaultdict import contextlib -from dataclasses import dataclass, fields +from dataclasses import dataclass, field, fields from datetime import datetime from typing import Any, Dict, List, Optional, Tuple @@ -15,26 +19,12 @@ class PendingChanges: Parameters ---------- - add_tasks - Keys are new task IDs - add_elem_iter_EAR_IDs - Keys are element iteration IDs, then EAR action index, and values are EAR IDs. - This is a list of EAR IDs to add to a given element iteration action. - add_elem_iter_IDs - Keys are element IDs, and values are iteration IDs to add to that element. - add_elem_IDs - Keys are task IDs, and values are element IDs to add to that task. - add_parameters - Keys are parameter indices and values are tuples whose first element is data to - add and whose second element is the source dict for the new data. - update_param_sources - Keys are parameter indices and values are dict parameter sources to merge with - existing source of that parameter. - set_EAR_starts - Keys are EAR IDs and values are tuples of start time, and start dir snapshot. - set_EAR_ends - Keys are EAR IDs and values are tuples of end time, end dir snapshot, exit - code, and success boolean. + app: App + The main application context. + store: PersistentStore + The persistent store that owns this object + resource_map: CommitResourceMap + Map of resources, used when processing commits. """ def __init__(self, app, store, resource_map): @@ -42,35 +32,64 @@ def __init__(self, app, store, resource_map): self.store = store self.resource_map = resource_map + #: Keys are new task IDs. self.add_tasks: Dict[int, AnySTask] = None + #: Keys are loop IDs, values are loop descriptors. self.add_loops: Dict[int, Dict] = None + #: Keys are submission IDs, values are submission descriptors. self.add_submissions: Dict[int, Dict] = None + #: Keys are element IDs. self.add_elements: Dict[int, AnySElement] = None + #: Keys are element iteration IDs. self.add_elem_iters: Dict[int, AnySElementIter] = None + #: Keys are element action run IDs. self.add_EARs: Dict[int, AnySEAR] = None + #: Keys are parameter indices and values are tuples whose first element is data + #: to add and whose second element is the source dict for the new data. self.add_parameters: Dict[int, AnySParameter] = None + #: Workflow-related files (inputs, outputs) added to the persistent store. self.add_files: List[Dict] = None + #: Template components to add. self.add_template_components: Dict[str, Dict[str, Dict]] = None + #: Keys are element set IDs, values are descriptors. self.add_element_sets: Dict[int, Dict] = None + #: Keys are task IDs, and values are element IDs to add to that task. self.add_elem_IDs: Dict[int, List] = None + #: Keys are element IDs, and values are iteration IDs to add to that element. self.add_elem_iter_IDs: Dict[int, List] = None + #: Keys are element iteration IDs, then EAR action index, and values are EAR IDs. + #: This is a list of EAR IDs to add to a given element iteration action. self.add_elem_iter_EAR_IDs: Dict[int, Dict[int, List]] = None + #: Submission parts to add. self.add_submission_parts: Dict[int, Dict[str, List[int]]] = None + #: IDs of EARs to mark as initialised. self.set_EARs_initialised: List[int] = None + #: Submission IDs to attach to EARs. self.set_EAR_submission_indices: Dict[int, int] = None + #: IDs of EARs to mark as skipped. self.set_EAR_skips: List[int] = None + #: Keys are EAR IDs and values are tuples of start time, and start dir snapshot. self.set_EAR_starts: Dict[int, Tuple[datetime, Dict], str] = None + #: Keys are EAR IDs and values are tuples of end time, end dir snapshot, exit + #: code, and success boolean. self.set_EAR_ends: Dict[int, Tuple[datetime, Dict, int, bool]] = None + #: Keys are IDs of jobscripts. self.set_js_metadata: Dict[int, Dict[int, Any]] = None + #: Keys are IDs of parameters to add or modify. self.set_parameters: Dict[int, AnySParameter] = None + #: Keys are parameter indices and values are dict parameter sources to merge + #: with existing source of that parameter. self.update_param_sources: Dict[int, Dict] = None + #: Keys are indices of loops, values are descriptions of what to update. self.update_loop_indices: Dict[int, Dict[str, int]] = None + #: Keys are indices of loops, values are number of iterations. self.update_loop_num_iters: Dict[int, int] = None + #: Keys are indices of loops, values are list of parent names. self.update_loop_parents: Dict[int, List[str]] = None self.reset(is_init=True) # set up initial data structures @@ -106,6 +125,9 @@ def __bool__(self): ) def where_pending(self) -> List[str]: + """ + Get the list of items for which there is some outstanding pending items. + """ return [ k for k, v in self.__dict__.items() @@ -114,6 +136,9 @@ def where_pending(self) -> List[str]: @property def logger(self): + """ + The logger. + """ return self.app.persistence_logger @TimeIt.decorator @@ -149,7 +174,7 @@ def commit_tasks(self) -> None: self.add_elem_IDs = { k: v for k, v in self.add_elem_IDs.items() if k not in task_ids } - self.clear_add_tasks() + self._clear_add_tasks() @TimeIt.decorator def commit_loops(self) -> None: @@ -170,7 +195,7 @@ def commit_loops(self) -> None: k: v for k, v in self.update_loop_parents.items() if k not in loop_ids } - self.clear_add_loops() + self._clear_add_loops() @TimeIt.decorator def commit_submissions(self) -> None: @@ -183,17 +208,23 @@ def commit_submissions(self) -> None: f"commit: adding pending submissions with indices {sub_ids!r}" ) self.store._append_submissions(subs) - self.clear_add_submissions() + self._clear_add_submissions() @TimeIt.decorator def commit_submission_parts(self) -> None: + """ + Commit pending submission parts to disk. + """ if self.add_submission_parts: self.logger.debug(f"commit: adding pending submission parts") self.store._append_submission_parts(self.add_submission_parts) - self.clear_add_submission_parts() + self._clear_add_submission_parts() @TimeIt.decorator def commit_elem_IDs(self) -> None: + """ + Commit pending element ID updates to disk. + """ # TODO: could be batched up? for task_ID, elem_IDs in self.add_elem_IDs.items(): self.logger.debug( @@ -201,10 +232,13 @@ def commit_elem_IDs(self) -> None: ) self.store._append_task_element_IDs(task_ID, elem_IDs) self.store.task_cache.pop(task_ID, None) # invalidate cache - self.clear_add_elem_IDs() + self._clear_add_elem_IDs() @TimeIt.decorator def commit_elements(self) -> None: + """ + Commit pending elements to disk. + """ if self.add_elements: elems = self.store.get_elements(self.add_elements) elem_ids = list(self.add_elements.keys()) @@ -214,18 +248,24 @@ def commit_elements(self) -> None: self.add_elem_iter_IDs = { k: v for k, v in self.add_elem_iter_IDs.items() if k not in elem_ids } - self.clear_add_elements() + self._clear_add_elements() @TimeIt.decorator def commit_element_sets(self) -> None: + """ + Commit pending element sets to disk. + """ # TODO: could be batched up? for task_id, es_js in self.add_element_sets.items(): self.logger.debug(f"commit: adding pending element sets.") self.store._append_element_sets(task_id, es_js) - self.clear_add_element_sets() + self._clear_add_element_sets() @TimeIt.decorator def commit_elem_iter_IDs(self) -> None: + """ + Commit pending element iteration ID updates to disk. + """ # TODO: could be batched up? for elem_ID, iter_IDs in self.add_elem_iter_IDs.items(): self.logger.debug( @@ -234,10 +274,13 @@ def commit_elem_iter_IDs(self) -> None: ) self.store._append_elem_iter_IDs(elem_ID, iter_IDs) self.store.element_cache.pop(elem_ID, None) # invalidate cache - self.clear_add_elem_iter_IDs() + self._clear_add_elem_iter_IDs() @TimeIt.decorator def commit_elem_iters(self) -> None: + """ + Commit pending element iterations to disk. + """ if self.add_elem_iters: iters = self.store.get_element_iterations(self.add_elem_iters.keys()) iter_ids = list(self.add_elem_iters.keys()) @@ -253,10 +296,13 @@ def commit_elem_iters(self) -> None: self.set_EARs_initialised = [ i for i in self.set_EARs_initialised if i not in iter_ids ] - self.clear_add_elem_iters() + self._clear_add_elem_iters() @TimeIt.decorator def commit_elem_iter_EAR_IDs(self) -> None: + """ + Commit pending element action run ID updates to disk. + """ # TODO: could be batched up? for iter_ID, act_EAR_IDs in self.add_elem_iter_EAR_IDs.items(): self.logger.debug( @@ -266,10 +312,13 @@ def commit_elem_iter_EAR_IDs(self) -> None: for act_idx, EAR_IDs in act_EAR_IDs.items(): self.store._append_elem_iter_EAR_IDs(iter_ID, act_idx, EAR_IDs) self.store.element_iter_cache.pop(iter_ID, None) # invalidate cache - self.clear_add_elem_iter_EAR_IDs() + self._clear_add_elem_iter_EAR_IDs() @TimeIt.decorator def commit_EARs(self) -> None: + """ + Commit pending element action runs to disk. + """ if self.add_EARs: EARs = self.store.get_EARs(self.add_EARs) EAR_ids = list(self.add_EARs.keys()) @@ -291,10 +340,13 @@ def commit_EARs(self) -> None: k: v for k, v in self.set_EAR_ends.items() if k not in EAR_ids } - self.clear_add_EARs() + self._clear_add_EARs() @TimeIt.decorator def commit_EARs_initialised(self) -> None: + """ + Commit pending element action run init state updates to disk. + """ if self.set_EARs_initialised: iter_ids = self.set_EARs_initialised self.logger.debug( @@ -305,10 +357,13 @@ def commit_EARs_initialised(self) -> None: for i in iter_ids: self.store._update_elem_iter_EARs_initialised(i) self.store.element_iter_cache.pop(i, None) # invalidate cache - self.clear_set_EARs_initialised() + self._clear_set_EARs_initialised() @TimeIt.decorator def commit_EAR_submission_indices(self) -> None: + """ + Commit pending element action run submission index updates to disk. + """ if self.set_EAR_submission_indices: self.logger.debug( f"commit: updating submission indices: " @@ -317,10 +372,13 @@ def commit_EAR_submission_indices(self) -> None: self.store._update_EAR_submission_indices(self.set_EAR_submission_indices) for EAR_ID_i in self.set_EAR_submission_indices.keys(): self.store.EAR_cache.pop(EAR_ID_i, None) # invalidate cache - self.clear_set_EAR_submission_indices() + self._clear_set_EAR_submission_indices() @TimeIt.decorator def commit_EAR_starts(self) -> None: + """ + Commit pending element action run start information to disk. + """ # TODO: could be batched up? for EAR_id, (time, snap, hostname) in self.set_EAR_starts.items(): self.logger.debug( @@ -329,10 +387,13 @@ def commit_EAR_starts(self) -> None: ) self.store._update_EAR_start(EAR_id, time, snap, hostname) self.store.EAR_cache.pop(EAR_id, None) # invalidate cache - self.clear_set_EAR_starts() + self._clear_set_EAR_starts() @TimeIt.decorator def commit_EAR_ends(self) -> None: + """ + Commit pending element action run finish information to disk. + """ # TODO: could be batched up? for EAR_id, (time, snap, ext, suc) in self.set_EAR_ends.items(): self.logger.debug( @@ -341,25 +402,31 @@ def commit_EAR_ends(self) -> None: ) self.store._update_EAR_end(EAR_id, time, snap, ext, suc) self.store.EAR_cache.pop(EAR_id, None) # invalidate cache - self.clear_set_EAR_ends() + self._clear_set_EAR_ends() @TimeIt.decorator def commit_EAR_skips(self) -> None: + """ + Commit pending element action skip flags to disk. + """ # TODO: could be batched up? for EAR_id in self.set_EAR_skips: self.logger.debug(f"commit: setting EAR ID {EAR_id!r} as skipped.") self.store._update_EAR_skip(EAR_id) self.store.EAR_cache.pop(EAR_id, None) # invalidate cache - self.clear_set_EAR_skips() + self._clear_set_EAR_skips() @TimeIt.decorator def commit_js_metadata(self) -> None: + """ + Commit pending jobscript metadata changes to disk. + """ if self.set_js_metadata: self.logger.debug( f"commit: setting jobscript metadata: {self.set_js_metadata!r}" ) self.store._update_js_metadata(self.set_js_metadata) - self.clear_set_js_metadata() + self._clear_set_js_metadata() @TimeIt.decorator def commit_parameters(self) -> None: @@ -369,7 +436,7 @@ def commit_parameters(self) -> None: param_ids = list(self.add_parameters.keys()) self.logger.debug(f"commit: adding pending parameters IDs: {param_ids!r}") self.store._append_parameters(params) - self.clear_add_parameters() + self._clear_add_parameters() if self.set_parameters: param_ids = list(self.set_parameters.keys()) @@ -378,7 +445,7 @@ def commit_parameters(self) -> None: for id_i in param_ids: self.store.parameter_cache.pop(id_i, None) - self.clear_set_parameters() + self._clear_set_parameters() @TimeIt.decorator def commit_files(self) -> None: @@ -386,14 +453,17 @@ def commit_files(self) -> None: if self.add_files: self.logger.debug(f"commit: adding pending files to the files directory.") self.store._append_files(self.add_files) - self.clear_add_files() + self._clear_add_files() @TimeIt.decorator def commit_template_components(self) -> None: + """ + Commit pending template components to disk. + """ if self.add_template_components: self.logger.debug(f"commit: adding template components.") self.store._update_template_components(self.store.get_template_components()) - self.clear_add_template_components() + self._clear_add_template_components() @TimeIt.decorator def commit_param_sources(self) -> None: @@ -404,7 +474,7 @@ def commit_param_sources(self) -> None: self.store._update_parameter_sources(self.update_param_sources) for id_i in param_ids: self.store.param_sources_cache.pop(id_i, None) # invalidate cache - self.clear_update_param_sources() + self._clear_update_param_sources() @TimeIt.decorator def commit_loop_indices(self) -> None: @@ -417,7 +487,7 @@ def commit_loop_indices(self) -> None: ) self.store._update_loop_index(iter_ID, loop_idx) self.store.element_iter_cache.pop(iter_ID, None) # invalidate cache - self.clear_update_loop_indices() + self._clear_update_loop_indices() @TimeIt.decorator def commit_loop_num_iters(self) -> None: @@ -427,7 +497,7 @@ def commit_loop_num_iters(self) -> None: f"commit: updating loop {index!r} number of iterations to {num_iters!r}." ) self.store._update_loop_num_iters(index, num_iters) - self.clear_update_loop_num_iters() + self._clear_update_loop_num_iters() @TimeIt.decorator def commit_loop_parents(self) -> None: @@ -435,81 +505,81 @@ def commit_loop_parents(self) -> None: for index, parents in self.update_loop_parents.items(): self.logger.debug(f"commit: updating loop {index!r} parents to {parents!r}.") self.store._update_loop_parents(index, parents) - self.clear_update_loop_parents() + self._clear_update_loop_parents() - def clear_add_tasks(self): + def _clear_add_tasks(self): self.add_tasks = {} - def clear_add_loops(self): + def _clear_add_loops(self): self.add_loops = {} - def clear_add_submissions(self): + def _clear_add_submissions(self): self.add_submissions = {} - def clear_add_submission_parts(self): + def _clear_add_submission_parts(self): self.add_submission_parts = defaultdict(dict) - def clear_add_elements(self): + def _clear_add_elements(self): self.add_elements = {} - def clear_add_element_sets(self): + def _clear_add_element_sets(self): self.add_element_sets = defaultdict(list) - def clear_add_elem_iters(self): + def _clear_add_elem_iters(self): self.add_elem_iters = {} - def clear_add_EARs(self): + def _clear_add_EARs(self): self.add_EARs = {} - def clear_add_elem_IDs(self): + def _clear_add_elem_IDs(self): self.add_elem_IDs = defaultdict(list) - def clear_add_elem_iter_IDs(self): + def _clear_add_elem_iter_IDs(self): self.add_elem_iter_IDs = defaultdict(list) - def clear_add_elem_iter_EAR_IDs(self): + def _clear_add_elem_iter_EAR_IDs(self): self.add_elem_iter_EAR_IDs = defaultdict(lambda: defaultdict(list)) - def clear_set_EARs_initialised(self): + def _clear_set_EARs_initialised(self): self.set_EARs_initialised = [] - def clear_set_EAR_submission_indices(self): + def _clear_set_EAR_submission_indices(self): self.set_EAR_submission_indices = {} - def clear_set_EAR_starts(self): + def _clear_set_EAR_starts(self): self.set_EAR_starts = {} - def clear_set_EAR_ends(self): + def _clear_set_EAR_ends(self): self.set_EAR_ends = {} - def clear_set_EAR_skips(self): + def _clear_set_EAR_skips(self): self.set_EAR_skips = [] - def clear_set_js_metadata(self): + def _clear_set_js_metadata(self): self.set_js_metadata = defaultdict(lambda: defaultdict(dict)) - def clear_add_parameters(self): + def _clear_add_parameters(self): self.add_parameters = {} - def clear_add_files(self): + def _clear_add_files(self): self.add_files = [] - def clear_add_template_components(self): + def _clear_add_template_components(self): self.add_template_components = defaultdict(dict) - def clear_set_parameters(self): + def _clear_set_parameters(self): self.set_parameters = {} - def clear_update_param_sources(self): + def _clear_update_param_sources(self): self.update_param_sources = {} - def clear_update_loop_indices(self): + def _clear_update_loop_indices(self): self.update_loop_indices = defaultdict(dict) - def clear_update_loop_num_iters(self): + def _clear_update_loop_num_iters(self): self.update_loop_num_iters = {} - def clear_update_loop_parents(self): + def _clear_update_loop_parents(self): self.update_loop_parents = {} def reset(self, is_init=False) -> None: @@ -522,87 +592,127 @@ def reset(self, is_init=False) -> None: if not is_init: self.logger.info("resetting pending changes.") - self.clear_add_tasks() - self.clear_add_loops() - self.clear_add_submissions() - self.clear_add_submission_parts() - self.clear_add_elements() - self.clear_add_element_sets() - self.clear_add_elem_iters() - self.clear_add_EARs() + self._clear_add_tasks() + self._clear_add_loops() + self._clear_add_submissions() + self._clear_add_submission_parts() + self._clear_add_elements() + self._clear_add_element_sets() + self._clear_add_elem_iters() + self._clear_add_EARs() - self.clear_set_EARs_initialised() - self.clear_add_elem_IDs() - self.clear_add_elem_iter_IDs() - self.clear_add_elem_iter_EAR_IDs() + self._clear_set_EARs_initialised() + self._clear_add_elem_IDs() + self._clear_add_elem_iter_IDs() + self._clear_add_elem_iter_EAR_IDs() - self.clear_add_parameters() - self.clear_add_files() - self.clear_add_template_components() + self._clear_add_parameters() + self._clear_add_files() + self._clear_add_template_components() - self.clear_set_EAR_submission_indices() - self.clear_set_EAR_starts() - self.clear_set_EAR_ends() - self.clear_set_EAR_skips() + self._clear_set_EAR_submission_indices() + self._clear_set_EAR_starts() + self._clear_set_EAR_ends() + self._clear_set_EAR_skips() - self.clear_set_js_metadata() - self.clear_set_parameters() + self._clear_set_js_metadata() + self._clear_set_parameters() - self.clear_update_param_sources() - self.clear_update_loop_indices() - self.clear_update_loop_num_iters() - self.clear_update_loop_parents() + self._clear_update_param_sources() + self._clear_update_loop_indices() + self._clear_update_loop_num_iters() + self._clear_update_loop_parents() @dataclass class CommitResourceMap: - """Map of `PendingChanges` commit method names to store resource labels, representing - the store resources required by each commit method, for a given `PersistentStore` + """ + Map of :py:class:`PendingChanges` commit method names to store resource labels, + representing the store resources required by each ``commit_*`` method, for a given + :py:class:`~.PersistentStore`. - When `PendingChanges.commit_all` is called, the resources specified will be opened in - "update" mode, for each `commit_` method. + When :py:meth:`PendingChanges.commit_all` is called, the resources specified will be + opened in "update" mode, for each ``commit_*`` method. + Notes + ----- + Normally only of interest to implementations of persistent stores. """ + #: Resources for :py:meth:`~.PendingChanges.commit_tasks`. commit_tasks: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_loops`. commit_loops: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_submissions`. commit_submissions: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_submission_parts`. commit_submission_parts: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_elem_IDs`. commit_elem_IDs: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_elements`. commit_elements: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_element_sets`. commit_element_sets: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_elem_iter_IDs`. commit_elem_iter_IDs: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_elem_iters`. commit_elem_iters: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_elem_iter_EAR_IDs`. commit_elem_iter_EAR_IDs: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_EARs_initialised`. commit_EARs_initialised: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_EARs`. commit_EARs: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_EAR_submission_indices`. commit_EAR_submission_indices: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_EAR_skips`. commit_EAR_skips: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_EAR_starts`. commit_EAR_starts: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_EAR_ends`. commit_EAR_ends: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_js_metadata`. commit_js_metadata: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_parameters`. commit_parameters: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_files`. commit_files: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_template_components`. commit_template_components: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_param_sources`. commit_param_sources: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_loop_indices`. commit_loop_indices: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_loop_num_iters`. commit_loop_num_iters: Optional[Tuple[str]] = tuple() + #: Resources for :py:meth:`~.PendingChanges.commit_loop_parents`. commit_loop_parents: Optional[Tuple[str]] = tuple() + #: A dict whose keys are tuples of resource labels and whose values are lists + #: of :py:class:`PendingChanges` commit method names that require those resources. + #: + #: This grouping allows us to batch up commit methods by resource requirements, + #: which in turn means we can potentially minimise, e.g., the number of network + #: requests. + groups: Dict[Tuple[str], List[str]] = field(init=False, repr=False, compare=False) def __post_init__(self): - self.groups = self.group_by_resource() + self.groups = self._group_by_resource() - def group_by_resource(self) -> Dict[Tuple[str], List[str]]: - """Return a dict whose keys are tuples of resource labels and whose values are - lists of `PendingChanges` commit method names that require those resource. - - This grouping allows us to batch up commit methods by resource requirements, which - in turn means we can potentially minimise e.g. the number of network requests. + def _group_by_resource(self) -> Dict[Tuple[str], List[str]]: + """ + Get a dict whose keys are tuples of resource labels and whose values are + lists of :py:class:`PendingChanges` commit method names that require those + resource. + This grouping allows us to batch up commit methods by resource requirements, + which in turn means we can potentially minimise e.g. the number of network + requests. """ groups = {} cur_res_group = None for fld in fields(self): + if not fld.name.startswith("commit_"): + continue res_labels = getattr(self, fld.name) if not cur_res_group: diff --git a/hpcflow/sdk/persistence/store_resource.py b/hpcflow/sdk/persistence/store_resource.py index fdf5dd365..7da772ced 100644 --- a/hpcflow/sdk/persistence/store_resource.py +++ b/hpcflow/sdk/persistence/store_resource.py @@ -1,3 +1,7 @@ +""" +Models of data stores as resources. +""" + from abc import ABC, abstractmethod import copy import json @@ -13,6 +17,12 @@ class StoreResource(ABC): A `PersistentStore` maps workflow data across zero or more store resources. Updates to persistent workflow data that live in the same store resource are performed together. + Parameters + ---------- + app: App + The main application context. + name: + The store name. """ def __init__(self, app, name: str) -> None: @@ -26,6 +36,9 @@ def __repr__(self) -> str: @property def logger(self): + """ + The logger. + """ return self.app.persistence_logger @abstractmethod @@ -37,6 +50,14 @@ def _dump(self, data): pass def open(self, action): + """ + Open the store. + + Parameters + ---------- + action: str + What we are opening the store for; typically either ``read`` or ``update``. + """ if action == "read": # reuse "update" data if set, rather than re-loading from disk -- but copy, # so changes made in the "read" scope do not update! @@ -64,6 +85,15 @@ def open(self, action): pass def close(self, action): + """ + Close the store for a particular action. + + Parameters + ---------- + action: str + What we are closing the store for. + Should match a previous call to :py:meth:`close`. + """ if action == "read": self.logger.debug(f"{self!r}: closing read.") elif action == "update": @@ -88,7 +118,22 @@ def _check_action(self, action: str): class JSONFileStoreResource(StoreResource): - """For caching reads and writes to a JSON file.""" + """ + For caching reads and writes to a JSON file. + + Parameters + ---------- + app: App + The main application context. + name: + The store name. + filename: + The name of the JSON file. + path: + The path to the directory containing the JSON file. + fs: + The filesystem that the JSON file resides within. + """ def __init__(self, app, name: str, filename: str, path: Union[str, Path], fs): self.filename = filename @@ -114,7 +159,18 @@ def _dump(self, data): class ZarrAttrsStoreResource(StoreResource): - """For caching reads and writes to Zarr attributes on groups and arrays.""" + """ + For caching reads and writes to Zarr attributes on groups and arrays. + + Parameters + ---------- + app: App + The main application context. + name: + The store name. + open_call: + How to actually perform an open on the underlying resource. + """ def __init__(self, app, name: str, open_call: Callable): self.open_call = open_call diff --git a/hpcflow/sdk/persistence/utils.py b/hpcflow/sdk/persistence/utils.py index 3bc500c95..d92db11e6 100644 --- a/hpcflow/sdk/persistence/utils.py +++ b/hpcflow/sdk/persistence/utils.py @@ -1,9 +1,17 @@ +""" +Miscellaneous persistence-related helpers. +""" + from getpass import getpass from hpcflow.sdk.core.errors import WorkflowNotFoundError def ask_pw_on_auth_exc(f, *args, add_pw_to=None, **kwargs): + """ + Run the given function on the given arguments and add a password if the function + fails with an SSHException. + """ from paramiko.ssh_exception import SSHException try: diff --git a/hpcflow/sdk/persistence/zarr.py b/hpcflow/sdk/persistence/zarr.py index 80ca6996f..78e3eb919 100644 --- a/hpcflow/sdk/persistence/zarr.py +++ b/hpcflow/sdk/persistence/zarr.py @@ -1,3 +1,7 @@ +""" +Persistence model based on writing Zarr arrays. +""" + from __future__ import annotations import copy @@ -132,6 +136,10 @@ def append_items_to_ragged_array(arr, items): @dataclass class ZarrStoreTask(StoreTask): + """ + Represents a task in a Zarr persistent store. + """ + def encode(self) -> Tuple[int, np.ndarray, Dict]: """Prepare store task data for the persistent store.""" wk_task = {"id_": self.id_, "element_IDs": np.array(self.element_IDs)} @@ -147,6 +155,10 @@ def decode(cls, task_dat: Dict) -> ZarrStoreTask: @dataclass class ZarrStoreElement(StoreElement): + """ + Represents an element in a Zarr persistent store. + """ + def encode(self, attrs: Dict) -> List: """Prepare store elements data for the persistent store. @@ -180,6 +192,10 @@ def decode(cls, elem_dat: List, attrs: Dict) -> ZarrStoreElement: @dataclass class ZarrStoreElementIter(StoreElementIter): + """ + Represents an element iteration in a Zarr persistent store. + """ + def encode(self, attrs: Dict) -> List: """Prepare store element iteration data for the persistent store. @@ -216,6 +232,10 @@ def decode(cls, iter_dat: List, attrs: Dict) -> StoreElementIter: @dataclass class ZarrStoreEAR(StoreEAR): + """ + Represents an element action run in a Zarr persistent store. + """ + def encode(self, attrs: Dict, ts_fmt: str) -> Tuple[List, Tuple[np.datetime64]]: """Prepare store EAR data for the persistent store. @@ -268,6 +288,10 @@ def decode(cls, EAR_dat: List, attrs: Dict, ts_fmt: str) -> ZarrStoreEAR: @dataclass class ZarrStoreParameter(StoreParameter): + """ + Represents a parameter in a Zarr persistent store. + """ + _encoders = { # keys are types np.ndarray: _encode_numpy_array, np.ma.core.MaskedArray: _encode_masked_array, @@ -301,6 +325,10 @@ def decode( class ZarrPersistentStore(PersistentStore): + """ + A persistent store implemented using Zarr. + """ + _name = "zarr" _features = PersistentStoreFeatures( create=True, @@ -346,6 +374,9 @@ def cached_load(self) -> Iterator[Dict]: yield attrs def remove_replaced_dir(self) -> None: + """ + Remove the directory containing replaced workflow details. + """ with self.using_resource("attrs", "update") as md: if "replaced_workflow" in md: self.logger.debug("removing temporarily renamed pre-existing workflow.") @@ -353,6 +384,9 @@ def remove_replaced_dir(self) -> None: md["replaced_workflow"] = None def reinstate_replaced_dir(self) -> None: + """ + Reinstate the directory containing replaced workflow details. + """ with self.using_resource("attrs", "read") as md: if "replaced_workflow" in md: self.logger.debug( @@ -380,6 +414,9 @@ def write_empty_workflow( compressor: Optional[Union[str, None]] = "blosc", compressor_kwargs: Optional[Dict[str, Any]] = None, ) -> None: + """ + Write an empty persistent workflow. + """ attrs = { "name": name, "ts_fmt": ts_fmt, @@ -796,6 +833,9 @@ def _get_num_persistent_added_tasks(self): @property def zarr_store(self) -> zarr.storage.Store: + """ + The underlying store object. + """ if self._zarr_store is None: self._zarr_store = self._get_zarr_store(self.path, self.fs) return self._zarr_store @@ -1139,18 +1179,30 @@ def _get_persistent_parameter_IDs(self) -> List[int]: return list(range(len(base_arr))) def get_ts_fmt(self): + """ + Get the format for timestamps. + """ with self.using_resource("attrs", action="read") as attrs: return attrs["ts_fmt"] def get_ts_name_fmt(self): + """ + Get the format for timestamps to use in names. + """ with self.using_resource("attrs", action="read") as attrs: return attrs["ts_name_fmt"] def get_creation_info(self): + """ + Get information about the creation of the workflow. + """ with self.using_resource("attrs", action="read") as attrs: return copy.deepcopy(attrs["creation_info"]) def get_name(self): + """ + Get the name of the workflow. + """ with self.using_resource("attrs", action="read") as attrs: return attrs["name"] @@ -1163,6 +1215,8 @@ def zip( include_rechunk_backups=False, ): """ + Convert the persistent store to zipped form. + Parameters ---------- path: @@ -1299,6 +1353,9 @@ def rechunk_parameter_base( backup: Optional[bool] = True, status: Optional[bool] = True, ): + """ + Rechunk the parameter data to be stored more efficiently. + """ arr = self._get_parameter_base_array() return self._rechunk_arr(arr, chunk_size, backup, status) @@ -1308,13 +1365,21 @@ def rechunk_runs( backup: Optional[bool] = True, status: Optional[bool] = True, ): + """ + Rechunk the run data to be stored more efficiently. + """ arr = self._get_EARs_arr() return self._rechunk_arr(arr, chunk_size, backup, status) class ZarrZipPersistentStore(ZarrPersistentStore): """A store designed mainly as an archive format that can be uploaded to data - repositories such as Zenodo.""" + repositories such as Zenodo. + + Note + ---- + Archive format persistent stores cannot be updated without being unzipped first. + """ _name = "zip" _features = PersistentStoreFeatures( @@ -1333,6 +1398,8 @@ def zip(self): def unzip(self, path=".", log=None): """ + Expand the persistent store. + Parameters ---------- path: diff --git a/hpcflow/sdk/runtime.py b/hpcflow/sdk/runtime.py index 70b362763..3225daafe 100644 --- a/hpcflow/sdk/runtime.py +++ b/hpcflow/sdk/runtime.py @@ -1,3 +1,7 @@ +""" +Information about the Python runtime. +""" + from importlib import import_module import logging import os @@ -15,17 +19,16 @@ class RunTimeInfo: """Get useful run-time information, including the executable name used to invoke the CLI, in the case a PyInstaller-built executable was used. - Attributes + Parameters ---------- - sys_prefix : str - From `sys.prefix`. If running in a virtual environment, this will point to the - environment directory. If not running in a virtual environment, this will point to - the Python installation root. - sys_base_prefix : str - From `sys.base_prefix`. This will be equal to `sys_prefix` (`sys.prefix`) if not - running within a virtual environment. However, if running within a virtual - environment, this will be the Python installation directory, and `sys_prefix` will - be equal to the virtual environment directory. + name: + Application name. + package_name: + Application package name. + version: + Application version. + logger: + Where to write logging versions. """ def __init__(self, name, package_name, version, logger): @@ -34,22 +37,35 @@ def __init__(self, name, package_name, version, logger): sys._MEIPASS if is_frozen else os.path.dirname(os.path.abspath(__file__)) ) + #: Application name. self.name = name.split(".")[0] # if name is given as __name__ # TODO: what? + #: Application package name. self.package_name = package_name + #: Application version. self.version = version + #: Whether this is a frozen application. self.is_frozen = is_frozen + #: Working directory. self.working_dir = os.getcwd() + #: Where to write log messages. self.logger = logger + #: Host that this is running on. self.hostname = socket.gethostname() + #: Whether this application is inside iPython. self.in_ipython = False + #: Whether this application is being used interactively. self.is_interactive = False + #: Whether this application is being used in test mode. self.in_pytest = False # set in `conftest.py` + #: Whether this application is being run from the CLI. self.from_CLI = False # set in CLI if self.is_frozen: + #: The bundle directory, if frozen. self.bundle_dir = Path(bundle_dir) else: + #: The path to Python itself. self.python_executable_path = Path(sys.executable) try: @@ -61,16 +77,30 @@ def __init__(self, name, package_name, version, logger): if hasattr(sys, "ps1"): self.is_interactive = True + #: The Python version. self.python_version = platform.python_version() + #: Whether the application is in a virtual environment. self.is_venv = hasattr(sys, "real_prefix") or sys.base_prefix != sys.prefix + #: Whether the application is in a Conda virtual environment. self.is_conda_venv = "CONDA_PREFIX" in os.environ + #: From `sys.prefix`. If running in a virtual environment, this will point to the + #: environment directory. If not running in a virtual environment, this will + #: point to the Python installation root. self.sys_prefix = getattr(sys, "prefix", None) + #: From `sys.base_prefix`. This will be equal to `sys_prefix` (`sys.prefix`) if + #: not running within a virtual environment. However, if running within a virtual + #: environment, this will be the Python installation directory, and `sys_prefix` + #: will be equal to the virtual environment directory. self.sys_base_prefix = getattr(sys, "base_prefix", None) + #: The old base prefix, from `sys.real_prefix`. Compatibility version of + #: :py:attr:`sys_base_prefix`. self.sys_real_prefix = getattr(sys, "real_prefix", None) + #: The Conda prefix, if defined. self.conda_prefix = os.environ.get("CONDA_PREFIX") try: + #: The virtual environment path. self.venv_path = self._set_venv_path() except ValueError: self.venv_path = None @@ -97,6 +127,9 @@ def __init__(self, name, package_name, version, logger): # warnings.warn(msg) def to_dict(self): + """ + Serialize this class as a dictionary. + """ out = { "name": self.name, "package_name": self.package_name, @@ -157,12 +190,21 @@ def _set_venv_path(self): return out def get_activate_env_command(self): + """ + Get the command to activate the virtual environment. + """ pass def get_deactivate_env_command(self): + """ + Get the command to deactivate the virtual environment. + """ pass def show(self): + """ + Display the information known by this class as a human-readable table. + """ tab = Table(show_header=False, box=None) tab.add_column() tab.add_column() diff --git a/hpcflow/sdk/submission/__init__.py b/hpcflow/sdk/submission/__init__.py index e69de29bb..91e2c28e9 100644 --- a/hpcflow/sdk/submission/__init__.py +++ b/hpcflow/sdk/submission/__init__.py @@ -0,0 +1,3 @@ +""" +Subsystem for submitting work to schedulers for enactment. +""" diff --git a/hpcflow/sdk/submission/jobscript.py b/hpcflow/sdk/submission/jobscript.py index 230606fdf..69106e1ff 100644 --- a/hpcflow/sdk/submission/jobscript.py +++ b/hpcflow/sdk/submission/jobscript.py @@ -1,3 +1,7 @@ +""" +Model of information submitted to a scheduler. +""" + from __future__ import annotations import copy @@ -28,8 +32,10 @@ def generate_EAR_resource_map( task: app.WorkflowTask, loop_idx: Dict, ) -> Tuple[List[app.ElementResources], List[int], NDArray, NDArray]: - """Generate an integer array whose rows represent actions and columns represent task - elements and whose values index unique resources.""" + """ + Generate an integer array whose rows represent actions and columns represent task + elements and whose values index unique resources. + """ # TODO: assume single iteration for now; later we will loop over Loop tasks for each # included task and call this func with specific loop indices none_val = -1 @@ -83,6 +89,9 @@ def group_resource_map_into_jobscripts( resource_map: Union[List, NDArray], none_val: Any = -1, ): + """ + Convert a resource map into a plan for what elements to group together into jobscripts. + """ resource_map = np.asanyarray(resource_map) resource_idx = np.unique(resource_map) jobscripts = [] @@ -152,6 +161,9 @@ def group_resource_map_into_jobscripts( @TimeIt.decorator def resolve_jobscript_dependencies(jobscripts, element_deps): + """ + Discover concrete dependencies between jobscripts. + """ # first pass is to find the mappings between jobscript elements: jobscript_deps = {} for js_idx, elem_deps in element_deps.items(): @@ -302,6 +314,52 @@ def jobscripts_to_list(jobscripts: Dict[int, Dict]) -> List[Dict]: class Jobscript(JSONLike): + """ + A group of actions that are submitted together to be executed by the underlying job + management system as a single unit. + + Parameters + ---------- + task_insert_IDs: list[int] + The task insertion IDs. + task_actions: list[tuple] + The actions of the tasks. + ``task insert ID, action_idx, index into task_loop_idx`` for each ``JS_ACTION_IDX`` + task_elements: dict[int, list[int]] + The elements of the tasks. + Maps ``JS_ELEMENT_IDX`` to list of ``TASK_ELEMENT_IDX`` for each ``TASK_INSERT_ID`` + EAR_ID: + Element action run information. + resources: ~hpcflow.app.ElementResources + Resources to use + task_loop_idx: list[dict] + Description of what loops are in play. + dependencies: dict[int, dict] + Description of dependencies. + submit_time: datetime + When the jobscript was submitted, if known. + submit_hostname: str + Where the jobscript was submitted, if known. + submit_machine: str + Description of what the jobscript was submitted to, if known. + submit_cmdline: str + The command line used to do the commit, if known. + scheduler_job_ID: str + The job ID from the scheduler, if known. + process_ID: int + The process ID of the subprocess, if known. + version_info: tuple[str, ...] + Version info about the target system. + os_name: str + The name of the OS. + shell_name: str + The name of the shell. + scheduler_name: str + The scheduler used. + running: bool + Whether the jobscript is currently running. + """ + _app_attr = "app" _EAR_files_delimiter = ":" _workflow_app_alias = "wkflow_app" @@ -399,9 +457,15 @@ def from_json_like(cls, json_like, shared_data=None): @property def workflow_app_alias(self): + """ + Alias for the workflow app in job scripts. + """ return self._workflow_app_alias def get_commands_file_name(self, js_action_idx, shell=None): + """ + Get the name of a file containing commands for a particular jobscript action. + """ return self.app.RunDirAppFiles.get_commands_file_name( js_idx=self.index, js_action_idx=js_action_idx, @@ -410,47 +474,74 @@ def get_commands_file_name(self, js_action_idx, shell=None): @property def task_insert_IDs(self): + """ + The insertion IDs of tasks in this jobscript. + """ return self._task_insert_IDs @property def task_actions(self): + """ + The IDs of actions of each task in this jobscript. + """ return self._task_actions @property def task_elements(self): + """ + The IDs of elements of each task in this jobscript. + """ return self._task_elements @property def EAR_ID(self): + """ + The array of EAR IDs. + """ return self._EAR_ID @property def all_EAR_IDs(self) -> List[int]: + """ + The IDs of all EARs in this jobscript. + """ return self.EAR_ID.flatten() @property @TimeIt.decorator def all_EARs(self) -> List: + """ + Description of EAR information for this jobscript. + """ if not self._all_EARs: self._all_EARs = self.workflow.get_EARs_from_IDs(self.all_EAR_IDs) return self._all_EARs @property def resources(self): + """ + The common resources that this jobscript requires. + """ return self._resources @property def task_loop_idx(self): + """ + The description of where various task loops are. + """ return self._task_loop_idx @property def dependencies(self): + """ + The dependency descriptor. + """ return self._dependencies @property @TimeIt.decorator def start_time(self): - """Get the first start time from all EARs.""" + """The first known start time of any EAR in this jobscript.""" if not self.is_submitted: return all_times = [i.start_time for i in self.all_EARs if i.start_time] @@ -462,7 +553,7 @@ def start_time(self): @property @TimeIt.decorator def end_time(self): - """Get the last end time from all EARs.""" + """The last known end time of any EAR in this jobscript.""" if not self.is_submitted: return all_times = [i.end_time for i in self.all_EARs if i.end_time] @@ -473,6 +564,9 @@ def end_time(self): @property def submit_time(self): + """ + When the jobscript was submitted, if known. + """ if self._submit_time_obj is None and self._submit_time: self._submit_time_obj = ( datetime.strptime(self._submit_time, self.workflow.ts_fmt) @@ -483,50 +577,86 @@ def submit_time(self): @property def submit_hostname(self): + """ + Where the jobscript was submitted, if known. + """ return self._submit_hostname @property def submit_machine(self): + """ + Description of what the jobscript was submitted to, if known. + """ return self._submit_machine @property def submit_cmdline(self): + """ + The command line used to do the commit, if known. + """ return self._submit_cmdline @property def scheduler_job_ID(self): + """ + The job ID from the scheduler, if known. + """ return self._scheduler_job_ID @property def process_ID(self): + """ + The process ID from direct execution, if known. + """ return self._process_ID @property def version_info(self): + """ + Version information about the execution environment (OS, etc). + """ return self._version_info @property def index(self): + """ + The index of this jobscript within its parent :py:class:`Submission`. + """ return self._index @property def submission(self): + """ + The parent submission. + """ return self._submission @property def workflow(self): + """ + The workflow this is all on behalf of. + """ return self.submission.workflow @property def num_actions(self): + """ + The number of actions in this jobscript. + """ return self.EAR_ID.shape[0] @property def num_elements(self): + """ + The number of elements in this jobscript. + """ return self.EAR_ID.shape[1] @property def is_array(self): + """ + Whether to generate an array job. + """ if self.scheduler_name == "direct": return False @@ -546,14 +676,23 @@ def is_array(self): @property def os_name(self) -> Union[str, None]: + """ + The name of the OS to use. + """ return self._os_name or self.resources.os_name @property def shell_name(self) -> Union[str, None]: + """ + The name of the shell to use. + """ return self._shell_name or self.resources.shell @property def scheduler_name(self) -> Union[str, None]: + """ + The name of the scheduler to use. + """ return self._scheduler_name or self.resources.scheduler def _get_submission_os_args(self): @@ -578,7 +717,7 @@ def _get_shell(self, os_name, shell_name, os_args=None, shell_args=None): @property def shell(self): - """Retrieve the shell object for submission.""" + """The shell for composing submission scripts.""" if self._shell_obj is None: self._shell_obj = self._get_shell( os_name=self.os_name, @@ -590,7 +729,7 @@ def shell(self): @property def scheduler(self): - """Retrieve the scheduler object for submission.""" + """The scheduler that submissions go to from this jobscript.""" if self._scheduler_obj is None: self._scheduler_obj = self.app.get_scheduler( scheduler_name=self.scheduler_name, @@ -601,52 +740,81 @@ def scheduler(self): @property def EAR_ID_file_name(self): + """ + The name of a file containing EAR IDs. + """ return f"js_{self.index}_EAR_IDs.txt" @property def element_run_dir_file_name(self): + """ + The name of a file containing run directory names. + """ return f"js_{self.index}_run_dirs.txt" @property def direct_stdout_file_name(self): - """For direct execution stdout.""" + """File for direct execution stdout.""" return f"js_{self.index}_stdout.log" @property def direct_stderr_file_name(self): - """For direct execution stderr.""" + """File for direct execution stderr.""" return f"js_{self.index}_stderr.log" @property def direct_win_pid_file_name(self): + """File for holding the direct execution PID.""" return f"js_{self.index}_pid.txt" @property def jobscript_name(self): + """The name of the jobscript file.""" return f"js_{self.index}{self.shell.JS_EXT}" @property def EAR_ID_file_path(self): + """ + The path to the file containing EAR IDs for this jobscript. + """ return self.submission.path / self.EAR_ID_file_name @property def element_run_dir_file_path(self): + """ + The path to the file containing run directory names for this jobscript. + """ return self.submission.path / self.element_run_dir_file_name @property def jobscript_path(self): + """ + The path to the file containing the jobscript file. + """ return self.submission.path / self.jobscript_name @property def direct_stdout_path(self): + """ + The path to the file containing the stdout from directly executed commands + for this jobscript. + """ return self.submission.path / self.direct_stdout_file_name @property def direct_stderr_path(self): + """ + The path to the file containing the stderr from directly executed commands + for this jobscript. + """ return self.submission.path / self.direct_stderr_file_name @property def direct_win_pid_file_path(self): + """ + The path to the file containing PIDs for directly executed commands for this + jobscript. Windows only. + """ return self.submission.path / self.direct_win_pid_file_name def _set_submit_time(self, submit_time: datetime) -> None: @@ -737,6 +905,9 @@ def _set_scheduler_name(self) -> None: ) def get_task_loop_idx_array(self): + """ + Get an array of task loop indices. + """ loop_idx = np.empty_like(self.EAR_ID) loop_idx[:] = np.array([i[2] for i in self.task_actions]).reshape( (len(self.task_actions), 1) @@ -916,6 +1087,9 @@ def write_jobscript( scheduler_name: Optional[str] = None, scheduler_args: Optional[Dict] = None, ): + """ + Write the jobscript to its file. + """ js_str = self.compose_jobscript( deps=deps, os_name=os_name, @@ -935,6 +1109,9 @@ def _get_EARs_arr(self): @TimeIt.decorator def make_artifact_dirs(self): + """ + Create the directories that will hold artifacts associated with this jobscript. + """ EARs_arr = self._get_EARs_arr() task_loop_idx_arr = self.get_task_loop_idx_array() @@ -1036,6 +1213,9 @@ def submit( scheduler_refs: Dict[int, (str, bool)], print_stdout: Optional[bool] = False, ) -> str: + """ + Submit the jobscript to the scheduler. + """ # map each dependency jobscript index to the JS ref (job/process ID) and if the # dependency is an array dependency: deps = {} @@ -1143,11 +1323,14 @@ def submit( @property def is_submitted(self): - """Return True if this jobscript has been submitted.""" + """Whether this jobscript has been submitted.""" return self.index in self.submission.submitted_jobscripts @property def scheduler_js_ref(self): + """ + The reference to the submitted job for the jobscript. + """ if isinstance(self.scheduler, Scheduler): return self.scheduler_job_ID else: @@ -1155,6 +1338,9 @@ def scheduler_js_ref(self): @property def scheduler_ref(self): + """ + The generalised scheduler reference descriptor. + """ out = {"js_refs": [self.scheduler_js_ref]} if not isinstance(self.scheduler, Scheduler): out["num_js_elements"] = self.num_elements @@ -1210,6 +1396,9 @@ def get_active_states( return out def cancel(self): + """ + Cancel this jobscript. + """ self.app.submission_logger.info( f"Cancelling jobscript {self.index} of submission {self.submission.index}" ) diff --git a/hpcflow/sdk/submission/jobscript_info.py b/hpcflow/sdk/submission/jobscript_info.py index 39fe5a3fe..e09896ec3 100644 --- a/hpcflow/sdk/submission/jobscript_info.py +++ b/hpcflow/sdk/submission/jobscript_info.py @@ -1,3 +1,7 @@ +""" +Jobscript state enumeration. +""" + import enum @@ -13,36 +17,42 @@ def __new__(cls, value, symbol, colour, doc=None): member.__doc__ = doc return member + #: Waiting for resource allocation. pending = ( 0, "○", "yellow", "Waiting for resource allocation.", ) + #: Waiting for one or more dependencies to finish. waiting = ( 1, "◊", "grey46", "Waiting for one or more dependencies to finish.", ) + #: Executing now. running = ( 2, "●", "dodger_blue1", "Executing now.", ) + #: Previously submitted but is no longer active. finished = ( 3, "■", "grey46", "Previously submitted but is no longer active.", ) + #: Cancelled by the user. cancelled = ( 4, "C", "red3", "Cancelled by the user.", ) + #: The scheduler reports an error state. errored = ( 5, "E", @@ -52,4 +62,7 @@ def __new__(cls, value, symbol, colour, doc=None): @property def rich_repr(self): + """ + Rich representation of this enumeration element. + """ return f"[{self.colour}]{self.symbol}[/{self.colour}]" diff --git a/hpcflow/sdk/submission/schedulers/__init__.py b/hpcflow/sdk/submission/schedulers/__init__.py index 63fec706d..84e2990f5 100644 --- a/hpcflow/sdk/submission/schedulers/__init__.py +++ b/hpcflow/sdk/submission/schedulers/__init__.py @@ -1,3 +1,7 @@ +""" +Job scheduler models. +""" + from pathlib import Path import sys import time @@ -5,7 +9,22 @@ class NullScheduler: + """ + Abstract base class for schedulers. + + Keyword Args + ------------ + shell_args: str + Arguments to pass to the shell. Pre-quoted. + shebang_args: str + Arguments to set on the shebang line. Pre-quoted. + options: dict + Options to the scheduler. + """ + + #: Default value for arguments to the shell. DEFAULT_SHELL_ARGS = "" + #: Default value for arguments on the shebang line. DEFAULT_SHEBANG_ARGS = "" def __init__( @@ -20,6 +39,9 @@ def __init__( @property def unique_properties(self): + """ + Unique properties, for hashing. + """ return (self.__class__.__name__,) def __eq__(self, other) -> bool: @@ -29,20 +51,52 @@ def __eq__(self, other) -> bool: return self.__dict__ == other.__dict__ def get_version_info(self): + """ + Get the version of the scheduler. + """ return {} def parse_submission_output(self, stdout: str) -> None: + """ + Parse the output from a submission to determine the submission ID. + """ return None @staticmethod def is_num_cores_supported(num_cores, core_range: List[int]): + """ + Test whether particular number of cores is supported in given range of cores. + """ step = core_range[1] if core_range[1] is not None else 1 upper = core_range[2] + 1 if core_range[2] is not None else sys.maxsize return num_cores in range(core_range[0], upper, step) class Scheduler(NullScheduler): + """ + Base class for schedulers that use a job submission system. + + Parameters + ---------- + submit_cmd: str + The submission command, if overridden from default. + show_cmd: str + The show command, if overridden from default. + del_cmd: str + The delete command, if overridden from default. + js_cmd: str + The job script command, if overridden from default. + login_nodes_cmd: str + The login nodes command, if overridden from default. + array_switch: str + The switch to enable array jobs, if overridden from default. + array_item_var: str + The variable for array items, if overridden from default. + """ + + #: Default command for logging into nodes. DEFAULT_LOGIN_NODES_CMD = None + #: Default pattern for matching the names of login nodes. DEFAULT_LOGIN_NODE_MATCH = "*login*" def __init__( @@ -72,6 +126,9 @@ def unique_properties(self): return (self.__class__.__name__, self.submit_cmd, self.show_cmd, self.del_cmd) def format_switch(self, switch): + """ + Format a particular switch to use the JS command. + """ return f"{self.js_cmd} {switch}" def is_jobscript_active(self, job_ID: str): @@ -79,6 +136,9 @@ def is_jobscript_active(self, job_ID: str): return bool(self.get_job_state_info([job_ID])) def wait_for_jobscripts(self, js_refs: List[Any]) -> None: + """ + Wait for jobscripts to update their state. + """ while js_refs: info = self.get_job_state_info(js_refs) print(info) diff --git a/hpcflow/sdk/submission/schedulers/direct.py b/hpcflow/sdk/submission/schedulers/direct.py index 77c6c6925..dff5ef72a 100644 --- a/hpcflow/sdk/submission/schedulers/direct.py +++ b/hpcflow/sdk/submission/schedulers/direct.py @@ -1,3 +1,7 @@ +""" +A direct job "scheduler" that just runs immediate subprocesses. +""" + from pathlib import Path import shutil import signal @@ -11,6 +15,22 @@ class DirectScheduler(NullScheduler): + """ + A direct scheduler, that just runs jobs immediately as direct subprocesses. + + The correct subclass (:py:class:`DirectPosix` or :py:class:`DirectWindows`) should + be used to create actual instances. + + Keyword Args + ------------ + shell_args: str + Arguments to pass to the shell. Pre-quoted. + shebang_args: str + Arguments to set on the shebang line. Pre-quoted. + options: dict + Options to the jobscript command. + """ + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -29,6 +49,9 @@ def get_submit_command( js_path: str, deps: List[Tuple], ) -> List[str]: + """ + Get the concrete submission command. + """ return shell.get_direct_submit_command(js_path) @staticmethod @@ -104,6 +127,10 @@ def cancel_jobs( js_refs: List[Tuple[int, List[str]]], jobscripts: List = None, ): + """ + Cancel some jobs. + """ + def callback(proc): try: js = js_proc_id[proc.pid] @@ -145,7 +172,21 @@ def is_jobscript_active(self, process_ID: int, process_cmdline: List[str]): class DirectPosix(DirectScheduler): + """ + A direct scheduler for POSIX systems. + + Keyword Args + ------------ + shell_args: str + Arguments to pass to the shell. Pre-quoted. + shebang_args: str + Arguments to set on the shebang line. Pre-quoted. + options: dict + Options to the jobscript command. + """ + _app_attr = "app" + #: Default shell. DEFAULT_SHELL_EXECUTABLE = "/bin/bash" def __init__(self, *args, **kwargs): @@ -153,7 +194,19 @@ def __init__(self, *args, **kwargs): class DirectWindows(DirectScheduler): + """ + A direct scheduler for Windows. + + Keyword Args + ------------ + shell_args: str + Arguments to pass to the shell. Pre-quoted. + options: dict + Options to the jobscript command. + """ + _app_attr = "app" + #: Default shell. DEFAULT_SHELL_EXECUTABLE = "powershell.exe" def __init__(self, *args, **kwargs): diff --git a/hpcflow/sdk/submission/schedulers/sge.py b/hpcflow/sdk/submission/schedulers/sge.py index 9e92631c2..81c01719e 100644 --- a/hpcflow/sdk/submission/schedulers/sge.py +++ b/hpcflow/sdk/submission/schedulers/sge.py @@ -1,3 +1,7 @@ +""" +An interface to SGE. +""" + from pathlib import Path import re from typing import Dict, List, Tuple @@ -15,6 +19,18 @@ class SGEPosix(Scheduler): """ + A scheduler that uses SGE. + + Keyword Args + ------------ + cwd_switch: str + Override of default switch to use to set the current working directory. + shell_args: str + Arguments to pass to the shell. Pre-quoted. + shebang_args: str + Arguments to set on the shebang line. Pre-quoted. + options: dict + Options to the jobscript command. Notes ----- @@ -29,17 +45,26 @@ class SGEPosix(Scheduler): _app_attr = "app" + #: Default args for shebang line. DEFAULT_SHEBANG_ARGS = "" + #: Default submission command. DEFAULT_SUBMIT_CMD = "qsub" + #: Default command to show the queue state. DEFAULT_SHOW_CMD = ["qstat"] + #: Default cancel command. DEFAULT_DEL_CMD = "qdel" + #: Default job control directive prefix. DEFAULT_JS_CMD = "#$" + #: Default prefix to enable array processing. DEFAULT_ARRAY_SWITCH = "-t" + #: Default shell variable with array ID. DEFAULT_ARRAY_ITEM_VAR = "SGE_TASK_ID" + #: Default switch to control CWD. DEFAULT_CWD_SWITCH = "-cwd" + #: Default command to get the login nodes. DEFAULT_LOGIN_NODES_CMD = ["qconf", "-sh"] - # maps scheduler states: + #: Maps scheduler state codes to :py:class:`JobscriptElementState` values. state_lookup = { "qw": JobscriptElementState.pending, "hq": JobscriptElementState.waiting, @@ -136,7 +161,7 @@ def get_login_nodes(self): nodes = stdout.strip().split("\n") return nodes - def format_core_request_lines(self, resources): + def _format_core_request_lines(self, resources): lns = [] if resources.num_cores > 1: lns.append( @@ -146,10 +171,10 @@ def format_core_request_lines(self, resources): lns.append(f"{self.js_cmd} -tc {resources.max_array_items}") return lns - def format_array_request(self, num_elements): + def _format_array_request(self, num_elements): return f"{self.js_cmd} {self.array_switch} 1-{num_elements}" - def format_std_stream_file_option_lines(self, is_array, sub_idx): + def _format_std_stream_file_option_lines(self, is_array, sub_idx): # note: we can't modify the file names base = f"./artifacts/submissions/{sub_idx}" return [ @@ -158,13 +183,16 @@ def format_std_stream_file_option_lines(self, is_array, sub_idx): ] def format_options(self, resources, num_elements, is_array, sub_idx): + """ + Format the options to the jobscript command. + """ opts = [] opts.append(self.format_switch(self.cwd_switch)) - opts.extend(self.format_core_request_lines(resources)) + opts.extend(self._format_core_request_lines(resources)) if is_array: - opts.append(self.format_array_request(num_elements)) + opts.append(self._format_array_request(num_elements)) - opts.extend(self.format_std_stream_file_option_lines(is_array, sub_idx)) + opts.extend(self._format_std_stream_file_option_lines(is_array, sub_idx)) for opt_k, opt_v in self.options.items(): if isinstance(opt_v, list): @@ -197,6 +225,13 @@ def get_submit_command( js_path: str, deps: List[Tuple], ) -> List[str]: + """ + Get the command to use to submit a job to the scheduler. + + Returns + ------- + List of argument words. + """ cmd = [self.submit_cmd, "-terse"] dep_job_IDs = [] @@ -285,6 +320,9 @@ def get_job_state_info( return info def cancel_jobs(self, js_refs: List[str], jobscripts: List = None): + """ + Cancel submitted jobs. + """ cmd = [self.del_cmd] + js_refs self.app.submission_logger.info( f"cancelling {self.__class__.__name__} jobscripts with command: {cmd}." diff --git a/hpcflow/sdk/submission/schedulers/slurm.py b/hpcflow/sdk/submission/schedulers/slurm.py index ee68f7892..0de233b24 100644 --- a/hpcflow/sdk/submission/schedulers/slurm.py +++ b/hpcflow/sdk/submission/schedulers/slurm.py @@ -1,3 +1,7 @@ +""" +An interface to SLURM. +""" + from pathlib import Path import subprocess import time @@ -18,12 +22,24 @@ class SlurmPosix(Scheduler): """ + A scheduler that uses SLURM. + + Keyword Args + ------------ + shell_args: str + Arguments to pass to the shell. Pre-quoted. + shebang_args: str + Arguments to set on the shebang line. Pre-quoted. + options: dict + Options to the jobscript command. Notes ----- - runs in current working directory by default [2] - # TODO: consider getting memory usage like: https://stackoverflow.com/a/44143229/5042280 + Todo + ---- + - consider getting memory usage like: https://stackoverflow.com/a/44143229/5042280 References ---------- @@ -34,16 +50,24 @@ class SlurmPosix(Scheduler): _app_attr = "app" + #: Default shell. DEFAULT_SHELL_EXECUTABLE = "/bin/bash" + #: Default args for shebang line. DEFAULT_SHEBANG_ARGS = "" + #: Default submission command. DEFAULT_SUBMIT_CMD = "sbatch" + #: Default command to show the queue state. DEFAULT_SHOW_CMD = ["squeue", "--me"] + #: Default cancel command. DEFAULT_DEL_CMD = "scancel" + #: Default job control directive prefix. DEFAULT_JS_CMD = "#SBATCH" + #: Default prefix to enable array processing. DEFAULT_ARRAY_SWITCH = "--array" + #: Default shell variable with array ID. DEFAULT_ARRAY_ITEM_VAR = "SLURM_ARRAY_TASK_ID" - # maps scheduler states: + #: Maps scheduler state codes to :py:class:`JobscriptElementState` values. state_lookup = { "PENDING": JobscriptElementState.pending, "RUNNING": JobscriptElementState.running, @@ -301,7 +325,7 @@ def process_resources(cls, resources, scheduler_config: Dict) -> None: if part_match: resources.SLURM_partition = part_match - def format_core_request_lines(self, resources): + def _format_core_request_lines(self, resources): lns = [] if resources.SLURM_partition: lns.append(f"{self.js_cmd} --partition {resources.SLURM_partition}") @@ -324,13 +348,13 @@ def format_core_request_lines(self, resources): return lns - def format_array_request(self, num_elements, resources): + def _format_array_request(self, num_elements, resources): # TODO: Slurm docs start indices at zero, why are we starting at one? # https://slurm.schedmd.com/sbatch.html#OPT_array max_str = f"%{resources.max_array_items}" if resources.max_array_items else "" return f"{self.js_cmd} {self.array_switch} 1-{num_elements}{max_str}" - def format_std_stream_file_option_lines(self, is_array, sub_idx): + def _format_std_stream_file_option_lines(self, is_array, sub_idx): base = r"%x_" if is_array: base += r"%A.%a" @@ -344,12 +368,15 @@ def format_std_stream_file_option_lines(self, is_array, sub_idx): ] def format_options(self, resources, num_elements, is_array, sub_idx): + """ + Format the options to the scheduler. + """ opts = [] - opts.extend(self.format_core_request_lines(resources)) + opts.extend(self._format_core_request_lines(resources)) if is_array: - opts.append(self.format_array_request(num_elements, resources)) + opts.append(self._format_array_request(num_elements, resources)) - opts.extend(self.format_std_stream_file_option_lines(is_array, sub_idx)) + opts.extend(self._format_std_stream_file_option_lines(is_array, sub_idx)) for opt_k, opt_v in self.options.items(): if isinstance(opt_v, list): @@ -387,6 +414,13 @@ def get_submit_command( js_path: str, deps: List[Tuple], ) -> List[str]: + """ + Get the command to use to submit a job to the scheduler. + + Returns + ------- + List of argument words. + """ cmd = [self.submit_cmd, "--parsable"] dep_cmd = [] @@ -528,6 +562,9 @@ def get_job_state_info( return info def cancel_jobs(self, js_refs: List[str], jobscripts: List = None): + """ + Cancel submitted jobs. + """ cmd = [self.del_cmd] + js_refs self.app.submission_logger.info( f"cancelling {self.__class__.__name__} jobscripts with command: {cmd}." diff --git a/hpcflow/sdk/submission/schedulers/utils.py b/hpcflow/sdk/submission/schedulers/utils.py index 55b367273..b3a0de5c8 100644 --- a/hpcflow/sdk/submission/schedulers/utils.py +++ b/hpcflow/sdk/submission/schedulers/utils.py @@ -1,3 +1,7 @@ +""" +Helper for running a subprocess. +""" + import subprocess diff --git a/hpcflow/sdk/submission/shells/__init__.py b/hpcflow/sdk/submission/shells/__init__.py index 16d38625b..4a1ac5f60 100644 --- a/hpcflow/sdk/submission/shells/__init__.py +++ b/hpcflow/sdk/submission/shells/__init__.py @@ -1,3 +1,6 @@ +""" +Adapters for various shells. +""" import os from typing import Dict, Optional @@ -7,6 +10,7 @@ from .bash import Bash, WSLBash from .powershell import WindowsPowerShell +#: All supported shells. ALL_SHELLS = { "bash": {"posix": Bash}, "powershell": {"nt": WindowsPowerShell}, @@ -14,7 +18,7 @@ "wsl": {"nt": WSLBash}, # TODO: cast this to wsl+bash in ResourceSpec? } -# used to set the default shell in the default config: +#: The default shell in the default config. DEFAULT_SHELL_NAMES = { "posix": "bash", "nt": "powershell", @@ -22,11 +26,17 @@ def get_supported_shells(os_name: Optional[str] = None) -> Dict[str, Shell]: + """ + Get shells supported on the current or given OS. + """ os_name = os_name or os.name return {k: v.get(os_name) for k, v in ALL_SHELLS.items() if v.get(os_name)} def get_shell(shell_name, os_name: Optional[str] = None, **kwargs) -> Shell: + """ + Get a shell interface with the given name for a given OS (or the current one). + """ # TODO: apply config default shell args? os_name = os_name or os.name diff --git a/hpcflow/sdk/submission/shells/base.py b/hpcflow/sdk/submission/shells/base.py index b87c3a65c..8667acdeb 100644 --- a/hpcflow/sdk/submission/shells/base.py +++ b/hpcflow/sdk/submission/shells/base.py @@ -1,3 +1,7 @@ +""" +Base model of a shell. +""" + from abc import ABC, abstractmethod from pathlib import Path from typing import Dict, List, Optional @@ -10,6 +14,12 @@ class Shell(ABC): bash on a POSIX OS, and provides snippets that are used to compose a jobscript for that combination. + Parameters + ---------- + executable: str + Which executable implements the shell. + os_args: + Arguments to pass to the shell. """ def __init__(self, executable=None, os_args=None): @@ -25,10 +35,16 @@ def __eq__(self, other) -> bool: @property def executable(self) -> List[str]: + """ + The executable to use plus any mandatory arguments. + """ return [self._executable] @property - def shebang_executable(self) -> List[str]: + def shebang_executable(self) -> str: + """ + The executable to use in a shebang line. + """ return self.executable def get_direct_submit_command(self, js_path) -> List[str]: @@ -40,6 +56,9 @@ def get_version_info(self, exclude_os: Optional[bool] = False) -> Dict: """Get shell and operating system information.""" def get_wait_command(self, workflow_app_alias: str, sub_idx: int, deps: Dict): + """ + Get the command to wait for a workflow. + """ if deps: return ( f'{workflow_app_alias} workflow $WK_PATH_ARG wait --jobscripts "{sub_idx}:' @@ -51,9 +70,15 @@ def get_wait_command(self, workflow_app_alias: str, sub_idx: int, deps: Dict): @staticmethod def process_app_invoc_executable(app_invoc_exe): + """ + Perform any post-processing of an application invocation command name. + """ return app_invoc_exe def process_JS_header_args(self, header_args: Dict) -> Dict: + """ + Process the application invocation key in the jobscript header arguments. + """ app_invoc = self.process_app_invoc_executable(header_args["app_invoc"][0]) if len(header_args["app_invoc"]) > 1: app_invoc += ' "' + header_args["app_invoc"][1] + '"' @@ -62,7 +87,13 @@ def process_JS_header_args(self, header_args: Dict) -> Dict: return header_args def prepare_JS_path(self, js_path: Path) -> str: + """ + Prepare the jobscript path for use. + """ return str(js_path) def prepare_element_run_dirs(self, run_dirs: List[List[Path]]) -> List[List[str]]: + """ + Prepare the element run directory names for use. + """ return [[str(j) for j in i] for i in run_dirs] diff --git a/hpcflow/sdk/submission/shells/bash.py b/hpcflow/sdk/submission/shells/bash.py index 6c6c5e12c..9b01b0819 100644 --- a/hpcflow/sdk/submission/shells/bash.py +++ b/hpcflow/sdk/submission/shells/bash.py @@ -1,3 +1,7 @@ +""" +Shell models based on the Bourne-Again Shell. +""" + from pathlib import Path import subprocess from textwrap import dedent, indent @@ -11,14 +15,22 @@ class Bash(Shell): - """Class to represent using bash on a POSIX OS to generate and submit a jobscript.""" + """ + Class to represent using bash on a POSIX OS to generate and submit a jobscript. + """ + #: Default for executable name. DEFAULT_EXE = "/bin/bash" + #: File extension for jobscripts. JS_EXT = ".sh" + #: Basic indent. JS_INDENT = " " + #: Indent for environment setup. JS_ENV_SETUP_INDENT = 2 * JS_INDENT + #: Template for the jobscript shebang line. JS_SHEBANG = """#!{shebang_executable} {shebang_args}""" + #: Template for the common part of the jobscript header. JS_HEADER = dedent( """\ {workflow_app_alias} () {{ @@ -39,6 +51,7 @@ class Bash(Shell): ELEM_RUN_DIR_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{element_run_dirs_file_path}" """ ) + #: Template for the jobscript header when scheduled. JS_SCHEDULER_HEADER = dedent( """\ {shebang} @@ -47,6 +60,7 @@ class Bash(Shell): {header} """ ) + #: Template for the jobscript header when directly executed. JS_DIRECT_HEADER = dedent( """\ {shebang} @@ -55,6 +69,7 @@ class Bash(Shell): {wait_command} """ ) + #: Template for the jobscript body. JS_MAIN = dedent( """\ elem_EAR_IDs=`sed "$((${{JS_elem_idx}} + 1))q;d" "$EAR_ID_FILE"` @@ -103,6 +118,7 @@ class Bash(Shell): done """ ) + #: Template for the element processing loop in a jobscript. JS_ELEMENT_LOOP = dedent( """\ for ((JS_elem_idx=0;JS_elem_idx<{num_elements};JS_elem_idx++)) @@ -112,6 +128,7 @@ class Bash(Shell): cd "$WK_PATH" """ ) + #: Template for the array handling code in a jobscript. JS_ELEMENT_ARRAY = dedent( """\ JS_elem_idx=$(({scheduler_array_item_var} - 1)) @@ -125,6 +142,9 @@ def __init__(self, *args, **kwargs): @property def linux_release_file(self): + """ + The name of the file describing the Linux version. + """ return self.os_args["linux_release_file"] def _get_OS_info_POSIX(self): @@ -169,6 +189,9 @@ def process_app_invoc_executable(app_invoc_exe): return app_invoc_exe def format_stream_assignment(self, shell_var_name, command): + """ + Produce code to assign the output of the command to a shell variable. + """ return f"{shell_var_name}=`{command}`" def format_save_parameter( @@ -180,6 +203,9 @@ def format_save_parameter( cmd_idx: int, stderr: bool, ): + """ + Produce code to save a parameter's value into the workflow persistent store. + """ # TODO: quote shell_var_name as well? e.g. if it's a white-space delimited list? # and test. stderr_str = " --stderr" if stderr else "" @@ -192,6 +218,9 @@ def format_save_parameter( ) def format_loop_check(self, workflow_app_alias: str, loop_name: str, run_ID: int): + """ + Produce code to check the looping status of part of a workflow. + """ return ( f"{workflow_app_alias} " f'internal workflow "$WK_PATH_ARG" check-loop ' @@ -248,8 +277,14 @@ def wrap_in_subshell(self, commands: str, abortable: bool) -> str: class WSLBash(Bash): + """ + A variant of bash that handles running under WSL on Windows. + """ + + #: Default name of the WSL interface executable. DEFAULT_WSL_EXE = "wsl" + #: Template for the common part of the jobscript header. JS_HEADER = Bash.JS_HEADER.replace( 'WK_PATH_ARG="$WK_PATH"', 'WK_PATH_ARG=`wslpath -m "$WK_PATH"`', diff --git a/hpcflow/sdk/submission/shells/os_version.py b/hpcflow/sdk/submission/shells/os_version.py index c5e165858..fc8260043 100644 --- a/hpcflow/sdk/submission/shells/os_version.py +++ b/hpcflow/sdk/submission/shells/os_version.py @@ -1,3 +1,7 @@ +""" +Operating system information discovery helpers. +""" + import os import platform import re @@ -8,6 +12,9 @@ def get_OS_info() -> Dict: + """ + Get basic operating system version info. + """ uname = platform.uname() return { "OS_name": uname.system, @@ -17,6 +24,9 @@ def get_OS_info() -> Dict: def get_OS_info_windows() -> Dict: + """ + Get operating system version info: Windows version. + """ return get_OS_info() @@ -26,15 +36,17 @@ def get_OS_info_POSIX( linux_release_file: Optional[str] = None, ) -> Dict: """ + Get operating system version info: POSIX version. + Parameters ---------- - WSL_executable + WSL_executable: Executable to run subprocess calls via WSL on Windows. - use_py - If True, use the `platform.uname` Python function to get the OS information. - Otherwise use subprocess to call `uname`. We set this to False when getting OS - info in WSL on Windows, since we need to call the WSL executable. - linux_release_file + use_py: + If True, use the :py:func:`platform.uname` Python function to get the OS + information. Otherwise use subprocess to call ``uname``. We set this to False + when getting OS info in WSL on Windows, since we need to call the WSL executable. + linux_release_file: If on Linux, record the name and version fields from this file. """ diff --git a/hpcflow/sdk/submission/shells/powershell.py b/hpcflow/sdk/submission/shells/powershell.py index 4659171a5..95b96725e 100644 --- a/hpcflow/sdk/submission/shells/powershell.py +++ b/hpcflow/sdk/submission/shells/powershell.py @@ -1,3 +1,7 @@ +""" +Shell models based on Microsoft PowerShell. +""" + import subprocess from textwrap import dedent, indent from typing import Dict, List, Optional @@ -11,12 +15,18 @@ class WindowsPowerShell(Shell): # TODO: add snippets that can be used in demo task schemas? + #: Default for executable name. DEFAULT_EXE = "powershell.exe" + #: File extension for jobscripts. JS_EXT = ".ps1" + #: Basic indent. JS_INDENT = " " + #: Indent for environment setup. JS_ENV_SETUP_INDENT = 2 * JS_INDENT + #: Template for the jobscript shebang line. JS_SHEBANG = "" + #: Template for the common part of the jobscript header. JS_HEADER = dedent( """\ function {workflow_app_alias} {{ @@ -60,6 +70,7 @@ class WindowsPowerShell(Shell): $ELEM_RUN_DIR_FILE = JoinMultiPath $WK_PATH artifacts submissions $SUB_IDX {element_run_dirs_file_path} """ ) + #: Template for the jobscript header when directly executed. JS_DIRECT_HEADER = dedent( """\ {shebang} @@ -68,6 +79,7 @@ class WindowsPowerShell(Shell): {wait_command} """ ) + #: Template for the jobscript body. JS_MAIN = dedent( """\ $elem_EAR_IDs = get_nth_line $EAR_ID_FILE $JS_elem_idx @@ -117,6 +129,7 @@ class WindowsPowerShell(Shell): }} """ ) + #: Template for the element processing loop in a jobscript. JS_ELEMENT_LOOP = dedent( """\ for ($JS_elem_idx = 0; $JS_elem_idx -lt {num_elements}; $JS_elem_idx += 1) {{ @@ -172,6 +185,9 @@ def process_app_invoc_executable(app_invoc_exe): return app_invoc_exe def format_stream_assignment(self, shell_var_name, command): + """ + Produce code to assign the output of the command to a shell variable. + """ return f"${shell_var_name} = {command}" def format_save_parameter( @@ -183,6 +199,9 @@ def format_save_parameter( cmd_idx: int, stderr: bool, ): + """ + Produce code to save a parameter's value into the workflow persistent store. + """ # TODO: quote shell_var_name as well? e.g. if it's a white-space delimited list? # and test. stderr_str = " --stderr" if stderr else "" @@ -195,6 +214,9 @@ def format_save_parameter( ) def format_loop_check(self, workflow_app_alias: str, loop_name: str, run_ID: int): + """ + Produce code to check the looping status of part of a workflow. + """ return ( f"{workflow_app_alias} " f"internal workflow $WK_PATH check-loop " diff --git a/hpcflow/sdk/submission/submission.py b/hpcflow/sdk/submission/submission.py index 849c6e20b..628f05433 100644 --- a/hpcflow/sdk/submission/submission.py +++ b/hpcflow/sdk/submission/submission.py @@ -1,3 +1,7 @@ +""" +A collection of submissions to a scheduler, generated from a workflow. +""" + from __future__ import annotations from collections import defaultdict @@ -24,6 +28,9 @@ def timedelta_format(td: timedelta) -> str: + """ + Convert time delta to string in standard form. + """ days, seconds = td.days, td.seconds hours = seconds // (60 * 60) seconds -= hours * (60 * 60) @@ -33,6 +40,9 @@ def timedelta_format(td: timedelta) -> str: def timedelta_parse(td_str: str) -> timedelta: + """ + Parse a string in standard form as a time delta. + """ days, other = td_str.split("-") days = int(days) hours, mins, secs = [int(i) for i in other.split(":")] @@ -40,12 +50,38 @@ def timedelta_parse(td_str: str) -> timedelta: class SubmissionStatus(enum.Enum): - PENDING = 0 # not yet submitted - SUBMITTED = 1 # all jobscripts submitted successfully - PARTIALLY_SUBMITTED = 2 # some jobscripts submitted successfully + """ + The overall status of a submission. + """ + + #: Not yet submitted. + PENDING = 0 + #: All jobscripts submitted successfully. + SUBMITTED = 1 + #: Some jobscripts submitted successfully. + PARTIALLY_SUBMITTED = 2 class Submission(JSONLike): + """ + A collection of jobscripts to be submitted to a scheduler. + + Parameters + ---------- + index: int + The index of this submission. + jobscripts: list[~hpcflow.app.Jobscript] + The jobscripts in the submission. + workflow: ~hpcflow.app.Workflow + The workflow this is part of. + submission_parts: dict + Description of submission parts. + JS_parallelism: bool + Whether to exploit jobscript parallelism. + environments: ~hpcflow.app.EnvironmentsList + The execution environments to use. + """ + _child_objects = ( ChildObjectSpec( name="jobscripts", @@ -77,6 +113,7 @@ def __init__( self._submission_parts_lst = None # assigned on first access; datetime objects if workflow: + #: The workflow this is part of. self.workflow = workflow self._set_parent_refs() @@ -156,14 +193,23 @@ def to_dict(self): @property def index(self) -> int: + """ + The index of this submission. + """ return self._index @property def environments(self) -> app.EnvironmentsList: + """ + The execution environments to use. + """ return self._environments @property def submission_parts(self) -> List[Dict]: + """ + Description of the parts of this submission. + """ if not self._submission_parts: return [] @@ -237,14 +283,23 @@ def end_time(self): @property def jobscripts(self) -> List: + """ + The jobscripts in this submission. + """ return self._jobscripts @property def JS_parallelism(self): + """ + Whether to exploit jobscript parallelism. + """ return self._JS_parallelism @property def workflow(self) -> List: + """ + The workflow this is part of. + """ return self._workflow @workflow.setter @@ -268,6 +323,9 @@ def outstanding_jobscripts(self) -> Tuple[int]: @property def status(self): + """ + The status of this submission. + """ if not self.submission_parts: return SubmissionStatus.PENDING else: @@ -278,6 +336,9 @@ def status(self): @property def needs_submit(self): + """ + Whether this submission needs a submit to be done. + """ return self.status in ( SubmissionStatus.PENDING, SubmissionStatus.PARTIALLY_SUBMITTED, @@ -285,19 +346,31 @@ def needs_submit(self): @property def path(self): + """ + The path to files associated with this submission. + """ return self.workflow.submissions_path / str(self.index) @property def all_EAR_IDs(self): + """ + The IDs of all EARs in this submission. + """ return [i for js in self.jobscripts for i in js.all_EAR_IDs] @property def all_EARs(self): + """ + All EARs in this this submission. + """ return [i for js in self.jobscripts for i in js.all_EARs] @property @TimeIt.decorator def EARs_by_elements(self): + """ + All EARs in this submission, grouped by element. + """ task_elem_EARs = defaultdict(lambda: defaultdict(list)) for i in self.all_EARs: task_elem_EARs[i.task.index][i.element.index].append(i) @@ -305,10 +378,16 @@ def EARs_by_elements(self): @property def abort_EARs_file_name(self): + """ + The name of a file describing what EARs have aborted. + """ return f"abort_EARs.txt" @property def abort_EARs_file_path(self): + """ + The path to the file describing what EARs have aborted in this submission. + """ return self.path / self.abort_EARs_file_name @TimeIt.decorator @@ -357,6 +436,9 @@ def get_unique_schedulers_of_jobscripts( Uniqueness is determines only by the `Scheduler.unique_properties` tuple. + Parameters + ---------- + jobscripts: list[~hpcflow.app.Jobscript] """ js_idx = [] schedulers = [] @@ -566,6 +648,9 @@ def submit( @TimeIt.decorator def cancel(self): + """ + Cancel the active jobs for this submission's jobscripts. + """ act_js = list(self.get_active_jobscripts()) if not act_js: print("No active jobscripts to cancel.") diff --git a/hpcflow/sdk/typing.py b/hpcflow/sdk/typing.py index f3d4cb7be..9f84c7a5b 100644 --- a/hpcflow/sdk/typing.py +++ b/hpcflow/sdk/typing.py @@ -1,9 +1,18 @@ +""" +Common type aliases. +""" from typing import Tuple, TypeVar from pathlib import Path +#: Type of a value that can be treated as a path. PathLike = TypeVar("PathLike", str, Path, None) # TODO: maybe don't need TypeVar? -# EAR: (task_insert_ID, element_idx, iteration_idx, action_idx, run_idx) +#: Type of an element index: +#: (task_insert_ID, element_idx) E_idx_type = Tuple[int, int] +#: Type of an element iteration index: +#: (task_insert_ID, element_idx, iteration_idx) EI_idx_type = Tuple[int, int, int] +#: Type of an element action run index: +#: (task_insert_ID, element_idx, iteration_idx, action_idx, run_idx) EAR_idx_type = Tuple[int, int, int, int, int]