diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3902461..dcea504 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,7 +106,8 @@ jobs: chmod a+x "$HOME/.cargo/bin/mdbook-linkcheck" - name: Add linkcheck configuration run: | - echo -e "[output.linkcheck]\nfollow-web-links=true" >> doc/book.toml + # echo -e "[output.linkcheck]\nfollow-web-links=true" >> doc/book.toml #TODO: enable web-link checks after row is public + echo -e "[output.linkcheck]\nfollow-web-links=false" >> doc/book.toml cat doc/book.toml - name: Build documentation run: mdbook build doc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3124cc1..f794679 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,3 +27,10 @@ repos: rev: v1.6.27 hooks: - id: actionlint +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.3.4' + hooks: + - id: ruff-format + - id: ruff + +# TODO: add fix-license-header diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..440c976 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,49 @@ +target-version = "py312" +line-length = 100 + +[lint] + +extend-select = [ + "A", + "B", + "D", + "E501", + "EM", + "I", + "ICN", + "ISC", + "N", + "NPY", + "PL", + "PT", + "RET", + "RUF", + "UP", + "W", +] + +ignore = [ + "N806", "N803", # Allow occasional use of uppercase variable and argument names (e.g. N). + "D107", # Do not document __init__ separately from the class. + "PLR09", # Allow "too many" statements/arguments/etc... + "N816", # Allow mixed case names like kT. + "RUF012", # Do not use typing hints. +] + +[lint.pydocstyle] +convention = "google" + +[lint.flake8-import-conventions] +# Prefer no import aliases +aliases = {} +# Always import hoomd and gsd without 'from' +banned-from = ["hoomd", "gsd"] + +# Ban standard import conventions and force common packages to be imported by their actual name. +[lint.flake8-import-conventions.banned-aliases] +"numpy" = ["np"] +"pandas" = ["pd"] +"matplotlib" = ["mpl"] + +[format] +quote-style = "single" diff --git a/Cargo.lock b/Cargo.lock index fd2fbdf..274864f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -451,6 +451,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys", +] + [[package]] name = "human_format" version = "1.1.0" @@ -754,6 +763,7 @@ dependencies = [ "clap-verbosity-flag", "console", "env_logger", + "home", "human_format", "indicatif", "indicatif-log-bridge", diff --git a/Cargo.toml b/Cargo.toml index 6f11e8b..1a6a84b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ clap = { version = "4.5.3", features = ["derive", "env"] } clap-verbosity-flag = "2.2.0" console = "0.15.8" env_logger = "0.11.3" +home = "0.5.9" human_format = "1.1.0" indicatif = "0.17.8" indicatif-log-bridge = "0.2.2" diff --git a/DESIGN.md b/DESIGN.md index 0ed9a73..cb16e73 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -51,11 +51,12 @@ Row is yet another workflow engine that automates the process of executing **act Ideas: * List scheduler jobs and show useful information. +* Cancel scheduler jobs specific to actions and/or directories. * Command to uncomplete an action for a set of directories. This would remove the product files and update the cache. -* Option for `scan` to clear the cache. This would allow users to discover - changed action names, changed products, and manually uncompleted actions. -* Require the use of a launcher when requesting more than one process? +* Some method to clear any cache (maybe this instead of uncomplete?). This would allow + users to discover changed action names, changed products, manually uncompleted + actions, and deal with corrupt cache files. ## Overview @@ -69,11 +70,12 @@ dispatches calls to the library. * `row` * `cli` - Top level command line commands. - * `cluster` - Read the cluster configuration file, determine the active cluster, and - make the settings available. - * `launcher` - TODO: Separate from cluster/scheduler? Or embedded within? - * `project` - Combine the workflow, state, and scheduler into one object and provide methods - that work with the project as a whole. + * `cluster` - Read the `clusters.toml` configuration file, determine the active + cluster, and make the settings available. + * `launcher` - Read the `launchers.toml` configuration file, provide code to construct + commands with launcher prefixes. + * `project` - Combine the workflow, state, and scheduler into one object and provide + methods that work with the project as a whole. * `scheduler` - Generic scheduler interface and implementations for shell and SLURM. * `state` - Row's internal state of the workspace. * `workflow` - Type defining workflow and utility functions to find and read it. @@ -91,7 +93,7 @@ executed, Row checks in the current working directory (and recursively in parent - The **workspace** - path - A static **value file** -- cluster specific +- cluster-specific `submit_options` - account - options - setup script @@ -106,7 +108,7 @@ executed, Row checks in the current working directory (and recursively in parent - threads_per_process - gpus_per_process - walltime (either per_submission or per_directory) - - Cluster specific + - Cluster- and action-specific `submit_options` - options - setup - partition @@ -140,8 +142,8 @@ Row maintains the state of the workflow in several files: * Cached copies of the user-provided static value file. * `completed.postcard` * Completion status for each **action**. -* `TODO: determine filename` - * The last submitted scheduler job ID for each **action**. +* `submitted.postcard` + * The last submitted job ID, referenced by action, directory, and cluster. When Row updates the state, it collects the completion staging files and updates the entries in the state accordingly. It also checks the scheduler for all known job IDs and removes any job IDs that @@ -170,9 +172,6 @@ resource usage and asks for confirmation before submitting the **job(s)** to the submitting, Row updates the **state** with a record of which scheduler **job** IDs were submitted for each **action**/**directory** combination. -When run in an interactive terminal, show a progress bar when there is more than one job to submit. -TODO: Also print the job IDs submitted? Or only the job IDs and no progress bar? - Provide a --dry-run option that shows the user what **job** script(s) would be submitted. End the remaining submission sequence on an error return value from the scheduler. Save the cache @@ -191,6 +190,20 @@ generated by user input. The group defining options **include** and **sort_by** use JSON pointer syntax. This allows users to select any element of their value when defining groups. +### Launcher configuration + +Launchers define prefixes that go in front of commands. These prefixes (e.g. +OMP_NUM_THREADS, srun) take arguments when the user requests certain resources. **Row** +provides built in support for OpenMP and MPI on the built-in clusters that **row +supports. Users can override these and provide new launchers in `launchers.toml`. + +Each launcher optionally emits an `executable`, and arguments for +* total number of processes +* threads per process +* gpus per process +when both the launcher defines such an argument and the user requests the relevant +resource. + ### Cluster configuration Row provides default configurations for many national HPC systems. Users can override these defaults @@ -198,10 +211,9 @@ or define new systems in `$HOME/.config/row/clusters.toml`. A single cluster defines: * name -* launcher - * TODO: determine how to define launchers -* autodetect - * TODO: determine how to autodetect clusters +* identify: one of + * by_environment: [string, string] + * always: bool * scheduler * partition (listed in priority order) * name diff --git a/README.md b/README.md index ec0698f..e87696c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ actions have been submitted on which directories so that you don't submit the sa twice. Once a job completes, subsequent actions become eligible allowing you to process your entire workflow to completion over many submissions. +The name is "row" as in "row, row, row your boat". + Notable features: * Support both arbitrary directories and [signac](https://signac.io) workspaces. * Execute actions via arbitrary shell commands. @@ -16,9 +18,11 @@ Notable features: * Execute groups in serial or parallel. * Schedule CPU and GPU resources. * Automatically determine the partition based on the batch job size. -* Includes configure for many national and university HPC systems. +* Built-in configurations for many national and university HPC systems. * Add custom cluster definitions for your resources. +TODO: better demo script to get output for README and row show documentation examples. + For example: ```bash > row show status @@ -29,7 +33,7 @@ two 0 200 800 1000 8K GPU-hours ```bash > row show directories --value "/value" -Directory Status Job /value +Directory Status Job ID /value dir1 submitted 1432876 0.9 dir2 submitted 1432876 0.8 dir3 submitted 1432876 0.7 @@ -41,5 +45,3 @@ dir6 completed 0.3 **Row** is a spiritual successor to [signac-flow](https://docs.signac.io/projects/flow/en/latest/). - -The name is "row" as in "row, row, row your boat". diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 5801f20..c7f2757 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -9,34 +9,45 @@ - [Hello, workflow!](guide/tutorial/hello.md) - [Managing multiple actions](guide/tutorial/multiple.md) - [Grouping directories](guide/tutorial/group.md) - - [Submitting jobs to a scheduler]() - - [Best practices for actions]() + - [Submitting jobs manually](guide/tutorial/scheduler.md) + - [Requesting resources with row](guide/tutorial/resources.md) + - [Submitting jobs with row](guide/tutorial/submit.md) - [Using row with Python and signac](guide/python/index.md) - [Working with signac projects](guide/python/signac.md) - [Writing action commands in Python](guide/python/actions.md) - [Concepts](guide/concepts/index.md) + - [Best practices](guide/concepts/best-practices.md) + - [Process parallelism](guide/concepts/process-parallelism.md) + - [Thread parallelism](guide/concepts/thread-parallelism.md) - [Directory status](guide/concepts/status.md) - - [The row cache](guide/concepts/cache.md) - [JSON pointers](guide/concepts/json-pointers.md) + - [The row cache](guide/concepts/cache.md) # Reference - [row](row/index.md) - [init](row/init.md) - - [show status](row/show-status.md) - - [show directories](row/show-directories.md) - [submit](row/submit.md) + - [show](row/show/index.md) + - [show status](row/show/status.md) + - [show directories](row/show/directories.md) + - [show cluster](row/show/cluster.md) + - [show launchers](row/show/launchers.md) - [scan](row/scan.md) - [uncomplete](row/uncomplete.md) - [`workflow.toml`](workflow/index.md) - [workspace](workflow/workspace.md) - - [cluster](workflow/cluster.md) + - [submit_options](workflow/submit-options.md) - [action](workflow/action/index.md) - [group](workflow/action/group.md) - [resources](workflow/action/resources.md) - - [cluster](workflow/action/cluster.md) + - [submit_options](workflow/action/submit-options.md) - [`clusters.toml`](clusters/index.md) + - [cluster](clusters/cluster.md) - [Built-in clusters](clusters/built-in.md) +- [`launchers.toml`](launchers/index.md) + - [Launcher configuration](launchers/launcher.md) + - [Built-in launchers](launchers/built-in.md) - [Environment variables](env.md) # Appendix diff --git a/doc/src/clusters/built-in.md b/doc/src/clusters/built-in.md index 17dc351..68dc76e 100644 --- a/doc/src/clusters/built-in.md +++ b/doc/src/clusters/built-in.md @@ -1,3 +1,50 @@ # Built-in clusters -TODO: Write this document. +**Row** includes built-in support for the following clusters. + +## Anvil (Purdue) + +[Anvil documentation](https://www.rcac.purdue.edu/knowledge/anvil). + +**Row** automatically selects from the following partitions: +* `shared` +* `wholenode` +* `gpu` + +Other partitions may be selected manually. + +There is no need to set `--mem-per-*` options on Anvil as the cluster automatically +chooses the largest amount of memory available per core by default. + +## Delta (NCSA) + +[Delta documentation](https://docs.ncsa.illinois.edu/systems/delta). + +**Row** automatically selects from the following partitions: +* `cpu` +* `gpuA100x4` + +Other partitions may be selected manually. + +Delta jobs default to a small amount of memory per core. **Row** inserts `--mem-per-cpu` +or `--mem-per-gpu` to select the maximum amount of memory possible that allows full-node +jobs and does not incur extra charges. + +## Great Lakes (University of Michigan) + +[Great Lakes documentation](https://arc.umich.edu/greatlakes/). + +**Row** automatically selects from the following partitions: +* `standard` +* `gpu_mig40,gpu` +* `gpu` + +Other partitions may be selected manually. + +Great Lakes jobs default to a small amount of memory per core. **Row** inserts +`--mem-per-cpu` or `--mem-per-gpu` to select the maximum amount of memory possible that +allows full-node jobs and does not incur extra charges. + +> Note: The `gpu_mig40,gpu` partition is selected only when there is one GPU per job. +> This is a combination of 2 partitions which decreases queue wait time due to the +> larger number of nodes that can run your job. diff --git a/doc/src/clusters/cluster.md b/doc/src/clusters/cluster.md new file mode 100644 index 0000000..2c087f9 --- /dev/null +++ b/doc/src/clusters/cluster.md @@ -0,0 +1,145 @@ +# cluster + +An element in `[[cluster]]` is a **table** that defines the configuration of a single +cluster. + +For example: +```toml +[[cluster]] +name = "cluster1" +identify.by_environment = ["CLUSTER_NAME", "cluster1"] +scheduler = "slurm" +[[cluster.partition]] +name = "shared" +maximum_cpus_per_job = 127 +maximum_gpus_per_job = 0 +[[cluster.partition]] +name = "gpu-shared" +minimum_gpus_per_job = 1 +[[cluster.partition]] +name = "compute" +require_cpus_multiple_of = 128 +maximum_gpus_per_job = 0 +[[cluster.partition]] +name = "debug" +maximum_gpus_per_job = 0 +prevent_auto_select = true +``` + +## name + +`cluster.name`: **string** - The name of the cluster. + +## identify + +`cluster.identify`: **table** - Set a condition to identify when **row** is executing +on this cluster. The table **must** have one of the following keys: + +* `by_environment`: **array** of two strings - Identify the cluster when the environment + variable `by_environment[0]` is set and equal to `by_environment[1]`. +* `always`: **bool** - Set to `true` to always identify this cluster. When `false`, + this cluster can only be chosen by an explicit `--cluster` option. + +> Note: The *first* cluster in the list that sets `identify.always = true` will prevent +> any later cluster from being identified. + +## scheduler + +`cluster.scheduler`: **string** - Set the job scheduler to use on this cluster. Must +be one of: + +* `"slurm"` +* `"bash"` + +## partition + +`cluster.partition`: **array** of **tables** - Define the scheduler partitions that +**row** may select from when submitting jobs. **Row** will check the partitions in the +order provided and choose the *first* partition where the job matches all the +provided conditions. All conditions are optional. + +### name + +`cluster.partition.name`: **string** - The name of the partition as it should be passed +to the cluster batch submission command. + +### maximum_cpus_per_job + +`cluster.partition.maximum_cpus_per_job`: **integer** - The maximum number of CPUs that +can be used by a single job on this partition: +```plaintext +total_cpus <= maximum_cpus_per_job +``` + +### require_cpus_multiple_of + +`cluster.partition.require_cpus_multiple_of`: **integer** - All jobs submitted to this +partition **must** use an integer multiple of the given number of cpus: +``` +total_cpus % require_cpus_multiple_of == 0 +``` + +### memory_per_cpu + +`cluster.partition.memory_per_cpu`: **string** - CPU Jobs submitted to this partition +will pass this option to the scheduler. For example SLURM schedulers will set +`--mem-per-cpu=`. + +### cpus_per_node + +`cluster.partition.cpus_per_node`: **string** - Number of CPUs per node. When +`cpus_per_node` is not set, **row** will ask the scheduler to schedule only a given +number of tasks. In this case, some schedulers are free to spread tasks among any +number of nodes (for example, shared partitions on Slurm schedulers). + +When `cpus_per_node` is set, **row** will request the minimal number of nodes needed +to satisfy `n_nodes * cpus_per_node >= total_cpus`. This may result in longer queue +times, but will lead to more stable performance for users. + +Set `cpus_per_node` only when all nodes in the partition have the same number of CPUs. + +### minimum_gpus_per_job + +`cluster.partition.minimum_gpus_per_job`: **integer** - The minimum number of gpus that +must be used by a single job on this partition: +```plaintext +total_gpus >= minimum_gpus_per_job +``` + +### maximum_gpus_per_job + +`cluster.partition.maximum_gpus_per_job`: **integer** - The maximum number of gpus that +can be used by a single job on this partition: +```plaintext +total_gpus <= maximum_gpus_per_job +``` + +### require_gpus_multiple_of + +`cluster.partition.require_gpus_multiple_of`: **integer** - All jobs submitted to this +partition **must** use an integer multiple of the given number of gpus: +``` +total_gpus % require_gpus_multiple_of == 0 +``` + +### memory_per_gpu + +`cluster.partition.memory_per_gpu`: **string** - GPU Jobs submitted to this partition +will pass this option to the scheduler. For example SLURM schedulers will set +`--mem-per-gpu=`. + +### gpus_per_node + +`cluster.partition.gpus_per_node`: **string** - Number of GPUs per node. Like +`cpus_per_node` but used on jobs that request GPUs. + +### prevent_auto_select + +`cluster.partition.prevent_auto_select`: **boolean** - Set to true to prevent row from +automatically selecting this partition. + +### account_suffix + +`cluster.partition.account_suffix`: **string** - Set to provide an account suffix +when submitting jobs to this partition. Useful when clusters define separate +`aacount-cpu` and `account-gpu` accounts. diff --git a/doc/src/clusters/index.md b/doc/src/clusters/index.md index 20aadb9..cccc058 100644 --- a/doc/src/clusters/index.md +++ b/doc/src/clusters/index.md @@ -1,3 +1,30 @@ # `clusters.toml` -TODO: Write this document. +**Row** includes [built-in cluster configurations](built-in.md) for a variety of +national and university HPC resources. You can override these and add new clusters in +the file `$HOME/.config/row/clusters.toml`. Each cluster includes a *name*, a method to +*identify* the cluster, the type of *scheduler*, and details on the *partitions*. +See [cluster configuration](cluster.md) for the full specification. + +The configuration defines the clusters in an *array*: +```toml +[[cluster]] +name = "cluster1" +# ... + +[[cluster]] +name = "cluster2" +# ... +``` + +User-provided clusters in `$HOME/.config/row/clusters.toml` are placed first in the +array. + +## Cluster identification + +On startup, **row** iterates over the array of clusters in order. If `--cluster` is not +set, **row** checks the `identify` condition in the configuration. If `--cluster` is +set, **row** checks to see if the name matches. + +> Note: **Row** uses the *first* such match. To override a built-in, your configuration +> should include a cluster by the same name and `identify` condition. diff --git a/doc/src/developers/testing.md b/doc/src/developers/testing.md index 1240962..eca87af 100644 --- a/doc/src/developers/testing.md +++ b/doc/src/developers/testing.md @@ -8,15 +8,26 @@ cargo test ``` in the source directory to execute the unit and integration tests. +All tests must be marked either `#[serial]` or `#[parallel]` explicitly. Some serial +tests set environment variables and/or the current working directory, which may conflict +with any test that is automatically run concurrently. Check for this with: +```bash +rg --multiline "#\[test\]\n *fn" +``` +(see the [saftey discussion](https://doc.rust-lang.org/std/env/fn.set_var.html) in +`std::env` for details. + ## Cluster-specific tests -TODO: Develop a strategy to test that both cluster auto-detection and the generated -jobs function correctly. +The file `validate/validate.py` in the source code repository provides a full suite of +tests to ensure that jobs are submitted correctly on clusters. The file docstring +describes how to run the tests. ## Tutorial tests -Tutorial scripts should be testable. Write scripts using mdBook's anchor feature to -include [portions of files](https://rust-lang.github.io/mdBook/format/mdbook.html) in -the documentation as needed. This way, the tutorial can be tested by executing the -script. This type of testing validates that the script *runs*, not that it produces -the correct output. +The tutorial scripts in `doc/src/guide/*.sh` are runnable. These are described in the +documentation using mdBook's anchor feature to include +[portions of files](https://rust-lang.github.io/mdBook/format/mdbook.html) in the +documentation as needed. This way, the tutorial can be tested by executing the script. +This type of testing validates that the script *runs*, not that it produces the correct +output. Developers should manually check the tutorial script output as needed. diff --git a/doc/src/env.md b/doc/src/env.md index 14b0f96..0222864 100644 --- a/doc/src/env.md +++ b/doc/src/env.md @@ -1,3 +1,16 @@ # Environment variables -TODO: Write this document +> Note: Environment variables that influence the execution of **row** are documented in +> [the command line options](row/index.md). + +**Row** sets the following environment variables in generated job scripts: + +| Environment variable | Value | +|----------------------|-------| +| `ACTION_CLUSTER` | Name of the cluster the action is executing on. | +| `ACTION_NAME` | The name of the action that is executing. | +| `ACTION_PROCESSES` | The total number of processes that this action uses. | +| `ACTION_WALLTIME_IN_MINUTES` | The requested job walltime in minutes. | +| `ACTION_PROCESSES_PER_DIRECTORY` | Set to the value of `action.resources.processes_per_directory`. Unset when `processes_per_submission`.| +| `ACTION_THREADS_PER_PROCESS` | Set to the value of `action.resources.threads_per_process`. Unset when `threads_per_process` is omitted. | +| `ACTION_GPUS_PER_PROCESS` | Set to the value of `action.resources.gpus_per_process`. Unset when `gpus_per_process` is omitted. | diff --git a/doc/src/guide/concepts/best-practices.md b/doc/src/guide/concepts/best-practices.md new file mode 100644 index 0000000..2731a94 --- /dev/null +++ b/doc/src/guide/concepts/best-practices.md @@ -0,0 +1,47 @@ +# Best practices + +Follow these guidelines to use **row** effectively. + +## Exit actions early when products already exist. + +There are some cases where **row** may fail to identify when your action completes: + +* Software exits with an unrecoverable error. +* Your job exceeds its walltime and is killed. +* And many others... + +To ensure that your action executes as intended, you should **check for the existence +of product files** when your action starts and **exit immediately** when they already +exist. This way, resubmitting an already completed job will not needlessly recompute +results or overwrite files you intended to keep. + +## Write to temporary files and move them to the final product location. + +For example, say `products = ["output.dat"]`. Write to `output.dat.in_progress` +while your calculation executes. Once the action is fully complete, *move* +`output.dat.in_progress` to `output.dat`. + +If you wrote directly to `output.dat`, **row** might identify your computation as +**complete** right after it starts. This pattern also allows you to *continue* running +one calculation over several job submissions. Move the output file to its final location +only after the final submission completes the calculation. + +## Group directories whenever possible, but not to an extreme degree. + +The **scheduler** does an excellent job handling the queue. However, there is some +overhead and the scheduler can only process so many jobs at a time. Your cluster may +even limit how many jobs you are allowed to queue. So please don't submit thousands of +jobs at a time to your cluster. You can improve your workflow's throughput by grouping +directories together into a smaller number of jobs. + +Group jobs that execute quickly in serial with `processes.per_submission` and +`walltime.per_directory`. After a given job has waited in the queue, it can process many +directories before exiting. Limit group sizes so that the total wall time of the job +remains reasonable. + +Group jobs that take a longer time in parallel using MPI partitions, +`processes.per_directory` and `walltime.per_submission`. Limit the group sizes to a +relatively small fraction of the cluster (*except on Leadership class machines*). +Huge parallel jobs may wait a long time in queue before starting. Experiment with the +`group.maximum_size` value and find a good job size (in number of nodes) that balances +queue time vs. scheduler overhead. diff --git a/doc/src/guide/concepts/process-parallelism.md b/doc/src/guide/concepts/process-parallelism.md new file mode 100644 index 0000000..e72cb1b --- /dev/null +++ b/doc/src/guide/concepts/process-parallelism.md @@ -0,0 +1,28 @@ +# Process parallelism + +In **row**, a *process* is one of many copies of an executable program. Copies may +(or may not) execute on different physical **compute nodes**. + +Neither **row** nor the **job scheduler** can execute more than **one process per +job**. When you request more than one **process** (via `processes.per_directory` or +`processes.per_submission`), you must pair it with a **launcher** that can execute those +processes: e.g. `launcher = ["mpi"]`. + +> In other words: The **scheduler** reserves enough **compute nodes** to satisfy +> the requested resources, but the **launcher** is responsible for executing those +> **processes**. + +At this time **MPI** is the only **process** launcher that **row** supports. You can +configure additional launchers in [`launchers.toml`](../../launchers/index.md) if your +cluster and application use a different launcher. + +Use **MPI** parallelism to launch: +* MPI-enabled applications on one directory (`processes.per_submission = N`, + `group.maximum_size = 1`). +* MPI-enabled applications on many directories in serial + (`processes.per_submission = N`). +* Serial applications on many directories in parallel (`processes.per_directory = 1`). + For example, use `mpi4py` and execute Python functions on directories indexed by rank. +* MPI-enable applications on many directories in parallel + (`processes.per_directory = N`). Instruct your application to *partition* the MPI + communicator. diff --git a/doc/src/guide/concepts/status.md b/doc/src/guide/concepts/status.md index 20866fb..2928b6c 100644 --- a/doc/src/guide/concepts/status.md +++ b/doc/src/guide/concepts/status.md @@ -3,11 +3,13 @@ For each action, each directory in the workspace that matches the action's [include condition](../../workflow/action/group.md#include) has a single status: -* **Completed** directories are those where all - [products](../../workflow/action/index.md#products) are present. -* **Submitted** directories have been submitted to the scheduler and currently remain - queued or are running. -* **Eligible** directories are those where all - [previous actions](../../workflow/action/index.md#previous_actions) have been - completed. -* **Waiting** directories are none of the above. +| Status | Description | +|--------|-------------| +| **Completed** | Directories where all [products](../../workflow/action/index.md#products) are present. | +| **Submitted** | Directories that been submitted to the scheduler and currently remain queued or are running. | +| **Eligible** | Directories where all [previous actions](../../workflow/action/index.md#previous_actions) are **completed**. | +| **Waiting** | None of the above. | + +Each directory may have only **one** status, evaluated in the order listed above. +For example, a directory will be **completed** if all of its products are present, +*even when a submitted job is still in queue*. diff --git a/doc/src/guide/concepts/thread-parallelism.md b/doc/src/guide/concepts/thread-parallelism.md new file mode 100644 index 0000000..5e86e03 --- /dev/null +++ b/doc/src/guide/concepts/thread-parallelism.md @@ -0,0 +1,25 @@ +# Thread parallelism + +In **row**, each **process** may execute many **threads** in parallel. The +**scheduler** gives each **thread** a dedicated **CPU core** on the *same physical* +**CPU node** as the host **process**. + +If you are familiar with operating system concepts, **row**/**scheduler** **threads** +may be realized both by **OS threads** and **OS processes**. Request +`threads_per_process` to schedule resources when your command uses: + +* OpenMP. +* a library that spawns threads, such as *some* **numpy** builds. +* operating system threads in general. +* the Python **multiprocessing** library. +* or otherwise executes many processes or threads (e.g. `make`, `ninja`). + +## Passing the number of threads to your application/library + +When launching OpenMP applications, set `launchers = ["openmp"]` and **row** will +set the `OMP_NUM_THREADS` environment variable accordingly. + +For all other cases, refer to the documentation of your application or library. Most +provide some way to set the number of threads/processes. Use the environment variable +`ACTION_THREADS_PER_PROCESS` to ensure that the number of executed threads matches that +requested. diff --git a/doc/src/guide/python/actions.md b/doc/src/guide/python/actions.md index 26e2e94..d19cd71 100644 --- a/doc/src/guide/python/actions.md +++ b/doc/src/guide/python/actions.md @@ -2,14 +2,14 @@ In **row**, actions execute arbitrary **shell commands**. When your action is **Python** code, you must structure that code so that it is a command line tool -that takes directories as arguments. There are many ways you can achieve that. +that takes directories as arguments. There are many ways you can achieve this goal. This guide will show you how to structure all of your actions in a single file: `actions.py`. This layout is inspired by **row's** predecessor: **signac-flow** and its `project.py`. -> Note: Also check out [migrating from signac-flow](../../signac-flow.md) if you are -> migrating from **signac-flow** to **row**. +> Note: If you are familiar with **signac-fow**, see +> [migrating from signac-flow](../../signac-flow.md) for many helpful tips. To demonstrate the structure of a project, let's build a workflow that computes the sum of squares. The focus of this guide is on structure and best practices. You need to @@ -48,10 +48,10 @@ Now, create a file `actions.py` with the contents: {{#include actions.py}} ``` -It defines each **action** as a function with the same name. These functions take an -array of jobs as an argument: `def square(*jobs)` and `def sum(*jobs)`. The `if __name__ -== "__main__":` block parses the command line arguments, forms an array of signac jobs -and calls the requested **action** function. +This file defines each **action** as a function with the same name. These functions take +an array of jobs as an argument: `def square(*jobs)` and `def compute_sum(*jobs)`. The +`if __name__ == "__main__":` block parses the command line arguments, forms an array of +signac jobs and calls the requested **action** function. > Note: This example demonstrates looping over directories in **serial**. However, this > structure also gives you the power to choose **serial** or **parallel** execution. @@ -84,7 +84,7 @@ threads, ...). ## Execute the workflow -Now, submit the square action: +Now, submit the *square* action: ```bash {{#include signac.sh:submit_square}} ``` @@ -95,7 +95,7 @@ Proceed? [Y/n]: y [1/1] Submitting action 'square' on directory 04bb77c1bbbb40e55ab9eb22d4c88447 and 9 more (0 seconds). ``` -Next, submit the sum action: +Next, submit the *compute_sum* action: ```bash {{#include signac.sh:submit_sum}} ``` @@ -103,7 +103,7 @@ and you should see: ```plaintext Submitting 1 job that may cost up to 0 CPU-hours. Proceed? [Y/n]: y -[1/1] Submitting action 'sum' on directory 04bb77c1bbbb40e55ab9eb22d4c88447 and 9 more (0 seconds). +[1/1] Submitting action 'compute_sum' on directory 04bb77c1bbbb40e55ab9eb22d4c88447 and 9 more (0 seconds). 285 ``` @@ -126,7 +126,7 @@ to your `workflow.toml` file. > Note: You may write functions that take only one job `def action(job)` without > modifying the given implementation of `__main__`. However, you will need to set > `action.group.maximum_size = 1` or use `{directory}` to ensure that `action.py` is -> given on a single directory. If you implement your code using arrays, you can use +> given a single directory. If you implement your code using arrays, you can use > **row's** grouping functionality to your benefit. ## Next steps diff --git a/doc/src/guide/python/actions.py b/doc/src/guide/python/actions.py index 39dfeec..a6a0935 100644 --- a/doc/src/guide/python/actions.py +++ b/doc/src/guide/python/actions.py @@ -1,6 +1,10 @@ -import signac -import os +"""Implement actions.""" + import argparse +import os + +import signac + def square(*jobs): """Implement the square action. @@ -10,35 +14,36 @@ def square(*jobs): """ for job in jobs: # If the product already exists, there is no work to do. - if job.isfile("square.out"): + if job.isfile('square.out'): continue # Open a temporary file so that the action is not completed early or on error. - with open(job.fn("square.out.in_progress"), "w") as file: - x = job.cached_statepoint["x"] - file.write(f"{x**2}") + with open(job.fn('square.out.in_progress'), 'w') as file: + x = job.cached_statepoint['x'] + file.write(f'{x**2}') # Done! Rename the temporary file to the product file. - os.rename(job.fn("square.out.in_progress"), job.fn("square.out")) + os.rename(job.fn('square.out.in_progress'), job.fn('square.out')) + -def sum(*jobs): - """Implement the sum action. +def compute_sum(*jobs): + """Implement the compute_sum action. Prints the sum of `square.out` from each job directory. """ total = 0 for job in jobs: - with open(job.fn("square.out")) as file: + with open(job.fn('square.out')) as file: total += int(file.read()) print(total) -if __name__ == "__main__": +if __name__ == '__main__': # Parse the command line arguments: python action.py --action [DIRECTORIES] parser = argparse.ArgumentParser() - parser.add_argument("--action", required=True) - parser.add_argument("directories", nargs="+") + parser.add_argument('--action', required=True) + parser.add_argument('directories', nargs='+') args = parser.parse_args() # Open the signac jobs diff --git a/doc/src/guide/python/populate_workspace.py b/doc/src/guide/python/populate_workspace.py index 076e7be..0a76e00 100644 --- a/doc/src/guide/python/populate_workspace.py +++ b/doc/src/guide/python/populate_workspace.py @@ -1,3 +1,5 @@ +"""Populate the workspace.""" + import signac N = 10 diff --git a/doc/src/guide/python/signac-workflow.toml b/doc/src/guide/python/signac-workflow.toml index 5382493..950f4f5 100644 --- a/doc/src/guide/python/signac-workflow.toml +++ b/doc/src/guide/python/signac-workflow.toml @@ -8,7 +8,7 @@ products = ["square.out"] resources.walltime.per_directory = "00:00:01" [[action]] -name = "sum" +name = "compute_sum" command = "python actions.py --action $ACTION_NAME {directories}" previous_actions = ["square"] resources.walltime.per_directory = "00:00:01" diff --git a/doc/src/guide/tutorial/hello.md b/doc/src/guide/tutorial/hello.md index 2af90db..3694449 100644 --- a/doc/src/guide/tutorial/hello.md +++ b/doc/src/guide/tutorial/hello.md @@ -77,4 +77,5 @@ you can find a way to submit this workflow only on `directory2`. ## Next steps You have created your first **row** workflow and executed it! The next section of this -tutorial will show you how to +tutorial will show you how to configure one **action** that will execute after another +action **completes**. diff --git a/doc/src/guide/tutorial/resources.md b/doc/src/guide/tutorial/resources.md new file mode 100644 index 0000000..bc3a126 --- /dev/null +++ b/doc/src/guide/tutorial/resources.md @@ -0,0 +1,128 @@ +# Requesting resources with row + +## Overview + +This section shows how you can use **row** to automatically generate **job scripts** +that request the **resources** your actions need to execute. This guide cannot +anticipate what codes you use, so it demonstrates commonly used patterns without +providing fully working examples. + +> Note: For a complete description, see +[resources in workflow.toml](../../workflow/action/resources.md). + +## Execute directories on 1 CPU in serial + +When you execute a script on a **group** of directories on 1 CPU in serial, request 1 +task *per job submission* (`processes.per_submission`) and provide the total time needed +to process a single directory in *HH:MM:SS* format (`walltime.per_directory`). + +```toml +[[action]] +name = "action" +command = serial_command {directory} + +[action.resources] +processes.per_submission = 1 +walltime.per_directory = "00:10:00" +``` + +When submitting a given **group**, **row** will compute the total `--time` request +from `walltime.per_submission * group_size`. + +## Execute a threaded (or multiprocessing) computation on 8 CPU cores + +For commands that execute with multiple threads (or multiple processes *on the same +node*), request `threads_per_process`. + +```toml +[[action]] +name = "action" +command = threaded_command {directory} + +[action.resources] +processes.per_submission = 1 +threads_per_process = 8 +walltime.per_directory = "00:10:00" +``` + +## Execute with OpenMP parallelism + +The same as above, but this example will place `OMP_NUM_THREADS=` +before the command: + +```toml +[[action]] +name = "action" +command = threaded_command {directory} +launchers = ["openmp"] + +[action.resources] +processes.per_submission = 1 +threads_per_process = 8 +walltime.per_directory = "00:10:00" +``` + + +# Execute MPI parallel calculations + +To launch MPI enabled applications, request more than one *process* and request the +`"mpi"` launcher. `launchers = ["mpi"]` will add the appropriate MPI launcher prefix +before your command (e.g. `srun --ntasks 16 parallel_application $directory`). + +```toml +[[action]] +name = "action" +command = parallel_application {directory} +launchers = ["mpi"] + +[action.resources] +processes.per_submission = 16 +walltime.per_directory = "04:00:00" +``` + +> Note: You should **not** manually insert `srun`, `mpirun` or other launcher commands. +> Use `launchers = ["mpi"]`. Configure [`launchers.toml`](../../launchers/index.md) +> if the default does not function correctly on your system. + +# Process many directories in parallel with MPI + +Structure your action script to split the MPI communicator and execute on each directory +based on the partition index. + +Unlike in previous examples, this one needs requests 4 ranks *per directory* +(`processes.per_directory`). These calculations run in parallel, so the walltime is +fixed *per submission* (`walltime.per_submission`). +```toml +[[action]] +name = "action" +command = partitioned_application {directories} +launchers = ["mpi"] + +[action.resources] +processes.per_directory = 4 +walltime.per_submission = "01:00:00" +``` + +## Execute a GPU accelerated application + +Request `gpus_per_process` to allocate a GPU node. + +```toml +[[action]] +name = "action" +command = gpu_application {directory} + +[action.resources] +processes.per_submission = 1 +gpus_per_process = 1 +walltime.per_directory = "08:00:00" +``` + +> Note: You can of course combine processes, threads, and GPUs all in the same +> submission, provided you **know** that your application will make full use of all +> requested resources. + +## Next steps + +You now have some idea how to instruct **row** to generate resource requests, you are +ready to **submit** your jobs to the **cluster**. diff --git a/doc/src/guide/tutorial/scheduler.md b/doc/src/guide/tutorial/scheduler.md new file mode 100644 index 0000000..63dd87d --- /dev/null +++ b/doc/src/guide/tutorial/scheduler.md @@ -0,0 +1,155 @@ +# Submitting jobs manually + +## Overview + +This section gives background information on **clusters**, **nodes**, and +**job schedulers**. It also outlines your responsibilities when using **row** on a +shared resource. + +## Clusters + +If you are interested in using **row**, you probably have access to a **cluster** where +you can execute the **jobs** in your workflows. **Row** is a tool that makes it easy to +generate *thousands* of **jobs**. Please use it responsibly. + +
+DO NOT ASSUME that the jobs that row generates are always correct. It is +YOUR RESPONSIBILITY to understand the contents of a proper job +and validate what row generates before submitting a large number of them. +If not, you could easily burn through your entire allocation with jobs that cost 100 +times what you expected them to. +
+ +With that warning out of the way, let's cover some of the basics you need to know. + +> Note: This guide is generic and covers only the topics directly related to **row**. +> You can find more information in your **cluster's** documentation. + +## Login and compute nodes + +**Clusters** are large groups of computers called **nodes**. When you use log in to +a **cluster**, you are given direct access to a **login node**. A typical cluster +might have 2-4 **login nodes**. Login nodes are **SHARED RESOURCES** that many others +actively use. You should use **login nodes** to edit text files, submit jobs, check +on job status, and *maybe* compile source code. In general, you should restrict your +**login node** usage to commands that will execute and complete *immediately* (or within +a minute). + +You should execute everything that takes longer than a minute or otherwise uses +extensive resources on one or more **compute nodes**. Typical clusters have *thousands* +of compute nodes. + +## Job scheduler + +The **job scheduler** controls access to the **compute nodes**. It ensures that each +**job** gets *exclusive* access to the resources it needs to execute. To see what jobs +are currently scheduled, run +```bash +squeue +``` +on a **login node**. + +> Note: This guide assumes your cluster uses Slurm. Refer to your cluster's +> documentation for equivalent commands if it uses a different scheduler. + +You will likely see some **PENDING** and **RUNNING** jobs. **RUNNING** jobs have been +assigned a number of (possibly fractional) **compute nodes** and are currently executing +on those resources. **PENDING** jobs are waiting for the **resources** that they request +to become available. + +## Submitting a job + +You should understand how to submit a job manually before you use **row** to automate +the process. Start with a "Hello, world" job. Place this text in a file called `job.sh`: +```bash +#!/bin/bash +#SBATCH --ntasks=1 +#SBATCH --time=1 + +echo "Hello, World!" +taskset -cp $$ +``` + +The first line of the script tells Slurm that this is a bash script. The next two are +options that will be processed by the scheduler: +* `--ntasks=1` requests that the **job scheduler** allocate *at least* 1 CPU core (it + *may* allocate and charge your account for more, see below). +* `--time=1` indicates that the script will execute in 1 minute or less. + +The last two lines are the body of our script. This example prints "Hello, World!" +and then the list of CPU cores the **job** is allowed to execute on. + +To submit the **job** to the **scheduler**, execute: +```bash +sbatch job.sh +``` + +> Note: Check the documentation for your cluster before submitting this job. If +> `sbatch` reported an error, you may also need to set `--account`, `--partition`, or +> other options. + +When `sbatch` successfully submits, it will inform you of the **job's ID**. You can +monitor the status of the **job** with: +```bash +squeue --me +``` + +The **job** will show first in the `PENDING` state. Once there is a **compute node** +available with the requested resources, the **scheduler** will start the job executing +on that **node**. `squeue` will then report that the job is `RUNNING`. It should +complete after a few moments, at which point `squeue` will no longer list the job. + +At this time, you should see a file `slurm-.out` appear in your current +directory. Inspect its contents to see the output of the script. For example: +``` +Hello, World! +pid 830675's current affinity list: 99 +``` + +> Note: If you see more than one number in the affinity list (e.g. 0-127), then the +> **scheduler** gave your job access to more CPU cores than `--ntasks=1` asks for. +> This may be because your **cluster** allocates **whole nodes** to jobs. Refer to +> your **cluster's** documentation to see specific details on how jobs are allocated +> to nodes and charged for resource usage. Remember, it is **YOUR RESPONSIBILITY** (not +> **row's**) to understand whether `--ntasks=1` costs 1 CPU-hour per hour or more (e.g. +> 128) CPU-hours per hour. If your cluster lacks a *shared* partition, then you need to +> structure your **actions** and **groups** in such a way to use all the cores you are +> given or else the resources are wasted. + +## Requesting resources + +There are many types of resources that you can request in a job script. One is time. +The above example requested 1 minute (`--time=1`). The `--time` option is a promise +to the **scheduler** that your job will complete in less than the given time. The +**scheduler** will use this information to efficiently plan other jobs to run after +yours. If your job is still running after the specified time limit, the **scheduler** +will terminate your job. + +Another resource you can request is more CPU cores. For example, add `--cpus-per-task=4` +to the above script: +```bash +#!/bin/bash +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --time=1 + +echo "Hello, World!" +taskset -cp $$ +``` + +Submit this script and see if the output is what you expect. + +You can also request GPUs, memory, licenses, and others. In the next section, you will +learn how to use **row** to automatically generate job scripts that request **CPUs**, +**GPUs**, and **time**. You can set +[`custom` submit options](../../workflow/submit-options.md) to request others. + +Most **clusters** also have separate **partitions** (requested with +`--partition=` for certain resources (e.g. GPU). See your **cluster's** +documentation for details. + +## Next steps + +Now that you know all about **compute nodes** and **job schedulers**, you can now learn +how to define these resource requests in `workflow.toml` so that **row** can generate +appropriate **job scripts**. diff --git a/doc/src/guide/tutorial/submit.md b/doc/src/guide/tutorial/submit.md new file mode 100644 index 0000000..5478ecb --- /dev/null +++ b/doc/src/guide/tutorial/submit.md @@ -0,0 +1,144 @@ +# Submitting jobs with row + +## Overview + +This section explains how to **submit** jobs to the **scheduler** with **row**. + +## Preliminary steps + +**Row** has built-in support for a number of [clusters](../../clusters/built-in.md): +* Anvil (Purdue) +* Delta (NCSA) +* Great Lakes (University of Michigan) + +You can skip to the [next heading](#checking-your-job-script) if you are using one of +these clusters. + +If not, then you need to create one or two configuration files that describe your +cluster and its launchers. + +* [`$HOME/.config/row/clusters.toml`](../../clusters/index.md) gives your cluster + a name, instructions on how to identify it, and lists the partitions your cluster + provides. +* [`$HOME/.config/row/launchers.toml`](../../launchers/index.md) defines how the + launcher command prefixes (e.g. MPI, OpenMP) expand. The default for MPI is to use + `srun`. If this doesn't work on your cluster, write `launchers.toml` to use a + different command and/or options. + +Many clusters have separate **partitions** for different resources (e.g. shared, whole +node, GPU, etc...). Your final script must request the correct `--partition` to execute +the command and charge accounts properly. `clusters.toml` describes rules by which +**row** automatically selects partitions when it generates job scripts. + +> Note: Feel free to ask on the +> [discussion board](https://github.com/glotzerlab/row/discussions) if you need help +> writing configuration files for your cluster. + +Check that the output of `row show cluster` and `row show launchers` is what you expect +before continuing. + +## Checking your job script + +For demonstration purposes, this guide will continue using the +[Hello, workflow](hello.md) example. In fact, you already learned how to submit jobs +in that section. + +However, you should *always* check that the job script is correct before you **submit** +on a **cluster**. The `--dry-run` option prints the submission script (or scripts) +instead of submitting with `sbatch`: +```plaintext +row submit --dry-run +``` +Remember, **YOU ARE RESPONSIBLE** for the content of the scripts that you submit. +Make sure that the script is requesting the correct resources and is routed to the +correct **partition**. + +For example, the example workflow might generate a job script like this on Anvil: +``` +#!/bin/bash +#SBATCH --job-name=hello-directory0+2 +#SBATCH --partition=shared +#SBATCH --ntasks=1 +#SBATCH --time=180 + +directories=( +'directory0' +'directory1' +'directory2' +) + +export ACTION_CLUSTER="anvil" +export ACTION_NAME="hello" +export ACTION_PROCESSES="1" +export ACTION_WALLTIME_IN_MINUTES="180" + +trap 'printf %s\\n "${directories[@]}" | /home/x-joaander/.cargo/bin/row scan --no-progress -a hello - || exit 3' EXIT +for directory in "${directories[@]}" +do + echo "Hello, $directory!" || { >&2 echo "[ERROR row::action] Error executing command."; exit 2; } +done +``` +Notice the selection of 1 task on the `shared` **partition**. This is correct for Anvil, +where the `shared` **partition** allows jobs smaller than one node and charges based +on the number of CPU cores quested. + +> Note: If you are using **row** on one of the built-in clusters, then **row** should +> always select the correct partition for your jobs. If you find it does not, please +> open an [issue](https://github.com/glotzerlab/row/issues). + +### Submitting jobs + +When you are *sure* that the **job script** is correct, submit it with: +``` +row submit +``` + +> If your cluster does not default to the correct account, you can set it in +> `workflow.toml`: +> ```toml +> [submit_options] +> .account = "" +> ``` + +### The submitted status + +**Row** tracks the **Job IDs** that it submits. Every time you execute `row show status` +(or just about any `row` command), it will execute `squeue` in the background to see +which jobs are still **submitted** (in any state). + +Use the `row show` family of commands to query details about submitted jobs. +For the `hello` workflow: +```bash +row show status +``` +will show: +```plaintext +Action Completed Submitted Eligible Waiting Remaining cost +hello 0 3 0 0 3 CPU-hours +``` + +Similarly, +```bash +row show directories hello +``` +will show something like: +``` +Directory Status Job ID +directory0 submitted anvil/5044933 +directory1 submitted anvil/5044933 +directory2 submitted anvil/5044933 +``` + +`row submit` is safe to use while submitted jobs remain in the queue. **Submitted** +directories are not eligible for execution, so `row submit` will not submit them again. + +Wait a moment for the job to finish executing (you can verify with `squeue --me`). +Then `row show status` should indicate that the jobs are *eligible* once +more (recall that the hello example creates no products, so it will never *complete*). + +## Next steps + +Now you know how to use all the features of **row**. You are ready to deploy it and +*responsibly* execute jobs on thousands of directories in your workflows. Read on in the +next section if you would like to learn how to use **signac** to manage your workspace +and/or write your **action** commands in **Python**. diff --git a/doc/src/launchers/built-in.md b/doc/src/launchers/built-in.md new file mode 100644 index 0000000..63a8c3c --- /dev/null +++ b/doc/src/launchers/built-in.md @@ -0,0 +1,13 @@ +# Built-in launchers + +**Row** includes built-in support for OpenMP and MPI via the launchers `"openmp"` +and `"mpi"`. These have been tested on the [built-in clusters](../clusters/built-in.md). +You may need to add new configurations for your specific cluster or adjust the `none` +launcher to match your system. Execute [`row show launchers`](../row/show/launchers.md) +to see the current launcher configuration. + +When using OpenMP/MPI hybrid applications, place `"openmp"` first in the list of +launchers (`launchers = ["openmp", "mpi"]`) to generate the appropriate command: +``` +OMP_NUM_THREADS=T srun --ntasks=N --cpus-per-task=T command $directory +``` diff --git a/doc/src/launchers/index.md b/doc/src/launchers/index.md new file mode 100644 index 0000000..14715ca --- /dev/null +++ b/doc/src/launchers/index.md @@ -0,0 +1,45 @@ +# `launchers.toml` + +**Row** includes [built-in launchers](built-in.md) to enable OpenMP and MPI on the +[built-in clusters](../clusters/built-in.md). You can override these configurations +and add new launchers in the file `$HOME/.config/row/launchers.toml`. It defines how +each **launcher** expands into a **command prefix**, with the possibility for specific +settings on each [**cluster**](../clusters/index.md). For example, an +[**action**](../workflow/action/index.md) with the configuration: +```toml +[[action]] +name = "action" +command = "command {directory}" +launchers = ["launcher1", "launcher2"] +``` +will expand to: +```plaintext + command $directory || {{ ... handle errors }} +``` +in the submission script. + +The prefix is a function of the job's [**resources**](../workflow/action/resources.md) +and the size of the [**group**](../workflow/action/group.md) in the current submission. +The section [Launcher configuration](launcher.md) details how this prefix is +constructed. + +## Default launcher configuration + +The **default** configuration will be used when there is no cluster-specific +configuration for the currently active cluster. Every launcher **must** have a +default configuration. Create a new launcher by creating a table named ``.default`` in `launchers.toml`. For example: +```toml +[launcher1.default] +# launcher1's default configuration +``` + +## Cluster-specific launcher configuration + +Define a launcher configuration specific to a cluster in the table +`.`, where `` is one of the cluster names in +[`clusters.toml`](../clusters/index.md). For example: +```toml +[launcher1.none] +# launcher1's configuration for the cluster `none`. +``` diff --git a/doc/src/launchers/launcher.md b/doc/src/launchers/launcher.md new file mode 100644 index 0000000..3e61fff --- /dev/null +++ b/doc/src/launchers/launcher.md @@ -0,0 +1,48 @@ +# Launcher configuration + +Each launcher configuration may set any (or none) of the following keys. The command +prefix constructed from this configuration will be: +```plaintext +{launcher.executable} [option1] [option2] ... +``` + +See [Built-in launchers](built-in.md) for examples. + +## executable + +`..executable`: **string** - Set the launcher's executable. May +be omitted. + +## gpus_per_process + +`..gpus_per_process`: **string** + +When `launcher.gpus_per_process` *and* `resources.gpus_per_process` are both +set, add the following option to the launcher prefix: +```plaintext +{launcher.gpus_per_process}{resource.gpus_per_process} +``` + +## processes + +`..processes`: **string** + +When `launcher.processes` is set, add the following option to the launcher prefix: +```plaintext +{launcher.processes}{total_processes} +``` +where `total_processes` is `n_directories * resources.processes.per_directory` or +`resources.processes.per_submission` depending on the resource configuration. + +It is an error when `total_processes > 1` and the action requests *no* launchers that +set `processes`. + +## threads_per_process + +`..threads_per_process`: **string** + +When `launcher.threads_per_process` *and* `resources.threads_per_process` are both +set, add the following option to the launcher prefix: +```plaintext +{launcher.threads_per_process}{resource.threads_per_process} +``` diff --git a/doc/src/row/index.md b/doc/src/row/index.md index 9f9db5e..ce20f3c 100644 --- a/doc/src/row/index.md +++ b/doc/src/row/index.md @@ -7,16 +7,15 @@ row [OPTIONS] `` must be one of: * [`init`](init.md) -* [`show status`](show-status.md) -* [`show directories`](show-directories.md) +* [`show`](show/index.md) * [`submit`](submit.md) * [`scan`](scan.md) * [`uncomplete`](uncomplete.md)
You should execute only one instance of row at a time for a given project. -Row maintains a cache and concurrent invocations may corrupt it. -The scan command is the one exception to this rule. +Row maintains a cache and concurrent invocations may corrupt it. The +scan command is excepted from this rule.
## `[OPTIONS]` @@ -41,7 +40,7 @@ default. ### `--cluster` Set the name of the cluster to check for submitted jobs. By default, **row** autodetects -the cluster based on the rules in [clusters.toml](../clusters/index.md). Set the +the cluster based on the rules in [`clusters.toml`](../clusters/index.md). Set the environment variable `ROW_CLUSTER` to change this default. **Row** always shows the count of submitted jobs from *all* clusters known in the diff --git a/doc/src/row/show/cluster.md b/doc/src/row/show/cluster.md new file mode 100644 index 0000000..3fd1cfb --- /dev/null +++ b/doc/src/row/show/cluster.md @@ -0,0 +1,32 @@ +# show cluster + +Usage: +```bash +row show cluster [OPTIONS] +``` + +Print the [current cluster configuration](../../clusters/index.md) (or for the cluster +given in `--cluster`). + +Example output: +``` +name = "none" +scheduler = "bash" + +[identify] +always = true + +[[partition]] +name = "none" +prevent_auto_select = false +``` + +## `[OPTIONS]` + +### `--all` + +Show the configuration of all clusters: both user-defined and built-in. + +### `--name` + +Show only the cluster's name. diff --git a/doc/src/row/show-directories.md b/doc/src/row/show/directories.md similarity index 80% rename from doc/src/row/show-directories.md rename to doc/src/row/show/directories.md index 6c0778d..d50fdfc 100644 --- a/doc/src/row/show-directories.md +++ b/doc/src/row/show/directories.md @@ -18,15 +18,15 @@ dir6 completed 0.3 ``` `row show directories` lists each selected directory with its -[status](../guide/concepts/status.md) and scheduler job ID (when submitted) for the +[status](../../guide/concepts/status.md) and scheduler job ID (when submitted) for the given ``. You can also show elements from the directory's value, accessed by -[JSON pointer](../guide/concepts/json-pointers.md). Blank lines separate -[groups](../workflow/action/group.md). +[JSON pointer](../../guide/concepts/json-pointers.md). Blank lines separate +[groups](../../workflow/action/group.md). ## `[DIRECTORIES]` List these specific directories. By default, **row** shows all directories that match -the action's [include condition](../workflow/action/group.md#include) +the action's [include condition](../../workflow/action/group.md#include) For example: ```bash row show directories action dir1 dir2 dir3 diff --git a/doc/src/row/show/index.md b/doc/src/row/show/index.md new file mode 100644 index 0000000..86fd120 --- /dev/null +++ b/doc/src/row/show/index.md @@ -0,0 +1,9 @@ +# show + +`row show ` commands display information about **row's** configuration or the +workflow. + +* [`status`](status.md) +* [`directories`](directories.md) +* [`cluster`](cluster.md) +* [`launchers`](launchers.md) diff --git a/doc/src/row/show/launchers.md b/doc/src/row/show/launchers.md new file mode 100644 index 0000000..c1db970 --- /dev/null +++ b/doc/src/row/show/launchers.md @@ -0,0 +1,28 @@ +# show launchers + +Usage: +```bash +row show launchers [OPTIONS] +``` + +Print the [launchers](../../launchers/index.md) defined for the current cluster (or the +cluster given in `--cluster`). + +Example output: +``` +[mpi] +executable = "mpirun" +processes = "-n " + +[openmp] +threads_per_process = "OMP_NUM_THREADS=" +``` + +This includes the user-provided launchers in [`launchers.toml`](../../launchers/index.md) +and the built-in launchers (or the user-provided overrides). + +## `[OPTIONS]` + +### `--all` + +Show the launcher configurations for all clusters. diff --git a/doc/src/row/show-status.md b/doc/src/row/show/status.md similarity index 90% rename from doc/src/row/show-status.md rename to doc/src/row/show/status.md index da97314..ccadfed 100644 --- a/doc/src/row/show-status.md +++ b/doc/src/row/show/status.md @@ -13,10 +13,10 @@ two 0 200 800 1000 8K GPU-hours ``` For each action, the summary details the number of directories in each -[status](../guide/concepts/status.md). +[status](../../guide/concepts/status.md). `row show status` also estimates the remaining cost in either CPU-hours or GPU-hours based on the number of submitted, eligible, and waiting jobs and the -[resources used by the action](../workflow/action/resources.md). +[resources used by the action](../../workflow/action/resources.md). ## `[DIRECTORIES]` diff --git a/doc/src/signac-flow.md b/doc/src/signac-flow.md index 2b326ee..ce9915f 100644 --- a/doc/src/signac-flow.md +++ b/doc/src/signac-flow.md @@ -19,8 +19,8 @@ Concepts: Commands: | flow | row | |------|-----| -| `project.py status` | [`row show status`](row/show-status.md) | -| `project.py status --detailed` | [`row show directories `](row/show-directories.md) | +| `project.py status` | [`row show status`](row/show/status.md) | +| `project.py status --detailed` | [`row show directories `](row/show/directories.md) | | `project.py run` | [`row submit --cluster=none`](row/submit.md) | | `project.py run --parallel` | A command may execute groups in parallel. | | `project.py exec ...` | Execute your action's command in the shell. | @@ -58,4 +58,4 @@ Execution: | directives: `np`, `ngpu`, `omp_num_threads`, `walltime` | [resources](workflow/action/resources.md) in `workflow.toml` | | directives: Launch with MPI | [`launchers`](workflow/action/index.md#launchers) `= ["mpi"]` | | directives: Launch with OpenMP | [`launchers`](workflow/action/index.md#launchers) `= ["openmp"]` | -| template job script: `script.sh` | [`cluster`](workflow/cluster.md) in `workflow.toml` | +| template job script: `script.sh` | [`submit_options`](workflow/submit-options.md) in `workflow.toml` | diff --git a/doc/src/workflow/action/cluster.md b/doc/src/workflow/action/cluster.md deleted file mode 100644 index aa0c9e8..0000000 --- a/doc/src/workflow/action/cluster.md +++ /dev/null @@ -1,11 +0,0 @@ -# cluster - -`action.cluster`: **table** - Override the global cluster-specific parameters with -values specific to this action. Any key that can be set in the global -[`cluster.`](../cluster.md) can be overridden in `action.cluster.`. - -Example: -```toml -[action.cluster.anvil] -setup = "echo Executing action three on anvil..." -``` diff --git a/doc/src/workflow/action/group.md b/doc/src/workflow/action/group.md index d2a7b15..e4109d0 100644 --- a/doc/src/workflow/action/group.md +++ b/doc/src/workflow/action/group.md @@ -16,7 +16,7 @@ reverse_sort = true > Note: You may omit `[action.group]` entirely. -Execute [`row show directories `](../../row/show-directories.md) to display the +Execute [`row show directories `](../../row/show/directories.md) to display the groups of directories included in a given action. ## include diff --git a/doc/src/workflow/action/index.md b/doc/src/workflow/action/index.md index e2db266..d48b8d8 100644 --- a/doc/src/workflow/action/index.md +++ b/doc/src/workflow/action/index.md @@ -64,7 +64,7 @@ command = "echo Message && python action.py {directory}" command. A launcher is a prefix placed before the command in the submission script. The cluster configuration [`clusters.toml`](../../clusters/index.md) defines what launchers are available on each cluster and how they are invoked. The example for `action_two` -above (`launchers = ["openmpi", "mpi"]`) would expand into something like: +above (`launchers = ["openmp", "mpi"]`) would expand into something like: ```bash OM_NUM_THREADS=4 srun --ntasks=128 --cpus-per-task=4 python action_two.py ... ``` diff --git a/doc/src/workflow/action/submit-options.md b/doc/src/workflow/action/submit-options.md new file mode 100644 index 0000000..c4ccd5c --- /dev/null +++ b/doc/src/workflow/action/submit-options.md @@ -0,0 +1,12 @@ +# submit_options + +`action.submit_options`: **table** - Override the global cluster-specific +submission options with values specific to this action. Any key that can be set +in the global [`submit_options.`](../submit-options.md) can be overridden in +`action.submit_options.`. + +Example: +```toml +[action.submit_options.cluster1] +setup = "echo Executing action on cluster1..." +``` diff --git a/doc/src/workflow/cluster.md b/doc/src/workflow/cluster.md deleted file mode 100644 index 3e4443b..0000000 --- a/doc/src/workflow/cluster.md +++ /dev/null @@ -1,57 +0,0 @@ -# cluster - -The `cluster` table sets the default cluster-specific parameters for all *actions*. You -can set action-specific cluster parameters in [`action.cluster`](action/cluster.md). -Keys in `cluster` must be one of the named clusters defined in -[`clusters.toml`](../clusters/index.md). - -Example: -```toml -[cluster.delta] -account = "my_account" -setup = """ -module reset -module load cuda -""" -options = ["--mem-per-cpu=1g"] -partition = "shared" - -[cluster.anvil] -account = "other_account" -setup = "module load openmpi" -``` - -> Note: You may omit `[cluster]` entirely. - -## account - -`cluster..account`: **string** - Submit cluster jobs to this account on cluster -``. When you omit `account`, **row** does not add the `--account=` line to the -submission script. - -## setup - -`cluster..setup`: **string** - Lines to include in the submission script on -cluster ``. The setup is executed *before* the action's command. You may omit -`setup` to leave this portion of the script blank. - -## options - -`cluster..options`: **array** of **strings** - List of additional command line -options to pass to the batch submission script on cluster ``. For example. -`options = ["--mem-per-cpu=2g"]` will add the line -``` -#SBATCH --mem-per-cpu=2g -``` -to the top of a SLURM submission script. When you omit `options`, it defaults to an -empty array. - -## partition - -`cluster..partition`: **string** - Force the use of a particular partition -when submitting jobs to the queue on cluster `. When omitted, **row** -will automatically determine the correct partition based on the configuration in -[`clusters.toml`](../clusters/index.md). - -> Note: You should almost always omit `partition`. Set it *only* when you need a -> specialty partition that is not automatically selected. diff --git a/doc/src/workflow/index.md b/doc/src/workflow/index.md index 405c78a..9b2dbd9 100644 --- a/doc/src/workflow/index.md +++ b/doc/src/workflow/index.md @@ -1,7 +1,7 @@ # workflow.toml The file `workflow.toml` defines the [workspace](workspace.md), -[actions](action/index.md), and [cluster specific parameters](cluster.md). Place +[actions](action/index.md), and [submission options](submit-options.md). Place `workflow.toml` in a directory to identify it as a **row** *project*. The [`row` command line tool](../row/index.md) will identify the current project by finding `workflow.toml` in the current working directory or any parent directory, diff --git a/doc/src/workflow/submit-options.md b/doc/src/workflow/submit-options.md new file mode 100644 index 0000000..0464344 --- /dev/null +++ b/doc/src/workflow/submit-options.md @@ -0,0 +1,57 @@ +# submit_options + +The `submit_options` table sets the default cluster-specific submission options for all +*actions*. You can set action-specific submission options in +[`action.submit_options`](action/submit-options.md). Keys in `submit_options` must be +one of the named clusters defined in [`clusters.toml`](../clusters/index.md). + +Example: +```toml +[submit_options.cluster1] +account = "my_account" +setup = """ +module reset +module load cuda +""" +custom = ["--mail-user=user@example.com"] +partition = "shared" + +[submit_options.cluster2] +account = "other_account" +setup = "module load openmpi" +``` + +> Note: You may omit `[submit_options]` entirely. + +## account + +`submit_options..account`: **string** - Submit jobs to this account on cluster +``. When you omit `account`, **row** does not add the `--account=` line to the +submission script. + +## setup + +`submit_options..setup`: **string** - Lines to include in the submission script on +cluster ``. The setup is executed *before* the action's command. You may omit +`setup` to leave this portion of the script blank. + +## custom + +`submit_options..custom`: **array** of **strings** - List of additional command +line options to pass to the batch submission script on cluster ``. For example. +`custom = ["--mail-user=user@example.com"]` will add the line +``` +#SBATCH --mail-user=user@example.com +``` +to the top of a SLURM submission script. `custom` defaults to an empty array when +omitted. + +## partition + +`submit_options..partition`: **string** - Force the use of a particular partition +when submitting jobs to the queue on cluster `. When omitted, **row** +will automatically determine the correct partition based on the configuration in +[`clusters.toml`](../clusters/index.md). + +> Note: You should almost always omit `partition`. Set it *only* when you need a +> specialty partition that is not automatically selected. diff --git a/src/builtin.rs b/src/builtin.rs new file mode 100644 index 0000000..39f21a7 --- /dev/null +++ b/src/builtin.rs @@ -0,0 +1,269 @@ +use std::collections::HashMap; + +use crate::cluster::{ + Cluster, ClusterConfiguration, IdentificationMethod, Partition, SchedulerType, +}; +use crate::launcher::{Launcher, LauncherConfiguration}; + +pub(crate) trait BuiltIn { + fn built_in() -> Self; +} + +impl BuiltIn for LauncherConfiguration { + /// Construct the built-in launchers + /// + fn built_in() -> Self { + let mut result = Self { + launchers: HashMap::with_capacity(2), + }; + + let mut openmp = HashMap::with_capacity(1); + openmp.insert( + "default".into(), + Launcher { + threads_per_process: Some("OMP_NUM_THREADS=".into()), + ..Launcher::default() + }, + ); + + result.launchers.insert("openmp".into(), openmp); + + let mut mpi = HashMap::with_capacity(3); + mpi.insert( + "default".into(), + Launcher { + executable: Some("srun".into()), + processes: Some("--ntasks=".into()), + threads_per_process: Some("--cpus-per-task=".into()), + gpus_per_process: Some("--gpus-per-task=".into()), + }, + ); + + mpi.insert( + "anvil".into(), + Launcher { + executable: Some("srun --mpi=pmi2".into()), + processes: Some("--ntasks=".into()), + threads_per_process: Some("--cpus-per-task=".into()), + gpus_per_process: Some("--gpus-per-task=".into()), + }, + ); + + mpi.insert( + "none".into(), + Launcher { + executable: Some("mpirun".into()), + processes: Some("-n ".into()), + ..Launcher::default() + }, + ); + + result.launchers.insert("mpi".into(), mpi); + + result + } +} + +impl BuiltIn for ClusterConfiguration { + fn built_in() -> Self { + let cluster = vec![ + //////////////////////////////////////////////////////////////////////////////////////// + // Purdue Anvil + Cluster { + name: "anvil".into(), + identify: IdentificationMethod::ByEnvironment( + "RCAC_CLUSTER".into(), + "anvil".into(), + ), + scheduler: SchedulerType::Slurm, + partition: vec![ + // Auto-detected partitions: shared | wholenode | gpu + Partition { + name: "shared".into(), + maximum_cpus_per_job: Some(127), + maximum_gpus_per_job: Some(0), + ..Partition::default() + }, + Partition { + name: "wholenode".into(), + require_cpus_multiple_of: Some(128), + maximum_gpus_per_job: Some(0), + ..Partition::default() + }, + Partition { + name: "gpu".into(), + minimum_gpus_per_job: Some(1), + gpus_per_node: Some(4), + ..Partition::default() + }, + // The following partitions may only be selected manually. + Partition { + name: "wide".into(), + require_cpus_multiple_of: Some(128), + maximum_gpus_per_job: Some(0), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "highmem".into(), + maximum_gpus_per_job: Some(0), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "debug".into(), + maximum_gpus_per_job: Some(0), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "gpu-debug".into(), + minimum_gpus_per_job: Some(1), + prevent_auto_select: true, + ..Partition::default() + }, + ], + }, + //////////////////////////////////////////////////////////////////////////////////////// + // NCSA delta + Cluster { + name: "delta".into(), + identify: IdentificationMethod::ByEnvironment( + "LMOD_SYSTEM_NAME".into(), + "Delta".into(), + ), + scheduler: SchedulerType::Slurm, + partition: vec![ + // Auto-detected partitions: cpu | gpuA100x4 + Partition { + name: "cpu".into(), + maximum_gpus_per_job: Some(0), + cpus_per_node: Some(128), + memory_per_cpu: Some("1970M".into()), + account_suffix: Some("-cpu".into()), + ..Partition::default() + }, + Partition { + name: "gpuA100x4".into(), + minimum_gpus_per_job: Some(1), + memory_per_gpu: Some("62200M".into()), + gpus_per_node: Some(4), + account_suffix: Some("-gpu".into()), + ..Partition::default() + }, + // The following partitions may only be selected manually. + Partition { + name: "gpuA100x8".into(), + minimum_gpus_per_job: Some(1), + memory_per_gpu: Some("256000M".into()), + gpus_per_node: Some(8), + account_suffix: Some("-gpu".into()), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "gpuA40x4".into(), + minimum_gpus_per_job: Some(1), + memory_per_gpu: Some("62200M".into()), + gpus_per_node: Some(4), + account_suffix: Some("-gpu".into()), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "gpuMI100x8".into(), + minimum_gpus_per_job: Some(1), + memory_per_gpu: Some("256000M".into()), + gpus_per_node: Some(8), + account_suffix: Some("-gpu".into()), + prevent_auto_select: true, + ..Partition::default() + }, + ], + }, + //////////////////////////////////////////////////////////////////////////////////////// + // Great Lakes + Cluster { + name: "greatlakes".into(), + identify: IdentificationMethod::ByEnvironment( + "CLUSTER_NAME".into(), + "greatlakes".into(), + ), + scheduler: SchedulerType::Slurm, + partition: vec![ + // Auto-detected partitions: standard | gpu_mig40,gpu | gpu. + Partition { + name: "standard".into(), + maximum_gpus_per_job: Some(0), + cpus_per_node: Some(36), + memory_per_cpu: Some("5G".into()), + ..Partition::default() + }, + Partition { + name: "gpu_mig40,gpu".into(), + minimum_gpus_per_job: Some(1), + maximum_gpus_per_job: Some(1), + memory_per_cpu: Some("60G".into()), + ..Partition::default() + }, + Partition { + name: "gpu".into(), + minimum_gpus_per_job: Some(1), + memory_per_cpu: Some("60G".into()), + // cannot set gpus_per_node, the partition is heterogeneous + ..Partition::default() + }, + // The following partitions may only be selected manually. + Partition { + name: "gpu_mig40".into(), + minimum_gpus_per_job: Some(1), + memory_per_cpu: Some("125G".into()), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "spgpu".into(), + minimum_gpus_per_job: Some(1), + memory_per_cpu: Some("47000M".into()), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "largemem".into(), + maximum_gpus_per_job: Some(0), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "standard-oc".into(), + maximum_gpus_per_job: Some(0), + cpus_per_node: Some(36), + memory_per_cpu: Some("5G".into()), + prevent_auto_select: true, + ..Partition::default() + }, + Partition { + name: "debug".into(), + maximum_gpus_per_job: Some(0), + cpus_per_node: Some(36), + memory_per_cpu: Some("5G".into()), + prevent_auto_select: true, + ..Partition::default() + }, + ], + }, + // Fallback none cluster. + Cluster { + name: "none".into(), + identify: IdentificationMethod::Always(true), + scheduler: SchedulerType::Bash, + partition: vec![Partition { + name: "none".into(), + ..Partition::default() + }], + }, + ]; + + ClusterConfiguration { cluster } + } +} diff --git a/src/cli.rs b/src/cli.rs index c918fe1..a1a84d3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,4 +1,6 @@ +pub mod cluster; pub mod directories; +pub mod launchers; pub mod scan; pub mod status; pub mod submit; @@ -9,7 +11,9 @@ use log::trace; use std::io; use std::path::PathBuf; +use cluster::ClusterArgs; use directories::DirectoriesArgs; +use launchers::LaunchersArgs; use scan::ScanArgs; use status::StatusArgs; use submit::SubmitArgs; @@ -44,7 +48,10 @@ pub struct GlobalOptions { /// Clear progress bars on exit. #[arg(long, global = true, env = "ROW_CLEAR_PROGRESS", display_order = 2)] pub clear_progress: bool, - // TODO: make cluster a global option + + /// Check the job submission status on the given cluster. Autodetected by default. + #[arg(long, global = true, env = "ROW_CLUSTER", display_order = 2)] + cluster: Option, } #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] @@ -66,6 +73,12 @@ pub enum ShowArgs { /// List directories in the workspace. Directories(DirectoriesArgs), + + /// Show the cluster configuration. + Cluster(ClusterArgs), + + /// Show launcher configurations. + Launchers(LaunchersArgs), } #[derive(Subcommand, Debug)] diff --git a/src/cli/cluster.rs b/src/cli/cluster.rs new file mode 100644 index 0000000..4c695a5 --- /dev/null +++ b/src/cli/cluster.rs @@ -0,0 +1,48 @@ +use clap::Args; +use log::{debug, info}; +use std::error::Error; +use std::io::Write; + +use crate::cli::GlobalOptions; +use row::cluster::ClusterConfiguration; + +#[derive(Args, Debug)] +pub struct ClusterArgs { + /// Show all clusters. + #[arg(long, group = "select", display_order = 0)] + all: bool, + + /// Show only the autodetected cluster's name. + #[arg(long, group = "select", display_order = 0)] + name: bool, +} + +/// Show the cluster. +/// +/// Print the cluster to stdout in toml format. +/// +pub fn cluster( + options: GlobalOptions, + args: ClusterArgs, + output: &mut W, +) -> Result<(), Box> { + debug!("Showing clusters."); + + let clusters = ClusterConfiguration::open()?; + + if args.all { + info!("All cluster configurations:"); + write!(output, "{}", &toml::to_string_pretty(&clusters)?)?; + } else { + let cluster = clusters.identify(options.cluster.as_deref())?; + info!("Cluster configurations for '{}':", cluster.name); + + if args.name { + writeln!(output, "{}", cluster.name)?; + } else { + write!(output, "{}", &toml::to_string_pretty(&cluster)?)?; + } + } + + Ok(()) +} diff --git a/src/cli/directories.rs b/src/cli/directories.rs index b3a8058..7ba23d8 100644 --- a/src/cli/directories.rs +++ b/src/cli/directories.rs @@ -16,10 +16,6 @@ pub struct DirectoriesArgs { /// Select the action to scan (defaults to all). action: String, - /// Print job IDs on the given cluster. Autodetected by default. - #[arg(long, env = "ROW_CLUSTER", display_order = 0)] - cluster: Option, - /// Select directories to summarize (defaults to all). Use 'show directories -' to read from stdin. directories: Vec, @@ -48,7 +44,7 @@ pub fn directories( ) -> Result<(), Box> { debug!("Showing directories."); - let mut project = Project::open(options.io_threads, multi_progress)?; + let mut project = Project::open(options.io_threads, options.cluster, multi_progress)?; let query_directories = cli::parse_directories(args.directories, || Ok(project.state().list_directories()))?; @@ -74,6 +70,7 @@ pub fn directories( table.header = vec![ Item::new("Directory".to_string(), Style::new().underlined()), Item::new("Status".to_string(), Style::new().underlined()), + Item::new("Job ID".to_string(), Style::new().underlined()), ]; for pointer in &args.value { table @@ -83,6 +80,7 @@ pub fn directories( for (group_idx, group) in groups.iter().enumerate() { for directory in group { + // Format the directory status. let status = if completed.contains(directory) { Item::new("completed".to_string(), Style::new().green().italic()) } else if submitted.contains(directory) { @@ -96,11 +94,27 @@ pub fn directories( }; let mut row = Vec::new(); + + // The directory name row.push(Item::new( directory.display().to_string(), Style::new().bold(), )); + + // Status row.push(status); + + // Job ID + let submitted = project.state().submitted(); + + // Values + if let Some((cluster, job_id)) = + submitted.get(&action.name).and_then(|d| d.get(directory)) + { + row.push(Item::new(format!("{}/{}", cluster, job_id), Style::new())); + } else { + row.push(Item::new("".into(), Style::new())); + } for pointer in &args.value { let value = project.state().values()[directory] .pointer(pointer) diff --git a/src/cli/launchers.rs b/src/cli/launchers.rs new file mode 100644 index 0000000..4d9ca0a --- /dev/null +++ b/src/cli/launchers.rs @@ -0,0 +1,50 @@ +use clap::Args; +use log::{debug, info}; +use std::error::Error; +use std::io::Write; + +use crate::cli::GlobalOptions; +use row::cluster::ClusterConfiguration; +use row::launcher::LauncherConfiguration; + +#[derive(Args, Debug)] +pub struct LaunchersArgs { + /// Show all launchers. + #[arg(long, display_order = 0)] + all: bool, +} + +/// Show the launchers. +/// +/// Print the launchers to stdout in toml format. +/// +pub fn launchers( + options: GlobalOptions, + args: LaunchersArgs, + output: &mut W, +) -> Result<(), Box> { + debug!("Showing launchers."); + + let launchers = LauncherConfiguration::open()?; + + if args.all { + info!("All launcher configurations:"); + write!( + output, + "{}", + &toml::to_string_pretty(launchers.full_config())? + )?; + } else { + let clusters = ClusterConfiguration::open()?; + let cluster = clusters.identify(options.cluster.as_deref())?; + + info!("Launcher configurations for cluster '{}':", cluster.name); + write!( + output, + "{}", + &toml::to_string_pretty(&launchers.by_cluster(&cluster.name))? + )?; + } + + Ok(()) +} diff --git a/src/cli/status.rs b/src/cli/status.rs index 611cf0b..ac94a8b 100644 --- a/src/cli/status.rs +++ b/src/cli/status.rs @@ -19,10 +19,6 @@ pub struct StatusArgs { #[arg(short, long, value_name = "pattern", default_value_t=String::from("*"), display_order=0)] action: String, - /// Check the job submission status on the given cluster. Autodetected by default. - #[arg(long, env = "ROW_CLUSTER", display_order = 0)] - cluster: Option, - /// Hide the table header. #[arg(long, display_order = 0)] no_header: bool, @@ -87,7 +83,7 @@ pub fn status( debug!("Showing the workflow's status."); let action_matcher = WildMatch::new(&args.action); - let mut project = Project::open(options.io_threads, multi_progress)?; + let mut project = Project::open(options.io_threads, options.cluster, multi_progress)?; let query_directories = cli::parse_directories(args.directories, || Ok(project.state().list_directories()))?; diff --git a/src/cli/submit.rs b/src/cli/submit.rs index 13787b9..d5f542f 100644 --- a/src/cli/submit.rs +++ b/src/cli/submit.rs @@ -15,8 +15,6 @@ use wildmatch::WildMatch; use crate::cli::GlobalOptions; use row::project::Project; -use row::scheduler::bash::Bash; -use row::scheduler::Scheduler; use row::workflow::{Action, ResourceCost}; use row::MultiProgressContainer; @@ -26,10 +24,6 @@ pub struct SubmitArgs { #[arg(short, long, value_name = "pattern", default_value_t=String::from("*"), display_order=0)] action: String, - /// Check the job submission status on the given cluster. Autodetected by default. - #[arg(long, env = "ROW_CLUSTER", display_order = 0)] - cluster: Option, - /// Select directories to summarize (defaults to all). directories: Vec, @@ -57,7 +51,7 @@ pub fn submit( debug!("Submitting workflow actions to the scheduler."); let action_matcher = WildMatch::new(&args.action); - let mut project = Project::open(options.io_threads, multi_progress)?; + let mut project = Project::open(options.io_threads, options.cluster, multi_progress)?; let query_directories = if args.directories.is_empty() { project.state().list_directories() @@ -139,11 +133,8 @@ pub fn submit( // TODO: Validate submit_whole - // TODO: Move scheduler into project, which will dynamically create a scheduler depending on the - // detected cluster. - let scheduler = Bash::new(&args.cluster.unwrap_or("none".into())); - if args.dry_run { + let scheduler = project.scheduler(); info!("Would submit the following scripts..."); for (index, (action, directories)) in action_directories.iter().enumerate() { info!("script {}/{}:", index + 1, action_directories.len()); @@ -222,6 +213,7 @@ pub fn submit( let instant = Instant::now(); for (index, (action, directories)) in action_directories.iter().enumerate() { + let scheduler = project.scheduler(); let mut message = format!( "[{}/{}] Submitting action '{}' on directory {}", HumanCount((index + 1) as u64), @@ -237,8 +229,6 @@ pub fn submit( message += &format!(" ({}).", style(HumanDuration(instant.elapsed())).dim()); println!("{message}"); - // TODO: Change to the project directory before submitting. - let result = scheduler.submit( &project.workflow().root, action, @@ -246,13 +236,22 @@ pub fn submit( Arc::clone(&should_terminate), ); - // TODO: Implement the job ID store. Save it after all jobs are - // submitted, and when an error occurs. Need to capture successfully - // submitted jobs in the store even when later jobs fail. - if let Err(error) = result { - return Err(error.into()); + match result { + Err(error) => { + // Save the submitted cache for any jobs submitted so far. + project.close(multi_progress)?; + return Err(error.into()); + } + Ok(Some(job_id)) => { + println!("Row submitted job {job_id}."); + project.add_submitted(&action.name, directories, job_id); + continue; + } + Ok(None) => continue, } } + project.close(multi_progress)?; + Ok(()) } diff --git a/src/cluster.rs b/src/cluster.rs new file mode 100644 index 0000000..9e36451 --- /dev/null +++ b/src/cluster.rs @@ -0,0 +1,755 @@ +use log::{debug, info, trace}; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::prelude::*; +use std::io::{self, BufReader}; +use std::path::{Path, PathBuf}; + +use crate::builtin::BuiltIn; +use crate::workflow::Resources; +use crate::Error; + +/// Cluster configuration +/// +/// `ClusterConfiguration` stores the cluster configuration for each defined +/// cluster. +/// +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ClusterConfiguration { + /// The cluster configurations. + #[serde(default)] + pub(crate) cluster: Vec, +} + +/// Cluster +/// +/// `Cluster` stores everything needed to define a single cluster. It is read +/// from the `clusters.toml` file. +/// +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Cluster { + /// The cluster's name. + pub name: String, + + /// The method used to automatically identify this cluster. + pub identify: IdentificationMethod, + + /// The scheduler used on the cluster. + pub scheduler: SchedulerType, + + /// The partitions in the cluster's queue. + pub partition: Vec, +} + +/// Methods to identify clusters. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum IdentificationMethod { + /// Identify a cluster when an environment variable is equal to a value. + ByEnvironment(String, String), + /// Identify a cluster always (true) or never (false) + Always(bool), +} + +/// Types of schedulers. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SchedulerType { + /// Submit jobs to run immediately in bash. + Bash, + /// Submit jobs to a Slurm queue. + Slurm, +} + +/// Partition parameters. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Partition { + /// The partition's name. + pub name: String, + + /// Maximum number of CPUs per job. + pub maximum_cpus_per_job: Option, + + /// Require CPUs to be a multiple of this value. + pub require_cpus_multiple_of: Option, + + /// Memory per CPU. + pub memory_per_cpu: Option, + + /// CPUs per node. + pub cpus_per_node: Option, + + /// Minimum number of GPUs per job. + pub minimum_gpus_per_job: Option, + + /// Maximum number of GPUs per job. + pub maximum_gpus_per_job: Option, + + /// Require GPUs to be a multiple of this value. + pub require_gpus_multiple_of: Option, + + /// Memory per GPU. + pub memory_per_gpu: Option, + + /// GPUs per node. + pub gpus_per_node: Option, + + /// Prevent auto-selection + #[serde(default)] + pub prevent_auto_select: bool, + + /// Suffix the account name + pub account_suffix: Option, +} + +impl ClusterConfiguration { + /// Identify the cluster. + /// + /// Identifying the current cluster consumes the `ClusterConfiguration`. + /// + pub fn identify(self, name: Option<&str>) -> Result { + let cluster = if let Some(name) = name { + self.cluster + .into_iter() + .find(|c| c.name == name) + .ok_or_else(|| Error::ClusterNameNotFound(name.to_string()))? + } else { + self.cluster + .into_iter() + .find(|c| c.identity_matches()) + .ok_or_else(Error::ClusterNotFound)? + }; + + info!("Identified cluster '{}'.", cluster.name); + Ok(cluster) + } + + /// Open the cluster configuration + /// + /// Open `$HOME/.config/row/clusters.toml` if it exists and merge it with + /// the built-in configuration. + /// + /// # Errors + /// Returns `Err(row::Error)` when the file cannot be read or if there is + /// as parse error. + /// + pub fn open() -> Result { + let home = match env::var("ROW_HOME") { + Ok(row_home) => PathBuf::from(row_home), + Err(_) => home::home_dir().ok_or_else(Error::NoHome)?, + }; + let clusters_toml_path = home.join(".config").join("row").join("clusters.toml"); + Self::open_from_path(clusters_toml_path) + } + + fn open_from_path(clusters_toml_path: PathBuf) -> Result { + let mut clusters = Self::built_in(); + + let clusters_file = match File::open(&clusters_toml_path) { + Ok(file) => file, + Err(error) => match error.kind() { + io::ErrorKind::NotFound => { + trace!( + "'{}' does not exist, using built-in clusters.", + &clusters_toml_path.display() + ); + return Ok(clusters); + } + _ => return Err(Error::FileRead(clusters_toml_path, error)), + }, + }; + + let mut buffer = BufReader::new(clusters_file); + let mut clusters_string = String::new(); + buffer + .read_to_string(&mut clusters_string) + .map_err(|e| Error::FileRead(clusters_toml_path.clone(), e))?; + + trace!("Parsing '{}'.", &clusters_toml_path.display()); + let user_config = Self::parse_str(&clusters_toml_path, &clusters_string)?; + clusters.merge(user_config); + Ok(clusters) + } + + /// Parse a `ClusterConfiguration` from a TOML string + /// + /// Does *NOT* merge with the built-in configuration. + /// + pub(crate) fn parse_str(path: &Path, toml: &str) -> Result { + let cluster: ClusterConfiguration = + toml::from_str(toml).map_err(|e| Error::TOMLParse(path.join("clusters.toml"), e))?; + Ok(cluster) + } + + /// Merge keys from another configuration into this one. + /// + /// Merging adds new keys from `b` into self. It also overrides any keys in + /// both with the value in `b`. + /// + fn merge(&mut self, b: Self) { + let mut new_cluster = b.cluster.clone(); + new_cluster.extend(self.cluster.clone()); + self.cluster = new_cluster + } +} + +impl Cluster { + /// Check if the cluster's identity matches the current environment. + fn identity_matches(&self) -> bool { + trace!( + "Checking cluster '{}' via '{:?}'.", + self.name, + self.identify + ); + match &self.identify { + IdentificationMethod::Always(condition) => *condition, + IdentificationMethod::ByEnvironment(variable, value) => { + env::var(variable).is_ok_and(|x| x == *value) + } + } + } + + /// Find the partition to use for the given job. + pub fn find_partition( + &self, + partition_name: Option<&str>, + resources: &Resources, + n_directories: usize, + ) -> Result<&Partition, Error> { + debug!( + "Finding partition for {} CPUs and {} GPUs.", + resources.total_cpus(n_directories), + resources.total_gpus(n_directories) + ); + let mut reason = String::new(); + + let partition = if let Some(partition_name) = partition_name { + let named_partition = self + .partition + .iter() + .find(|p| p.name == partition_name) + .ok_or_else(|| Error::PartitionNameNotFound(partition_name.to_string()))?; + + if !named_partition.matches(resources, n_directories, &mut reason) { + return Err(Error::PartitionNotFound(reason)); + } + + named_partition + } else { + self.partition + .iter() + .find(|p| p.matches(resources, n_directories, &mut reason)) + .ok_or_else(|| Error::PartitionNotFound(reason))? + }; + + Ok(partition) + } +} + +impl Partition { + /// Check if a given job may use this partition. + fn matches(&self, resources: &Resources, n_directories: usize, reason: &mut String) -> bool { + let total_cpus = resources.total_cpus(n_directories); + let total_gpus = resources.total_gpus(n_directories); + + trace!("Checking partition '{}'.", self.name); + + if self.prevent_auto_select { + reason.push_str(&format!("{}: Must be manually selected.\n", self.name)); + return false; + } + + if self.maximum_cpus_per_job.map_or(false, |x| total_cpus > x) { + reason.push_str(&format!("{}: Too many CPUs ({}).\n", self.name, total_cpus)); + return false; + } + + if self + .require_cpus_multiple_of + .map_or(false, |x| total_cpus % x != 0) + { + reason.push_str(&format!( + "{}: CPUs ({}) not a required multiple.\n", + self.name, total_cpus + )); + return false; + } + + if self.minimum_gpus_per_job.map_or(false, |x| total_gpus < x) { + reason.push_str(&format!( + "{}: Not enough GPUs ({}).\n", + self.name, total_gpus + )); + return false; + } + + if self.maximum_gpus_per_job.map_or(false, |x| total_gpus > x) { + reason.push_str(&format!("{}: Too many GPUs ({}).\n", self.name, total_gpus)); + return false; + } + + trace!("total_gpus {}", total_gpus); + if let Some(v) = self.require_gpus_multiple_of { + trace!("total_gpus % v = {}", total_gpus % v); + } + if self + .require_gpus_multiple_of + .map_or(false, |x| total_gpus == 0 || total_gpus % x != 0) + { + reason.push_str(&format!( + "{}: GPUs ({}) not a required multiple.\n", + self.name, total_gpus + )); + return false; + } + + true + } +} + +impl Default for Partition { + fn default() -> Self { + Partition { + name: "partition".into(), + maximum_cpus_per_job: None, + memory_per_cpu: None, + cpus_per_node: None, + require_cpus_multiple_of: None, + minimum_gpus_per_job: None, + maximum_gpus_per_job: None, + memory_per_gpu: None, + gpus_per_node: None, + require_gpus_multiple_of: None, + prevent_auto_select: false, + account_suffix: None, + } + } +} + +#[cfg(test)] +mod tests { + use assert_fs::prelude::*; + use assert_fs::TempDir; + use serial_test::{parallel, serial}; + + use super::*; + use crate::workflow::Processes; + + fn setup() { + let _ = env_logger::builder() + .filter_level(log::LevelFilter::max()) + .is_test(true) + .try_init(); + } + + #[test] + #[serial] + fn identify() { + setup(); + let clusters = vec![ + Cluster { + name: "cluster0".into(), + identify: IdentificationMethod::Always(false), + scheduler: SchedulerType::Bash, + partition: Vec::new(), + }, + Cluster { + name: "cluster1".into(), + identify: IdentificationMethod::ByEnvironment("_row_select".into(), "a".into()), + scheduler: SchedulerType::Bash, + partition: Vec::new(), + }, + Cluster { + name: "cluster2".into(), + identify: IdentificationMethod::ByEnvironment("_row_select".into(), "b".into()), + scheduler: SchedulerType::Bash, + partition: Vec::new(), + }, + Cluster { + name: "cluster3".into(), + identify: IdentificationMethod::Always(true), + scheduler: SchedulerType::Bash, + partition: Vec::new(), + }, + Cluster { + name: "cluster4".into(), + identify: IdentificationMethod::ByEnvironment("_row_Select".into(), "b".into()), + scheduler: SchedulerType::Bash, + partition: Vec::new(), + }, + ]; + let cluster_configuration = ClusterConfiguration { cluster: clusters }; + assert_eq!( + cluster_configuration + .clone() + .identify(Some("cluster4")) + .unwrap(), + cluster_configuration.cluster[4] + ); + assert!(matches!( + cluster_configuration + .clone() + .identify(Some("not a cluster")), + Err(Error::ClusterNameNotFound(_)) + )); + + env::remove_var("_row_select"); + assert_eq!( + cluster_configuration.clone().identify(None).unwrap(), + cluster_configuration.cluster[3] + ); + + env::set_var("_row_select", "b"); + assert_eq!( + cluster_configuration.clone().identify(None).unwrap(), + cluster_configuration.cluster[2] + ); + + env::set_var("_row_select", "a"); + assert_eq!( + cluster_configuration.clone().identify(None).unwrap(), + cluster_configuration.cluster[1] + ); + + assert_eq!( + cluster_configuration + .clone() + .identify(Some("cluster0")) + .unwrap(), + cluster_configuration.cluster[0] + ); + } + + #[test] + #[parallel] + fn empty_partition() { + setup(); + + let partition = Partition::default(); + + let resources = Resources { + processes: Processes::PerDirectory(1), + threads_per_process: Some(2), + gpus_per_process: Some(3), + ..Resources::default() + }; + let mut reason = String::new(); + assert!(partition.matches(&resources, 10, &mut reason)); + } + + #[test] + #[parallel] + fn partition_checks() { + setup(); + + let resources = Resources { + processes: Processes::PerDirectory(1), + threads_per_process: Some(2), + gpus_per_process: Some(3), + ..Resources::default() + }; + let mut reason = String::new(); + + let partition = Partition { + maximum_cpus_per_job: Some(10), + ..Partition::default() + }; + + assert!(!partition.matches(&resources, 6, &mut reason)); + assert!(partition.matches(&resources, 5, &mut reason)); + + let partition = Partition { + require_cpus_multiple_of: Some(10), + ..Partition::default() + }; + + assert!(!partition.matches(&resources, 6, &mut reason)); + assert!(partition.matches(&resources, 5, &mut reason)); + assert!(partition.matches(&resources, 10, &mut reason)); + assert!(partition.matches(&resources, 15, &mut reason)); + + let partition = Partition { + minimum_gpus_per_job: Some(9), + ..Partition::default() + }; + + assert!(!partition.matches(&resources, 1, &mut reason)); + assert!(!partition.matches(&resources, 2, &mut reason)); + assert!(partition.matches(&resources, 3, &mut reason)); + + let partition = Partition { + maximum_gpus_per_job: Some(9), + ..Partition::default() + }; + + assert!(partition.matches(&resources, 1, &mut reason)); + assert!(partition.matches(&resources, 2, &mut reason)); + assert!(partition.matches(&resources, 3, &mut reason)); + assert!(!partition.matches(&resources, 4, &mut reason)); + + let partition = Partition { + require_gpus_multiple_of: Some(9), + ..Partition::default() + }; + + assert!(!partition.matches(&resources, 1, &mut reason)); + assert!(!partition.matches(&resources, 2, &mut reason)); + assert!(partition.matches(&resources, 3, &mut reason)); + assert!(!partition.matches(&resources, 4, &mut reason)); + assert!(!partition.matches(&resources, 5, &mut reason)); + assert!(partition.matches(&resources, 6, &mut reason)); + + let partition = Partition { + prevent_auto_select: true, + ..Partition::default() + }; + + assert!(!partition.matches(&resources, 1, &mut reason)); + assert!(!partition.matches(&resources, 2, &mut reason)); + assert!(!partition.matches(&resources, 3, &mut reason)); + assert!(!partition.matches(&resources, 4, &mut reason)); + assert!(!partition.matches(&resources, 5, &mut reason)); + assert!(!partition.matches(&resources, 6, &mut reason)); + } + + #[test] + #[parallel] + fn find_partition() { + setup(); + + let partitions = vec![ + Partition { + name: "cpu".into(), + maximum_cpus_per_job: Some(10), + maximum_gpus_per_job: Some(0), + ..Partition::default() + }, + Partition { + name: "gpu".into(), + maximum_gpus_per_job: Some(10), + minimum_gpus_per_job: Some(1), + ..Partition::default() + }, + Partition { + name: "other".into(), + maximum_cpus_per_job: Some(20), + maximum_gpus_per_job: Some(20), + ..Partition::default() + }, + ]; + + let cluster = Cluster { + name: "cluster".into(), + identify: IdentificationMethod::Always(true), + scheduler: SchedulerType::Bash, + partition: partitions, + }; + + let cpu_resources = Resources { + processes: Processes::PerDirectory(1), + ..Resources::default() + }; + + let gpu_resources = Resources { + processes: Processes::PerDirectory(1), + gpus_per_process: Some(1), + ..Resources::default() + }; + + assert!( + cluster + .find_partition(None, &cpu_resources, 1) + .unwrap() + .name + == "cpu" + ); + assert!( + cluster + .find_partition(None, &cpu_resources, 10) + .unwrap() + .name + == "cpu" + ); + assert!( + cluster + .find_partition(None, &gpu_resources, 1) + .unwrap() + .name + == "gpu" + ); + assert!( + cluster + .find_partition(None, &gpu_resources, 10) + .unwrap() + .name + == "gpu" + ); + + assert!( + cluster + .find_partition(None, &cpu_resources, 11) + .unwrap() + .name + == "other" + ); + assert!( + cluster + .find_partition(None, &gpu_resources, 11) + .unwrap() + .name + == "other" + ); + assert!( + cluster + .find_partition(None, &cpu_resources, 20) + .unwrap() + .name + == "other" + ); + assert!( + cluster + .find_partition(None, &gpu_resources, 20) + .unwrap() + .name + == "other" + ); + + assert!(matches!( + cluster.find_partition(None, &cpu_resources, 21), + Err(Error::PartitionNotFound(_)) + )); + assert!(matches!( + cluster.find_partition(Some("not_a_partition"), &cpu_resources, 1), + Err(Error::PartitionNameNotFound(_)) + )); + + assert!( + cluster + .find_partition(Some("other"), &gpu_resources, 20) + .unwrap() + .name + == "other" + ); + assert!(matches!( + cluster.find_partition(Some("other"), &cpu_resources, 21), + Err(Error::PartitionNotFound(_)) + )); + } + + #[test] + #[parallel] + fn open_no_file() { + setup(); + let temp = TempDir::new().unwrap().child("clusters.json"); + let clusters = + ClusterConfiguration::open_from_path(temp.path().into()).expect("valid clusters"); + assert_eq!(clusters, ClusterConfiguration::built_in()); + } + + #[test] + #[parallel] + fn open_empty_file() { + setup(); + let temp = TempDir::new().unwrap().child("clusters.json"); + temp.write_str("").unwrap(); + let clusters = + ClusterConfiguration::open_from_path(temp.path().into()).expect("valid clusters"); + assert_eq!(clusters, ClusterConfiguration::built_in()); + } + + #[test] + #[parallel] + fn minimal_cluster() { + setup(); + let temp = TempDir::new().unwrap().child("clusters.json"); + temp.write_str( + r#" +[[cluster]] +name = "a" +identify.always = true +scheduler = "bash" + +[[cluster.partition]] +name = "b" +"#, + ) + .unwrap(); + let clusters = ClusterConfiguration::open_from_path(temp.path().into()).unwrap(); + let built_in_clusters = ClusterConfiguration::built_in(); + assert_eq!(clusters.cluster.len(), 1 + built_in_clusters.cluster.len()); + + let cluster = clusters.cluster.first().unwrap(); + assert_eq!(cluster.name, "a"); + assert_eq!(cluster.identify, IdentificationMethod::Always(true)); + assert_eq!(cluster.scheduler, SchedulerType::Bash); + assert_eq!( + cluster.partition, + vec![Partition { + name: "b".into(), + ..Partition::default() + }] + ); + } + + #[test] + #[parallel] + fn maximal_cluster() { + setup(); + let temp = TempDir::new().unwrap().child("clusters.json"); + temp.write_str( + r#" +[[cluster]] +name = "a" +identify.by_environment = ["b", "c"] +scheduler = "slurm" + +[[cluster.partition]] +name = "d" +maximum_cpus_per_job = 2 +require_cpus_multiple_of = 4 +memory_per_cpu = "e" +minimum_gpus_per_job = 8 +maximum_gpus_per_job = 16 +require_gpus_multiple_of = 32 +memory_per_gpu = "f" +cpus_per_node = 10 +gpus_per_node = 11 +account_suffix = "-gpu" +"#, + ) + .unwrap(); + let clusters = ClusterConfiguration::open_from_path(temp.path().into()).unwrap(); + let built_in_clusters = ClusterConfiguration::built_in(); + assert_eq!(clusters.cluster.len(), 1 + built_in_clusters.cluster.len()); + + let cluster = clusters.cluster.first().unwrap(); + assert_eq!(cluster.name, "a"); + assert_eq!( + cluster.identify, + IdentificationMethod::ByEnvironment("b".into(), "c".into()) + ); + assert_eq!(cluster.scheduler, SchedulerType::Slurm); + assert_eq!( + cluster.partition, + vec![Partition { + name: "d".into(), + + maximum_cpus_per_job: Some(2), + require_cpus_multiple_of: Some(4), + memory_per_cpu: Some("e".into()), + minimum_gpus_per_job: Some(8), + maximum_gpus_per_job: Some(16), + require_gpus_multiple_of: Some(32), + memory_per_gpu: Some("f".into()), + prevent_auto_select: false, + cpus_per_node: Some(10), + gpus_per_node: Some(11), + account_suffix: Some("-gpu".into()), + }] + ); + } +} diff --git a/src/expr.rs b/src/expr.rs index 59ae228..037897a 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -4,175 +4,6 @@ use std::iter; use crate::workflow::Comparison; -// Commented code related to evalexpr. Will test with users first and see if they find a need for -// more flexible expressions and whether it is worth the complexity. Also evalexpr appears to be -// no longer actively maintained. - -// use evalexpr::{ContextWithMutableVariables, HashMapContext, TupleType}; - -// /// Convert a JSON Tuple element to the evalexpr Value. -// /// -// /// evalexpr has no way to store maps as tuple elements. Store them as JSON -// /// strings to preserve some information. -// /// -// fn value_to_tuple_element(value: &Value) -> Result { -// Ok(match value { -// Value::Object(_obj) => { -// todo!("Implement maps in arrays"); -// } -// Value::String(string) => evalexpr::Value::String(string.clone()), -// Value::Array(array) => { -// let tuple: TupleType = array -// .iter() -// .map(value_to_tuple_element) -// .collect::>()?; -// evalexpr::Value::Tuple(tuple) -// } -// Value::Bool(bool) => evalexpr::Value::Boolean(*bool), -// Value::Null => evalexpr::Value::Empty, -// Value::Number(number) => { -// if let Some(v) = number.as_i64() { -// evalexpr::Value::Int(v) -// } else if let Some(v) = number.as_f64() { -// evalexpr::Value::Float(v) -// } else { -// return Err(Error::InvalidNumber(number.to_string())); -// } -// } -// }) -// } - -// /// Recursively build up a context -// /// -// /// Inspired by https://github.com/ISibboI/evalexpr/issues/117#issuecomment-1792496021 -// /// -// fn add_value_to_context( -// prefix: &str, -// value: &Value, -// context: &mut HashMapContext, -// ) -> Result<(), Error> { -// match value { -// Value::Object(obj) => { -// for (key, value) in obj { -// let new_key = if prefix.is_empty() { -// key.to_string() -// } else { -// format!("{}.{}", prefix, key) -// }; -// add_value_to_context(&new_key, value, context)?; -// } -// } -// Value::String(string) => { -// context.set_value(prefix.into(), evalexpr::Value::String(string.clone()))?; -// } -// Value::Array(array) => { -// let tuple: TupleType = array -// .iter() -// .map(value_to_tuple_element) -// .collect::>()?; -// context.set_value(prefix.into(), evalexpr::Value::Tuple(tuple))?; -// } -// Value::Bool(bool) => { -// context.set_value(prefix.into(), evalexpr::Value::Boolean(*bool))?; -// } -// Value::Null => { -// context.set_value(prefix.into(), evalexpr::Value::Empty)?; -// } -// Value::Number(number) => { -// if let Some(v) = number.as_i64() { -// context.set_value(prefix.into(), evalexpr::Value::Int(v))?; -// } else if let Some(v) = number.as_f64() { -// context.set_value(prefix.into(), evalexpr::Value::Float(v))?; -// } else { -// return Err(Error::InvalidNumber(number.to_string())); -// } -// } -// } - -// Ok(()) -// } - -// /// Convert a JSON value to an evalexpr::HashMapContext. -// /// -// /// The top level map is flattened with its keys becoming variables. Nested -// /// maps are expanded with a dotted notation. -// /// -// fn json_value_to_context(base: &str, value: &Value) -> Result { -// let mut context = HashMapContext::new(); -// add_value_to_context(base, value, &mut context)?; -// Ok(context) -// } - -// /// Compare two expreval::Values lexicographically. -// /// -// /// # Panics -// /// Panics when the two values are not the same type. -// /// -// fn cmp_values(a: &evalexpr::Value, b: &evalexpr::Value) -> Ordering { -// match a { -// evalexpr::Value::String(a_str) => { -// if let evalexpr::Value::String(b_str) = b { -// a_str.cmp(b_str) -// } else { -// panic!("Cannot compare {:?} and {:?}", a, b); -// } -// } -// evalexpr::Value::Float(a_float) => { -// if let evalexpr::Value::Float(b_float) = b { -// a_float.partial_cmp(b_float).expect(&format!( -// "Valid floating point values, got: {:?} and {:?}", -// a, b -// )) -// } else { -// panic!("Cannot compare {:?} and {:?}", a, b); -// } -// } -// evalexpr::Value::Int(a_int) => { -// if let evalexpr::Value::Int(b_int) = b { -// a_int.cmp(b_int) -// } else { -// panic!("Cannot compare {:?} and {:?}", a, b); -// } -// } -// evalexpr::Value::Boolean(a_bool) => { -// if let evalexpr::Value::Boolean(b_bool) = b { -// a_bool.cmp(b_bool) -// } else { -// panic!("Cannot compare {:?} and {:?}", a, b); -// } -// } -// evalexpr::Value::Empty => { -// if b.is_empty() { -// Ordering::Equal -// } else { -// panic!("Cannot compare {:?} and {:?}", a, b); -// } -// } -// evalexpr::Value::Tuple(a_tuple) => { -// if let evalexpr::Value::Tuple(b_tuple) = b { -// if a_tuple.len() != b_tuple.len() { -// panic!("Cannot compare {:?} and {:?}", a, b); -// } - -// if a_tuple.is_empty() && b_tuple.is_empty() { -// Ordering::Equal -// } else { -// for (c, d) in iter::zip(a_tuple, b_tuple) { -// match cmp_values(c, d) { -// Ordering::Less => return Ordering::Less, -// Ordering::Greater => return Ordering::Greater, -// Ordering::Equal => (), -// }; -// } -// Ordering::Equal -// } -// } else { -// panic!("Cannot compare {:?} and {:?}", a, b); -// } -// } -// } -// } - /// Compares two Values lexicographically. /// /// # Returns @@ -236,188 +67,12 @@ pub(crate) fn evaluate_json_comparison( #[cfg(test)] mod tests { - use super::*; - - // use evalexpr::{Context, IterateVariablesContext, Value}; - // use serde_json::json; - - // #[test] - // fn string() -> Result<(), Box> { - // let str = "this is a string"; - // let json_value = json!(str); - // let result = json_value_to_context("v", &json_value)?; - - // assert_eq!(result.iter_variables().count(), 1); - // assert_eq!(result.get_value("v"), Some(&Value::String(str.into()))); - // Ok(()) - // } - - // #[test] - // fn bool() -> Result<(), Box> { - // let bool = true; - // let json_value = json!(bool); - // let result = json_value_to_context("v", &json_value)?; - - // assert_eq!(result.iter_variables().count(), 1); - // assert_eq!(result.get_value("v"), Some(&Value::Boolean(bool))); - // Ok(()) - // } - - // #[test] - // fn empty() -> Result<(), Box> { - // let json_value = Value::Null; - // let result = json_value_to_context("v", &json_value)?; + use serial_test::parallel; - // assert_eq!(result.iter_variables().count(), 1); - // assert_eq!(result.get_value("v"), Some(&Value::Empty)); - // Ok(()) - // } - - // #[test] - // fn integer() -> Result<(), Box> { - // let int = 1_321_987_654; - // let json_value = json!(int); - // let result = json_value_to_context("v", &json_value)?; - - // assert_eq!(result.iter_variables().count(), 1); - // assert_eq!(result.get_value("v"), Some(&Value::Int(int))); - // Ok(()) - // } - - // #[test] - // fn float() -> Result<(), Box> { - // let float = 1.234e16; - // let json_value = json!(float); - // let result = json_value_to_context("v", &json_value)?; - - // assert_eq!(result.iter_variables().count(), 1); - // assert_eq!(result.get_value("v"), Some(&Value::Float(float))); - // Ok(()) - // } - - // #[test] - // fn array() -> Result<(), Box> { - // let json_value = json!(["a", 1, 3.5, true]); - // let result = json_value_to_context("v", &json_value)?; - - // assert_eq!(result.iter_variables().count(), 1); - // let expected = vec![ - // Value::String("a".into()), - // Value::Int(1), - // Value::Float(3.5), - // Value::Boolean(true), - // ]; - // assert_eq!(result.get_value("v"), Some(&Value::Tuple(expected))); - // Ok(()) - // } - - // #[test] - // fn array_array() -> Result<(), Box> { - // let json_value = json!([[1, 2], [3, 4, 5]]); - // let result = json_value_to_context("v", &json_value)?; - - // assert_eq!(result.iter_variables().count(), 1); - // let zero = vec![Value::Int(1), Value::Int(2)]; - // let one = vec![Value::Int(3), Value::Int(4), Value::Int(5)]; - // let expected = vec![Value::Tuple(zero), Value::Tuple(one)]; - // assert_eq!(result.get_value("v"), Some(&Value::Tuple(expected))); - // Ok(()) - // } - - // #[test] - // fn map() -> Result<(), Box> { - // let json_value = json!({ - // "a": "b", - // "c": 10, - // "d": -12.5, - // "e": {"f": 1, "g": "h"}, - // "i": [14, "j", 3.5], - // "l": null - // }); - // let result = json_value_to_context("v", &json_value)?; - - // assert_eq!(result.iter_variables().count(), 7); - // assert_eq!(result.get_value("v.a"), Some(&Value::from("b"))); - // assert_eq!(result.get_value("v.c"), Some(&Value::Int(10))); - // assert_eq!(result.get_value("v.d"), Some(&Value::Float(-12.5))); - // assert_eq!(result.get_value("v.e.f"), Some(&Value::Int(1))); - // assert_eq!(result.get_value("v.e.g"), Some(&Value::from("h"))); - // assert_eq!(result.get_value("v.e.g"), Some(&Value::from("h"))); - // let expected = vec![Value::Int(14), Value::from("j"), Value::Float(3.5)]; - // assert_eq!(result.get_value("v.i"), Some(&Value::Tuple(expected))); - // assert_eq!(result.get_value("v.l"), Some(&Value::Empty)); - // Ok(()) - // } - - // #[test] - // fn cmp_valid() { - // assert_eq!(cmp_values(&Value::Int(0), &Value::Int(10)), Ordering::Less); - // assert_eq!( - // cmp_values(&Value::Int(10), &Value::Int(0)), - // Ordering::Greater - // ); - // assert_eq!(cmp_values(&Value::Int(0), &Value::Int(0)), Ordering::Equal); - // assert_eq!( - // cmp_values(&Value::from("abcd"), &Value::from("abce")), - // Ordering::Less - // ); - // assert_eq!( - // cmp_values(&Value::from("abcd"), &Value::from("abcd")), - // Ordering::Equal - // ); - // assert_eq!( - // cmp_values(&Value::from("abce"), &Value::from("abcd")), - // Ordering::Greater - // ); - // assert_eq!( - // cmp_values(&Value::from(1.0), &Value::from(2.0)), - // Ordering::Less - // ); - // assert_eq!( - // cmp_values(&Value::from(1.0), &Value::from(1.0)), - // Ordering::Equal - // ); - // assert_eq!( - // cmp_values(&Value::from(2.0), &Value::from(1.0)), - // Ordering::Greater - // ); - // assert_eq!( - // cmp_values(&Value::from(false), &Value::from(true)), - // Ordering::Less - // ); - // assert_eq!( - // cmp_values(&Value::from(true), &Value::from(false)), - // Ordering::Greater - // ); - // assert_eq!( - // cmp_values(&Value::from(true), &Value::from(true)), - // Ordering::Equal - // ); - // assert_eq!(cmp_values(&Value::Empty, &Value::Empty), Ordering::Equal); - - // let a = Value::Tuple(vec![Value::Int(14), Value::from("j"), Value::Float(3.5)]); - // let b = Value::Tuple(vec![Value::Int(13), Value::from("j"), Value::Float(3.5)]); - // let c = Value::Tuple(vec![Value::Int(13), Value::from("j"), Value::Float(3.0)]); - - // assert_eq!(cmp_values(&a, &b), Ordering::Greater); - // assert_eq!(cmp_values(&b, &a), Ordering::Less); - // assert_eq!(cmp_values(&b, &c), Ordering::Greater); - // } - - // #[test] - // #[should_panic] - // fn cmp_invalid_float_int() { - // cmp_values(&Value::from(10), &Value::from(3.5)); - // } - // #[test] - // #[should_panic] - // fn cmp_invalid_tuple() { - // let a = Value::Tuple(vec![Value::Int(14), Value::from("j"), Value::Float(3.5)]); - // let b = Value::Tuple(vec![Value::Int(13), Value::from("j")]); - // cmp_values(&a, &b); - // } + use super::*; #[test] + #[parallel] fn cmp_valid_json() { assert_eq!( partial_cmp_json_values(&Value::from(0), &Value::from(10)), @@ -482,6 +137,7 @@ mod tests { } #[test] + #[parallel] fn cmp_invalid_types_json() { assert_eq!( partial_cmp_json_values(&Value::from(10), &Value::from("abcd")), @@ -489,6 +145,7 @@ mod tests { ); } #[test] + #[parallel] fn cmp_invalid_tuple_json() { let a = Value::Array(vec![Value::from(14), Value::from("j"), Value::from(3.5)]); let b = Value::Array(vec![Value::from(13), Value::from("j")]); @@ -496,6 +153,7 @@ mod tests { } #[test] + #[parallel] fn eval() { assert_eq!( evaluate_json_comparison(&Comparison::EqualTo, &Value::from(5), &Value::from(5)), diff --git a/src/launcher.rs b/src/launcher.rs new file mode 100644 index 0000000..df41b3c --- /dev/null +++ b/src/launcher.rs @@ -0,0 +1,389 @@ +use log::trace; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::fs::File; +use std::io::prelude::*; +use std::io::{self, BufReader}; +use std::path::{Path, PathBuf}; + +use crate::builtin::BuiltIn; +use crate::workflow::Resources; +use crate::Error; + +/// Launcher configuration +/// +/// `LauncherConfiguration` stores the launcher configuration for each defined +/// launcher/cluster. +/// +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LauncherConfiguration { + /// The launcher configurations. + pub(crate) launchers: HashMap>, +} + +/// Launcher +/// +/// `Launcher` is one element of the launcher configuration. +/// +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Launcher { + pub executable: Option, + pub gpus_per_process: Option, + pub processes: Option, + pub threads_per_process: Option, +} + +impl Launcher { + /// Build the launcher prefix appropriate for the given resources + pub fn prefix(&self, resources: &Resources, n_directories: usize) -> String { + let mut result = String::new(); + let mut need_space = false; + + if let Some(executable) = &self.executable { + result.push_str(executable); + need_space = true; + } + + if let Some(processes) = &self.processes { + if need_space { + result.push(' '); + } + result.push_str(&format!( + "{processes}{}", + resources.total_processes(n_directories) + )); + need_space = true; + } + + if let (Some(self_threads), Some(resources_threads)) = + (&self.threads_per_process, resources.threads_per_process) + { + if need_space { + result.push(' '); + } + result.push_str(&format!("{self_threads}{resources_threads}")); + need_space = true; + } + + if let (Some(self_gpus), Some(resources_gpus)) = + (&self.gpus_per_process, resources.gpus_per_process) + { + if need_space { + result.push(' '); + } + result.push_str(&format!("{self_gpus}{resources_gpus}")); + need_space = true; + } + + if need_space { + result.push(' '); + } + result + } +} + +impl LauncherConfiguration { + /// Open the launcher configuration + /// + /// Open `$HOME/.config/row/launchers.toml` if it exists and merge it with + /// the built-in configuration. + /// + /// # Errors + /// Returns `Err(row::Error)` when the file cannot be read or if there is + /// as parse error. + /// + pub fn open() -> Result { + let home = match env::var("ROW_HOME") { + Ok(row_home) => PathBuf::from(row_home), + Err(_) => home::home_dir().ok_or_else(Error::NoHome)?, + }; + let launchers_toml_path = home.join(".config").join("row").join("launchers.toml"); + Self::open_from_path(launchers_toml_path) + } + + fn open_from_path(launchers_toml_path: PathBuf) -> Result { + let mut launchers = Self::built_in(); + + let launchers_file = match File::open(&launchers_toml_path) { + Ok(file) => file, + Err(error) => match error.kind() { + io::ErrorKind::NotFound => { + trace!( + "'{}' does not exist, using built-in launchers.", + &launchers_toml_path.display() + ); + return Ok(launchers); + } + _ => return Err(Error::FileRead(launchers_toml_path, error)), + }, + }; + + let mut buffer = BufReader::new(launchers_file); + let mut launchers_string = String::new(); + buffer + .read_to_string(&mut launchers_string) + .map_err(|e| Error::FileRead(launchers_toml_path.clone(), e))?; + + trace!("Parsing '{}'.", &launchers_toml_path.display()); + let user_config = Self::parse_str(&launchers_toml_path, &launchers_string)?; + launchers.merge(user_config); + launchers.validate()?; + Ok(launchers) + } + + /// Parse a `LauncherConfiguration` from a TOML string + /// + /// Does *NOT* merge with the built-in configuration. + /// + pub(crate) fn parse_str(path: &Path, toml: &str) -> Result { + Ok(LauncherConfiguration { + launchers: toml::from_str(toml) + .map_err(|e| Error::TOMLParse(path.join("launchers.toml"), e))?, + }) + } + + /// Merge keys from another configuration into this one. + /// + /// Merging adds new keys from `b` into self. It also overrides any keys in + /// both with the value in `b`. + /// + fn merge(&mut self, b: Self) { + for (launcher_name, launcher_clusters) in b.launchers { + self.launchers + .entry(launcher_name) + .and_modify(|e| e.extend(launcher_clusters.clone())) + .or_insert(launcher_clusters); + } + } + + /// Validate that the configuration is correct. + /// + /// Valid launcher configurations have a `default` cluster for all + /// launchers. + fn validate(&self) -> Result<(), Error> { + for (launcher_name, launcher_clusters) in &self.launchers { + if !launcher_clusters.contains_key("default") { + return Err(Error::LauncherMissingDefault(launcher_name.clone())); + } + } + + Ok(()) + } + + /// Get all launchers for a specific cluster. + pub fn by_cluster(&self, cluster_name: &str) -> HashMap { + let mut result = HashMap::with_capacity(self.launchers.len()); + + for (launcher_name, launcher_clusters) in &self.launchers { + if let Some(launcher) = launcher_clusters.get(cluster_name) { + result.insert(launcher_name.clone(), launcher.clone()); + } else { + result.insert( + launcher_name.clone(), + launcher_clusters + .get("default") + .expect("launcher should have a default") + .clone(), + ); + } + } + + result + } + + /// Get the complete launcher configuration. + pub fn full_config(&self) -> &HashMap> { + &self.launchers + } +} + +#[cfg(test)] +mod tests { + use assert_fs::prelude::*; + use assert_fs::TempDir; + use serial_test::parallel; + + use super::*; + use crate::workflow::Processes; + + fn setup() { + let _ = env_logger::builder() + .filter_level(log::LevelFilter::max()) + .is_test(true) + .try_init(); + } + + #[test] + #[parallel] + fn unset_launcher() { + setup(); + let launchers = LauncherConfiguration::built_in(); + let launchers_by_cluster = launchers.by_cluster("any_cluster"); + assert!(launchers_by_cluster.get("unset_launcher").is_none()); + } + + #[test] + #[parallel] + fn openmp_prefix() { + setup(); + let launchers = LauncherConfiguration::built_in(); + let launchers_by_cluster = launchers.by_cluster("any_cluster"); + let openmp = launchers_by_cluster + .get("openmp") + .expect("a valid Launcher"); + + let no_threads = Resources::default(); + assert_eq!(openmp.prefix(&no_threads, 10), ""); + assert_eq!(openmp.prefix(&no_threads, 1), ""); + + let threads = Resources { + threads_per_process: Some(5), + ..Resources::default() + }; + assert_eq!(openmp.prefix(&threads, 10), "OMP_NUM_THREADS=5 "); + assert_eq!(openmp.prefix(&threads, 1), "OMP_NUM_THREADS=5 "); + } + + #[test] + #[parallel] + fn mpi_prefix_none() { + setup(); + let launchers = LauncherConfiguration::built_in(); + let launchers_by_cluster = launchers.by_cluster("none"); + let mpi = launchers_by_cluster.get("mpi").expect("a valid Launcher"); + + let one_proc = Resources::default(); + assert_eq!(mpi.prefix(&one_proc, 10), "mpirun -n 1 "); + assert_eq!(mpi.prefix(&one_proc, 1), "mpirun -n 1 "); + + let procs_per_directory = Resources { + processes: Processes::PerDirectory(2), + ..Resources::default() + }; + assert_eq!(mpi.prefix(&procs_per_directory, 11), "mpirun -n 22 "); + assert_eq!(mpi.prefix(&procs_per_directory, 1), "mpirun -n 2 "); + + let all = Resources { + processes: Processes::PerDirectory(6), + threads_per_process: Some(3), + gpus_per_process: Some(8), + ..Resources::default() + }; + assert_eq!(mpi.prefix(&all, 11), "mpirun -n 66 "); + assert_eq!(mpi.prefix(&all, 1), "mpirun -n 6 "); + } + + #[test] + #[parallel] + fn mpi_prefix_default() { + setup(); + let launchers = LauncherConfiguration::built_in(); + let launchers_by_cluster = launchers.by_cluster("any_cluster"); + let mpi = launchers_by_cluster.get("mpi").expect("a valid Launcher"); + + let one_proc = Resources::default(); + assert_eq!(mpi.prefix(&one_proc, 10), "srun --ntasks=1 "); + assert_eq!(mpi.prefix(&one_proc, 1), "srun --ntasks=1 "); + + let procs_per_directory = Resources { + processes: Processes::PerDirectory(2), + ..Resources::default() + }; + assert_eq!(mpi.prefix(&procs_per_directory, 11), "srun --ntasks=22 "); + assert_eq!(mpi.prefix(&procs_per_directory, 1), "srun --ntasks=2 "); + + let all = Resources { + processes: Processes::PerDirectory(6), + threads_per_process: Some(3), + gpus_per_process: Some(8), + ..Resources::default() + }; + assert_eq!( + mpi.prefix(&all, 11), + "srun --ntasks=66 --cpus-per-task=3 --gpus-per-task=8 " + ); + assert_eq!( + mpi.prefix(&all, 1), + "srun --ntasks=6 --cpus-per-task=3 --gpus-per-task=8 " + ); + } + + #[test] + #[parallel] + fn open_no_file() { + setup(); + let temp = TempDir::new().unwrap().child("launchers.json"); + let launchers = + LauncherConfiguration::open_from_path(temp.path().into()).expect("valid launchers"); + assert_eq!(launchers, LauncherConfiguration::built_in()); + } + + #[test] + #[parallel] + fn open_empty_file() { + setup(); + let temp = TempDir::new().unwrap().child("launchers.json"); + temp.write_str("").unwrap(); + let launchers = + LauncherConfiguration::open_from_path(temp.path().into()).expect("valid launchers"); + assert_eq!(launchers, LauncherConfiguration::built_in()); + } + + #[test] + #[parallel] + fn no_default() { + setup(); + let temp = TempDir::new().unwrap().child("launchers.json"); + temp.write_str( + r#" +[new_launcher.not_default] +"#, + ) + .unwrap(); + let error = LauncherConfiguration::open_from_path(temp.path().into()); + assert!(matches!(error, Err(Error::LauncherMissingDefault(_)))); + } + + #[test] + #[parallel] + fn new_launcher() { + setup(); + let temp = TempDir::new().unwrap().child("launchers.json"); + temp.write_str( + r#" +[new_launcher.default] +executable = "a" +processes = "b" +threads_per_process = "c" +gpus_per_process = "d" + +[new_launcher.non_default] +executable = "e" +"#, + ) + .unwrap(); + let launchers = + LauncherConfiguration::open_from_path(temp.path().into()).expect("valid launcher"); + + let built_in = LauncherConfiguration::built_in(); + assert_eq!(launchers.launchers.len(), 3); + assert_eq!(launchers.launchers["openmp"], built_in.launchers["openmp"]); + assert_eq!(launchers.launchers["mpi"], built_in.launchers["mpi"]); + + let launchers_by_cluster = launchers.by_cluster("non_default"); + let non_default = launchers_by_cluster.get("new_launcher").unwrap(); + assert_eq!(non_default.executable, Some("e".into())); + assert_eq!(non_default.processes, None); + assert_eq!(non_default.threads_per_process, None); + assert_eq!(non_default.gpus_per_process, None); + + let launchers_by_cluster = launchers.by_cluster("any_cluster"); + let default = launchers_by_cluster.get("new_launcher").unwrap(); + assert_eq!(default.executable, Some("a".into())); + assert_eq!(default.processes, Some("b".into())); + assert_eq!(default.threads_per_process, Some("c".into())); + assert_eq!(default.gpus_per_process, Some("d".into())); + } +} diff --git a/src/lib.rs b/src/lib.rs index 57c7b63..f35dae9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,7 @@ +pub(crate) mod builtin; +pub mod cluster; mod expr; +pub mod launcher; pub mod progress_styles; pub mod project; pub mod scheduler; @@ -18,6 +21,7 @@ pub const MIN_PROGRESS_BAR_SIZE: usize = 1; const VALUE_CACHE_FILE_NAME: &str = "values.json"; const COMPLETED_CACHE_FILE_NAME: &str = "completed.postcard"; +const SUBMITTED_CACHE_FILE_NAME: &str = "submitted.postcard"; /// Hold a MultiProgress and all of its progress bars. /// @@ -34,7 +38,11 @@ pub struct MultiProgressContainer { pub enum Error { // OS errors #[error("OS error")] - OK(#[from] nix::errno::Errno), + OS(#[from] nix::errno::Errno), + + #[error("No home directory")] + NoHome(), + // IO errors #[error("I/O error: {0}")] IO(#[from] io::Error), @@ -110,9 +118,50 @@ pub enum Error { #[error("Cannot compare {0} and {1} while checking directory '{2}'.")] CannotCompareInclude(Value, Value, PathBuf), + // submission errors #[error("Error encountered while executing action '{0}': {1}.")] ExecuteAction(String, String), + #[error("Error encountered while submitting action '{0}': {1}.")] + SubmitAction(String, String), + + #[error("Unepxected output from {0}: {1}")] + UnexpectedOutput(String, String), + + #[error("Error encountered while running squeue: {0}.\n{1}")] + ExecuteSqueue(String, String), + + #[error("Interrupted")] + Interrupted, + + // launcher errors + #[error("Launcher '{0}' does not contain a default configuration")] + LauncherMissingDefault(String), + + #[error("Launcher '{0}' not found: Required by action '{1}'.")] + LauncherNotFound(String, String), + + #[error("No process launcher for action '{0}' which requests {1} processes.")] + NoProcessLauncher(String, usize), + + #[error("More than one process launcher for action '{0}'.")] + TooManyProcessLaunchers(String), + + // cluster errors + #[error( + "Cluster '{0}' not found: execute 'row show cluster --all' to see available clusters." + )] + ClusterNameNotFound(String), + + #[error("No cluster found: execute 'row show cluster -vvv' to see why.")] + ClusterNotFound(), + + #[error("Partition '{0}' not found: execute 'row show cluster' to see available partitions.")] + PartitionNameNotFound(String), + + #[error("No valid partitions:\n{0}\nExecute 'row show cluster' to see available partitions.")] + PartitionNotFound(String), + // command errors #[error("Action '{0}' not found in the workflow.")] ActionNotFound(String), diff --git a/src/main.rs b/src/main.rs index dcb29c0..a3ced34 100644 --- a/src/main.rs +++ b/src/main.rs @@ -80,6 +80,12 @@ fn main_detail() -> Result<(), Box> { &mut multi_progress_container, &mut output, )?, + ShowArgs::Cluster(args) => { + cli::cluster::cluster(options.global_options.clone(), args, &mut output)? + } + ShowArgs::Launchers(args) => { + cli::launchers::launchers(options.global_options.clone(), args, &mut output)? + } }, Some(Commands::Scan(args)) => cli::scan::scan( options.global_options.clone(), diff --git a/src/progress_styles.rs b/src/progress_styles.rs index a6cfeaf..8972c78 100644 --- a/src/progress_styles.rs +++ b/src/progress_styles.rs @@ -1,5 +1,13 @@ use indicatif::ProgressStyle; +pub(crate) const STEADY_TICK: u64 = 110; + +pub fn uncounted_spinner() -> ProgressStyle { + ProgressStyle::with_template("{spinner:.green.bold} {msg:.bold}... ({elapsed:.dim})") + .expect("Valid template") + .tick_strings(&["◐", "◓", "◑", "◒", "⊙"]) +} + pub fn counted_spinner() -> ProgressStyle { ProgressStyle::with_template("{spinner:.green.bold} {msg:.bold}: {human_pos} ({elapsed:.dim})") .expect("Valid template") diff --git a/src/project.rs b/src/project.rs index ae1dcdc..0fd54c7 100644 --- a/src/project.rs +++ b/src/project.rs @@ -1,10 +1,18 @@ +use indicatif::{ProgressBar, ProgressDrawTarget}; use log::{debug, trace, warn}; use serde_json::Value; use std::cmp::Ordering; use std::collections::HashMap; use std::path::PathBuf; +use std::time::Duration; +use crate::cluster::{ClusterConfiguration, SchedulerType}; use crate::expr; +use crate::launcher::LauncherConfiguration; +use crate::progress_styles; +use crate::scheduler::bash::Bash; +use crate::scheduler::slurm::Slurm; +use crate::scheduler::Scheduler; use crate::state::State; use crate::workflow::{Action, Workflow}; use crate::{Error, MultiProgressContainer}; @@ -13,28 +21,28 @@ use crate::{Error, MultiProgressContainer}; /// /// When opened, `Project`: /// +/// * Reads caches from disk. /// * Gets the status of submitted jobs from the scheduler. /// * Collects the staged completions. /// * Reads the workflow file -/// * And synchronizes the system state with the workspace on disk. +/// * Synchronizes the system state with the workspace on disk. +/// * And removes any completed jobs from the submitted cache. /// /// These are common operations used by many CLI commands. A command that needs /// only a subset of these should use the individual classes directly. /// -/// TODO: To provide the most responsive user interface, get the scheduler status -/// asynchronously while reading the cache and synchronizing the workspace. -/// -/// With a required call to `complete_pending_tasks`, saving the updated state -/// and removing the completion staging files can be deferred and running in -/// the background while the command completes. -/// -#[derive(Debug)] pub struct Project { /// The project's workflow definition. workflow: Workflow, /// The state associate with the directories in the project. state: State, + + /// The scheduler. + scheduler: Box, + + /// The cluster's name. + cluster_name: String, } /// Store individual sets of jobs, separated by status for a given action. @@ -64,16 +72,60 @@ impl Project { /// pub fn open( io_threads: u16, + cluster_name: Option, multi_progress: &mut MultiProgressContainer, ) -> Result { trace!("Opening project."); let workflow = Workflow::open()?; + let clusters = ClusterConfiguration::open()?; + let cluster = clusters.identify(cluster_name.as_deref())?; + let launchers = LauncherConfiguration::open()?.by_cluster(&cluster.name); + let cluster_name = cluster.name.clone(); + + let scheduler: Box = match cluster.scheduler { + SchedulerType::Bash => Box::new(Bash::new(cluster, launchers)), + SchedulerType::Slurm => Box::new(Slurm::new(cluster, launchers)), + }; let mut state = State::from_cache(&workflow)?; + // squeue will likely take the longest to finish, start it first. + let jobs = state.jobs_submitted_on(&cluster_name); + let mut progress = + ProgressBar::new_spinner().with_message("Checking submitted job statuses"); + if !jobs.is_empty() { + progress = multi_progress.multi_progress.add(progress); + multi_progress.progress_bars.push(progress.clone()); + progress.enable_steady_tick(Duration::from_millis(progress_styles::STEADY_TICK)); + } else { + progress.set_draw_target(ProgressDrawTarget::hidden()); + // TODO: Refactor these types of code blocks into the MultiProgressContainer? + } + + progress.set_style(progress_styles::uncounted_spinner()); + progress.tick(); + + let active_jobs = scheduler.active_jobs(&jobs)?; + + // Then synchronize with the workspace while squeue is running. state.synchronize_workspace(&workflow, io_threads, multi_progress)?; - Ok(Self { workflow, state }) + // Now, wait for squeue to finish and remove any inactive jobs. + let active_jobs = active_jobs.get()?; + progress.finish(); + + if active_jobs.len() != jobs.len() { + state.remove_inactive_submitted(&cluster_name, &active_jobs); + } else if !jobs.is_empty() { + trace!("All submitted jobs remain active on {cluster_name}."); + } + + Ok(Self { + workflow, + state, + scheduler, + cluster_name, + }) } /// Close the project. @@ -196,10 +248,11 @@ impl Project { } let completed = self.state.completed(); + if completed[&action.name].contains(&directory_name) { status.completed.push(directory_name) - // } else if directory.scheduled_job_ids().contains_key(&action.name) { - // status.submitted.push(name); + } else if self.state.is_submitted(&action.name, &directory_name) { + status.submitted.push(directory_name); } else if action .previous_actions .iter() @@ -296,6 +349,17 @@ impl Project { Ok(result) } + + /// Get the scheduler. + pub fn scheduler(&self) -> &dyn Scheduler { + self.scheduler.as_ref() + } + + /// Add a new submitted job. + pub fn add_submitted(&mut self, action_name: &str, directories: &[PathBuf], job_id: u32) { + self.state + .add_submitted(action_name, directories, &self.cluster_name, job_id); + } } #[cfg(test)] @@ -365,11 +429,11 @@ previous_actions = ["two"] temp.child("workflow.toml").write_str(&workflow).unwrap(); - Project::open(2, &mut multi_progress).unwrap() + Project::open(2, None, &mut multi_progress).unwrap() } #[test] - #[serial(set_current_dir)] + #[serial] fn matching() { let project = setup(8); @@ -406,7 +470,7 @@ previous_actions = ["two"] } #[test] - #[serial(set_current_dir)] + #[serial] fn status() { let project = setup(8); @@ -442,7 +506,7 @@ previous_actions = ["two"] } #[test] - #[serial(set_current_dir)] + #[serial] fn group() { let project = setup(8); @@ -457,7 +521,7 @@ previous_actions = ["two"] } #[test] - #[serial(set_current_dir)] + #[serial] fn group_reverse() { let project = setup(8); @@ -475,7 +539,7 @@ previous_actions = ["two"] } #[test] - #[serial(set_current_dir)] + #[serial] fn group_max_size() { let project = setup(8); @@ -498,7 +562,7 @@ previous_actions = ["two"] } #[test] - #[serial(set_current_dir)] + #[serial] fn group_sort() { let project = setup(8); @@ -526,7 +590,7 @@ previous_actions = ["two"] } #[test] - #[serial(set_current_dir)] + #[serial] fn group_sort_and_split() { let project = setup(8); diff --git a/src/scheduler.rs b/src/scheduler.rs index df7b1e0..e3dc341 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -1,7 +1,8 @@ pub mod bash; +pub mod slurm; -use std::path::Path; -use std::path::PathBuf; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicBool; use std::sync::Arc; @@ -10,15 +11,13 @@ use crate::Error; /// A `Scheduler` creates and submits job scripts. pub trait Scheduler { - // TODO: give new schedulers the Launcher and partition instances from Cluster. - fn new(cluster_name: &str) -> Self; - /// Make a job script given an `Action` and a list of directories. /// /// Useful for showing the script that would be submitted to the user. /// /// # Returns /// A `String` containing the job script. + /// fn make_script(&self, action: &Action, directories: &[PathBuf]) -> Result; /// Submit a job to the scheduler. @@ -48,5 +47,21 @@ pub trait Scheduler { should_terminate: Arc, ) -> Result, Error>; - // TODO: status -> run squeue and determine running jobs. + /// Query the scheduler and determine which jobs remain active. + /// + /// # Arguments + /// * `jobs`: Identifiers to query + /// + /// `active_jobs` returns a ActiveJobs object, which provides the final + /// result via a method. This allows implementations to be asynchronous so + /// that long-running subprocesses can complete in the background while the + /// collar performs other work. + /// + fn active_jobs(&self, jobs: &[u32]) -> Result, Error>; +} + +/// Deferred result containing jobs that are still active on the cluster. +pub trait ActiveJobs { + /// Complete the operation and return the currently active jobs. + fn get(self: Box) -> Result, Error>; } diff --git a/src/scheduler/bash.rs b/src/scheduler/bash.rs index 00a1480..09e983b 100644 --- a/src/scheduler/bash.rs +++ b/src/scheduler/bash.rs @@ -1,29 +1,32 @@ use log::{debug, error, trace}; use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; +use std::collections::{HashMap, HashSet}; use std::env; use std::io::Write; use std::os::unix::process::ExitStatusExt; -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::thread; use std::time::Duration; -use crate::scheduler::Scheduler; +use crate::cluster::Cluster; +use crate::launcher::Launcher; +use crate::scheduler::{ActiveJobs, Scheduler}; use crate::workflow::{Action, Processes}; use crate::Error; /// `BashScriptBuilder` builds `bash` scripts that execute row actions. -struct BashScriptBuilder<'a> { +pub(crate) struct BashScriptBuilder<'a> { walltime_in_minutes: i64, total_processes: usize, cluster_name: &'a str, action: &'a Action, directories: &'a [PathBuf], preamble: &'a str, + launchers: &'a HashMap, } impl<'a> BashScriptBuilder<'a> { @@ -32,6 +35,7 @@ impl<'a> BashScriptBuilder<'a> { cluster_name: &'a str, action: &'a Action, directories: &'a [PathBuf], + launchers: &'a HashMap, ) -> Self { let walltime_in_minutes = action .resources @@ -46,6 +50,7 @@ impl<'a> BashScriptBuilder<'a> { action, directories, preamble: "", + launchers, } } @@ -114,28 +119,32 @@ export ACTION_WALLTIME_IN_MINUTES="{}" } fn setup(&self) -> Result { - let mut user_setup = self + let mut result = String::new(); + let user_setup = self .action - .cluster + .submit_options .get(self.cluster_name) .and_then(|c| c.setup.clone()) .unwrap_or_default(); + if !user_setup.is_empty() { - user_setup.push_str( - r#" -test $? -eq 0 || { >&2 echo "[row] Error executing setup."; exit 1; }"#, + result.push('\n'); + result.push_str(&user_setup); + result.push_str("\n\n"); + result.push_str( + r#"test $? -eq 0 || { >&2 echo "[row] Error executing setup."; exit 1; }"#, ); } let action_name = &self.action.name; let row_executable = env::current_exe().map_err(Error::FindCurrentExecutable)?; let row_executable = row_executable.to_str().expect("UTF-8 path to executable."); - user_setup.push_str(&format!( + result.push_str(&format!( r#" trap 'printf %s\\n "${{directories[@]}}" | {row_executable} scan --no-progress -a {action_name} - || exit 3' EXIT"# )); - Ok(user_setup) + Ok(result) } fn execution(&self) -> Result { @@ -147,7 +156,29 @@ trap 'printf %s\\n "${{directories[@]}}" | {row_executable} scan --no-progress - )); } - // TODO: Apply launcher. + // Build up launcher prefix + let mut launcher_prefix = String::new(); + let mut process_launchers = 0; + for launcher in &self.action.launchers { + let launcher = self.launchers.get(launcher).ok_or_else(|| { + Error::LauncherNotFound(launcher.clone(), self.action.name.clone()) + })?; + launcher_prefix + .push_str(&launcher.prefix(&self.action.resources, self.directories.len())); + if launcher.processes.is_some() { + process_launchers += 1; + } + } + + if self.total_processes > 1 && process_launchers == 0 { + return Err(Error::NoProcessLauncher( + self.action.name.clone(), + self.total_processes, + )); + } + if process_launchers > 1 { + return Err(Error::TooManyProcessLaunchers(self.action.name.clone())); + } if contains_directory { let command = self.action.command.replace("{directory}", "$directory"); @@ -155,7 +186,7 @@ trap 'printf %s\\n "${{directories[@]}}" | {row_executable} scan --no-progress - r#" for directory in "${{directories[@]}}" do - {command} || {{ >&2 echo "[ERROR row::action] Error executing command."; exit 2; }} + {launcher_prefix}{command} || {{ >&2 echo "[ERROR row::action] Error executing command."; exit 2; }} done "# )) @@ -166,7 +197,7 @@ done .replace("{directories}", r#""${directories[@]}""#); Ok(format!( r#" -{command} || {{ >&2 echo "[row] Error executing command."; exit 1; }} +{launcher_prefix}{command} || {{ >&2 echo "[row] Error executing command."; exit 1; }} "# )) } else { @@ -181,26 +212,22 @@ done /// The `Bash` scheduler constructs bash scripts and executes them with `bash`. pub struct Bash { - /// The name of the cluster. - cluster_name: String, - // TODO: store partition and launcher (or maybe whole Cluster object reference?) + cluster: Cluster, + launchers: HashMap, } -impl Bash {} - -impl Scheduler for Bash { - fn new(cluster_name: &str) -> Self { - Self { - cluster_name: cluster_name.to_string(), - } +impl Bash { + /// Construct a new Bash scheduler. + pub fn new(cluster: Cluster, launchers: HashMap) -> Self { + Self { cluster, launchers } } +} +pub struct ActiveBashJobs {} + +impl Scheduler for Bash { fn make_script(&self, action: &Action, directories: &[PathBuf]) -> Result { - // TODO: Remove with_preamble when the slurm code is written. - // it is here only to hide a dead code warning. - BashScriptBuilder::new(&self.cluster_name, action, directories) - .with_preamble("") - .build() + BashScriptBuilder::new(&self.cluster.name, action, directories, &self.launchers).build() } fn submit( @@ -210,8 +237,8 @@ impl Scheduler for Bash { directories: &[PathBuf], should_terminate: Arc, ) -> Result, Error> { - let script = BashScriptBuilder::new(&self.cluster_name, action, directories).build()?; debug!("Executing '{}' in bash.", action.name); + let script = self.make_script(action, directories)?; let mut child = Command::new("bash") .stdin(Stdio::piped()) @@ -255,17 +282,35 @@ impl Scheduler for Bash { Ok(None) } + + /// Bash reports no active jobs. + /// + /// All jobs are executed immediately on submission. + /// + fn active_jobs(&self, _: &[u32]) -> Result, Error> { + Ok(Box::new(ActiveBashJobs {})) + } +} + +impl ActiveJobs for ActiveBashJobs { + fn get(self: Box) -> Result, Error> { + Ok(HashSet::new()) + } } #[cfg(test)] mod tests { use super::*; + use serial_test::parallel; use speedate::Duration; + use crate::builtin::BuiltIn; + use crate::cluster::{IdentificationMethod, SchedulerType}; + use crate::launcher::LauncherConfiguration; use crate::workflow::Walltime; - use crate::workflow::{ClusterParameters, Resources}; + use crate::workflow::{Resources, SubmitOptions}; - fn setup() -> (Action, Vec) { + fn setup() -> (Action, Vec, HashMap) { let resources = Resources { processes: Processes::PerDirectory(2), threads_per_process: Some(4), @@ -273,24 +318,26 @@ mod tests { walltime: Walltime::PerSubmission( Duration::new(true, 0, 240, 0).expect("Valid duration."), ), - ..Resources::default() }; let action = Action { name: "action".to_string(), command: "command {directory}".to_string(), + launchers: vec!["mpi".into()], resources, ..Action::default() }; let directories = vec![PathBuf::from("a"), PathBuf::from("b"), PathBuf::from("c")]; - (action, directories) + let launchers = LauncherConfiguration::built_in(); + (action, directories, launchers.by_cluster("cluster")) } #[test] - fn test_header() { - let (action, directories) = setup(); - let script = BashScriptBuilder::new("cluster", &action, &directories) + #[parallel] + fn header() { + let (action, directories, launchers) = setup(); + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .build() .expect("Valid script."); println!("{script}"); @@ -299,9 +346,10 @@ mod tests { } #[test] - fn test_preamble() { - let (action, directories) = setup(); - let script = BashScriptBuilder::new("cluster", &action, &directories) + #[parallel] + fn preamble() { + let (action, directories, launchers) = setup(); + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .with_preamble("#preamble") .build() .expect("Valid script."); @@ -311,9 +359,10 @@ mod tests { } #[test] - fn test_no_setup() { - let (action, directories) = setup(); - let script = BashScriptBuilder::new("cluster", &action, &directories) + #[parallel] + fn no_setup() { + let (action, directories, launchers) = setup(); + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .build() .expect("Valid script."); println!("{script}"); @@ -322,20 +371,21 @@ mod tests { } #[test] - fn test_setup() { - let (mut action, directories) = setup(); + #[parallel] + fn with_setup() { + let (mut action, directories, launchers) = setup(); action - .cluster - .insert("cluster".to_string(), ClusterParameters::default()); + .submit_options + .insert("cluster".to_string(), SubmitOptions::default()); - let script = BashScriptBuilder::new("cluster", &action, &directories) + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .build() .expect("Valid script."); println!("{script}"); assert!(!script.contains("test $? -eq 0 ||")); - action.cluster.get_mut("cluster").unwrap().setup = Some("my setup".to_string()); - let script = BashScriptBuilder::new("cluster", &action, &directories) + action.submit_options.get_mut("cluster").unwrap().setup = Some("my setup".to_string()); + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .build() .expect("Valid script."); println!("{script}"); @@ -344,9 +394,10 @@ mod tests { } #[test] - fn test_execution_directory() { - let (action, directories) = setup(); - let script = BashScriptBuilder::new("cluster", &action, &directories) + #[parallel] + fn execution_directory() { + let (action, directories, launchers) = setup(); + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .build() .expect("Valid script."); println!("{script}"); @@ -355,11 +406,12 @@ mod tests { } #[test] - fn test_execution_directories() { - let (mut action, directories) = setup(); + #[parallel] + fn execution_directories() { + let (mut action, directories, launchers) = setup(); action.command = "command {directories}".to_string(); - let script = BashScriptBuilder::new("cluster", &action, &directories) + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .build() .expect("Valid script."); println!("{script}"); @@ -368,11 +420,45 @@ mod tests { } #[test] - fn test_command_errors() { - let (mut action, directories) = setup(); + #[parallel] + fn execution_openmp() { + let (mut action, directories, launchers) = setup(); + action.resources.processes = Processes::PerSubmission(1); + action.launchers = vec!["openmp".into()]; + action.command = "command {directories}".to_string(); + + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) + .build() + .expect("Valid script."); + println!("{script}"); + + assert!(script.contains("OMP_NUM_THREADS=4 command \"${directories[@]}\"")); + } + + #[test] + #[parallel] + fn execution_mpi() { + let (mut action, directories, launchers) = setup(); + action.launchers = vec!["mpi".into()]; + action.command = "command {directories}".to_string(); + + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) + .build() + .expect("Valid script."); + println!("{script}"); + + assert!(script.contains( + "srun --ntasks=6 --cpus-per-task=4 --gpus-per-task=1 command \"${directories[@]}\"" + )); + } + + #[test] + #[parallel] + fn command_errors() { + let (mut action, directories, launchers) = setup(); action.command = "command {directory} {directories}".to_string(); - let result = BashScriptBuilder::new("cluster", &action, &directories).build(); + let result = BashScriptBuilder::new("cluster", &action, &directories, &launchers).build(); assert!(matches!( result, @@ -381,7 +467,7 @@ mod tests { action.command = "command".to_string(); - let result = BashScriptBuilder::new("cluster", &action, &directories).build(); + let result = BashScriptBuilder::new("cluster", &action, &directories, &launchers).build(); assert!(matches!( result, @@ -390,9 +476,10 @@ mod tests { } #[test] - fn test_variables() { - let (action, directories) = setup(); - let script = BashScriptBuilder::new("cluster", &action, &directories) + #[parallel] + fn variables() { + let (action, directories, launchers) = setup(); + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .build() .expect("Valid script."); @@ -408,15 +495,16 @@ mod tests { } #[test] - fn test_more_variables() { - let (mut action, directories) = setup(); + #[parallel] + fn more_variables() { + let (mut action, directories, launchers) = setup(); action.resources.processes = Processes::PerSubmission(10); action.resources.walltime = Walltime::PerDirectory(Duration::new(true, 0, 60, 0).expect("Valid duration.")); action.resources.threads_per_process = None; action.resources.gpus_per_process = None; - let script = BashScriptBuilder::new("cluster", &action, &directories) + let script = BashScriptBuilder::new("cluster", &action, &directories, &launchers) .build() .expect("Valid script."); @@ -432,13 +520,45 @@ mod tests { } #[test] - fn test_scheduler() { - let (action, directories) = setup(); - let script = Bash::new("cluster") + #[parallel] + fn scheduler() { + let (action, directories, launchers) = setup(); + let cluster = Cluster { + name: "cluster".into(), + scheduler: SchedulerType::Bash, + identify: IdentificationMethod::Always(false), + partition: Vec::new(), + }; + let script = Bash::new(cluster, launchers) .make_script(&action, &directories) .expect("Valid script"); println!("{script}"); assert!(script.contains("command $directory")); } + + #[test] + #[parallel] + fn launcher_required() { + let (mut action, directories, launchers) = setup(); + action.launchers = vec![]; + action.command = "command {directories}".to_string(); + + let result = BashScriptBuilder::new("cluster", &action, &directories, &launchers).build(); + + assert!(matches!(result, Err(Error::NoProcessLauncher(_, _)))); + } + + #[test] + #[parallel] + fn too_many_launchers() { + let (mut action, directories, launchers) = setup(); + action.resources.processes = Processes::PerSubmission(1); + action.launchers = vec!["mpi".into(), "mpi".into()]; + action.command = "command {directories}".to_string(); + + let result = BashScriptBuilder::new("cluster", &action, &directories, &launchers).build(); + + assert!(matches!(result, Err(Error::TooManyProcessLaunchers(_)))); + } } diff --git a/src/scheduler/slurm.rs b/src/scheduler/slurm.rs new file mode 100644 index 0000000..a78489a --- /dev/null +++ b/src/scheduler/slurm.rs @@ -0,0 +1,521 @@ +use log::{debug, error, trace}; +use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; +use std::io::Write; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::{str, thread}; + +use crate::cluster::Cluster; +use crate::launcher::Launcher; +use crate::scheduler::bash::BashScriptBuilder; +use crate::scheduler::{ActiveJobs, Scheduler}; +use crate::workflow::Action; +use crate::Error; + +/// The `Slurm` scheduler constructs bash scripts and executes them with `sbatch`. +pub struct Slurm { + cluster: Cluster, + launchers: HashMap, +} + +impl Slurm { + /// Construct a new Slurm scheduler. + pub fn new(cluster: Cluster, launchers: HashMap) -> Self { + Self { cluster, launchers } + } +} + +/// Track the running squeue process +/// +/// Or `None` when no process was launched. +pub struct ActiveSlurmJobs { + squeue: Option, + max_jobs: usize, +} + +impl Scheduler for Slurm { + fn make_script(&self, action: &Action, directories: &[PathBuf]) -> Result { + let mut preamble = String::with_capacity(512); + let mut user_partition = &None; + + write!(preamble, "#SBATCH --job-name={}", action.name).expect("valid format"); + let _ = match directories.first() { + Some(directory) => match directories.len() { + 0..=1 => writeln!(preamble, "-{}", directory.display()), + _ => writeln!( + preamble, + "-{}+{}", + directory.display(), + directories.len() - 1 + ), + }, + None => writeln!(preamble), + }; + + let _ = writeln!(preamble, "#SBATCH --output={}-%j.out", action.name); + + if let Some(submit_options) = action.submit_options.get(&self.cluster.name) { + user_partition = &submit_options.partition; + } + + // The partition + let partition = self.cluster.find_partition( + user_partition.as_deref(), + &action.resources, + directories.len(), + )?; + let _ = writeln!(preamble, "#SBATCH --partition={}", partition.name); + + // Resources + let _ = writeln!( + preamble, + "#SBATCH --ntasks={}", + action.resources.total_processes(directories.len()) + ); + + if let Some(threads_per_process) = action.resources.threads_per_process { + let _ = writeln!(preamble, "#SBATCH --cpus-per-task={threads_per_process}"); + } + if let Some(gpus_per_process) = action.resources.gpus_per_process { + let _ = writeln!(preamble, "#SBATCH --gpus-per-task={gpus_per_process}"); + + if let Some(ref gpus_per_node) = partition.gpus_per_node { + let n_nodes = (action.resources.total_gpus(directories.len()) + gpus_per_node - 1) + / gpus_per_node; + let _ = writeln!(preamble, "#SBATCH --nodes={n_nodes}"); + } + + if let Some(ref mem_per_gpu) = partition.memory_per_gpu { + let _ = writeln!(preamble, "#SBATCH --mem-per-gpu={mem_per_gpu}"); + } + } else { + if let Some(ref cpus_per_node) = partition.cpus_per_node { + let n_nodes = (action.resources.total_cpus(directories.len()) + cpus_per_node - 1) + / cpus_per_node; + let _ = writeln!(preamble, "#SBATCH --nodes={n_nodes}"); + } + + if let Some(ref mem_per_cpu) = partition.memory_per_cpu { + let _ = writeln!(preamble, "#SBATCH --mem-per-cpu={mem_per_cpu}"); + } + } + + // Slurm doesn't store times in seconds, so round up to the nearest minute. + let total = action + .resources + .total_walltime(directories.len()) + .signed_total_seconds(); + let minutes = (total + 59) / 60; + let _ = writeln!(preamble, "#SBATCH --time={minutes}"); + + // Use provided submission options + if let Some(submit_options) = action.submit_options.get(&self.cluster.name) { + if let Some(ref account) = submit_options.account { + if let Some(ref suffix) = partition.account_suffix { + let _ = writeln!(preamble, "#SBATCH --account={account}{suffix}"); + } else { + let _ = writeln!(preamble, "#SBATCH --account={account}"); + } + } + for option in &submit_options.custom { + let _ = writeln!(preamble, "#SBATCH {option}"); + } + } + + BashScriptBuilder::new(&self.cluster.name, action, directories, &self.launchers) + .with_preamble(&preamble) + .build() + } + + fn submit( + &self, + working_directory: &Path, + action: &Action, + directories: &[PathBuf], + should_terminate: Arc, + ) -> Result, Error> { + debug!("Submtitting '{}' with sbatch.", action.name); + + // output() below is blocking with no convenient way to interrupt it. + // If the user pressed ctrl-C, let the current call to submit() finish + // and update the cache. Assuming that there will be a next call to + // submit(), that next call will return with an Interrupted error before + // submitting the next job. + if should_terminate.load(Ordering::Relaxed) { + error!("Interrupted! Cancelling further job submissions."); + return Err(Error::Interrupted); + } + + let script = self.make_script(action, directories)?; + + let mut child = Command::new("sbatch") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .arg("--parsable") + .current_dir(working_directory) + .spawn() + .map_err(|e| Error::SpawnProcess("sbatch".into(), e))?; + + let mut stdin = child.stdin.take().expect("Piped stdin"); + let input_thread = thread::spawn(move || { + let _ = write!(stdin, "{}", script); + }); + + trace!("Waiting for sbatch to complete."); + let output = child + .wait_with_output() + .map_err(|e| Error::SpawnProcess("sbatch".into(), e))?; + + input_thread.join().expect("The thread should not panic"); + + if !output.status.success() { + let message = match output.status.code() { + None => match output.status.signal() { + None => "sbatch was terminated by a unknown signal".to_string(), + Some(signal) => format!("sbatch was terminated by signal {signal}"), + }, + Some(code) => format!("sbatch exited with code {code}"), + }; + Err(Error::SubmitAction(action.name.clone(), message)) + } else { + let job_id_string = str::from_utf8(&output.stdout).expect("Valid UTF-8 output"); + let job_id = job_id_string + .trim_end_matches(char::is_whitespace) + .parse::() + .map_err(|_| Error::UnexpectedOutput("sbatch".into(), job_id_string.into()))?; + Ok(Some(job_id)) + } + } + + /// Use `squeue` to determine the jobs that are still present in the queue. + /// + /// Launch `squeue --jobs job0,job1,job2 -o "%A" --noheader` to determine which of + /// these jobs are still in the queue. + /// + fn active_jobs(&self, jobs: &[u32]) -> Result, Error> { + if jobs.is_empty() { + return Ok(Box::new(ActiveSlurmJobs { + squeue: None, + max_jobs: 0, + })); + } + + debug!("Checking job status with squeue."); + + let mut jobs_string = String::with_capacity(9 * jobs.len()); + // Prefix the --jobs argument with "1,". Otherwise, squeue reports an + // error when a single job is not in the queue. + if jobs.len() == 1 { + jobs_string.push_str("1,"); + } + for job in jobs { + let _ = write!(jobs_string, "{},", job); + } + + let squeue = Command::new("squeue") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .arg("--jobs") + .arg(&jobs_string) + .args(["-o", "%A"]) + .arg("--noheader") + .spawn() + .map_err(|e| Error::SpawnProcess("squeue".into(), e))?; + + Ok(Box::new(ActiveSlurmJobs { + squeue: Some(squeue), + max_jobs: jobs.len(), + })) + } +} + +impl ActiveJobs for ActiveSlurmJobs { + fn get(self: Box) -> Result, Error> { + let mut result = HashSet::with_capacity(self.max_jobs); + + if let Some(squeue) = self.squeue { + trace!("Waiting for squeue to complete."); + let output = squeue + .wait_with_output() + .map_err(|e| Error::SpawnProcess("sbatch".into(), e))?; + + if !output.status.success() { + let message = match output.status.code() { + None => match output.status.signal() { + None => "squeue was terminated by a unknown signal".to_string(), + Some(signal) => format!("squeue was terminated by signal {signal}"), + }, + Some(code) => format!("squeue exited with code {code}"), + }; + return Err(Error::ExecuteSqueue( + message, + str::from_utf8(&output.stderr).expect("Valid UTF-8").into(), + )); + } + + let jobs = str::from_utf8(&output.stdout).expect("Valid UTF-8"); + for job in jobs.lines() { + result.insert( + job.parse() + .map_err(|_| Error::UnexpectedOutput("squeue".into(), job.into()))?, + ); + } + } + + Ok(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serial_test::parallel; + + use crate::builtin::BuiltIn; + use crate::cluster::{Cluster, IdentificationMethod, Partition, SchedulerType}; + use crate::launcher::LauncherConfiguration; + use crate::workflow::{Processes, SubmitOptions}; + + fn setup() -> (Action, Vec, Slurm) { + let action = Action { + name: "action".to_string(), + command: "command {directory}".to_string(), + launchers: vec!["mpi".into()], + ..Action::default() + }; + + let directories = vec![PathBuf::from("a"), PathBuf::from("b"), PathBuf::from("c")]; + let launchers = LauncherConfiguration::built_in(); + let cluster = Cluster { + name: "cluster".into(), + identify: IdentificationMethod::Always(false), + scheduler: SchedulerType::Slurm, + partition: vec![Partition::default()], + }; + + let slurm = Slurm::new(cluster, launchers.by_cluster("cluster")); + (action, directories, slurm) + } + + #[test] + #[parallel] + fn default() { + let (action, directories, slurm) = setup(); + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --job-name=action")); + assert!(script.contains("#SBATCH --ntasks=1")); + assert!(!script.contains("#SBATCH --account")); + assert!(script.contains("#SBATCH --partition=partition")); + assert!(!script.contains("#SBATCH --cpus-per-task")); + assert!(!script.contains("#SBATCH --gpus-per-task")); + assert!(script.contains("#SBATCH --time=180")); + } + + #[test] + #[parallel] + fn ntasks() { + let (mut action, directories, slurm) = setup(); + + action.resources.processes = Processes::PerDirectory(3); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --ntasks=9")); + } + + #[test] + #[parallel] + fn account() { + let (mut action, directories, slurm) = setup(); + + action.submit_options.insert( + "cluster".into(), + SubmitOptions { + account: Some("c".into()), + ..SubmitOptions::default() + }, + ); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --account=c")); + } + + #[test] + #[parallel] + fn custom() { + let (mut action, directories, slurm) = setup(); + + action.submit_options.insert( + "cluster".into(), + SubmitOptions { + custom: vec!["custom0".into(), "custom1".into()], + ..SubmitOptions::default() + }, + ); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH custom0")); + assert!(script.contains("#SBATCH custom1")); + } + + #[test] + #[parallel] + fn cpus_per_task() { + let (mut action, directories, slurm) = setup(); + + action.resources.threads_per_process = Some(5); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --cpus-per-task=5")); + } + + #[test] + #[parallel] + fn gpus_per_task() { + let (mut action, directories, slurm) = setup(); + + action.resources.gpus_per_process = Some(5); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --gpus-per-task=5")); + } + + #[test] + #[parallel] + fn mem_per_cpu() { + let (action, directories, _) = setup(); + + let launchers = LauncherConfiguration::built_in(); + let cluster = Cluster { + name: "cluster".into(), + identify: IdentificationMethod::Always(false), + scheduler: SchedulerType::Slurm, + partition: vec![Partition { + memory_per_cpu: Some("a".into()), + ..Partition::default() + }], + }; + + let slurm = Slurm::new(cluster, launchers.by_cluster("cluster")); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --mem-per-cpu=a")); + } + + #[test] + #[parallel] + fn mem_per_gpu() { + let (mut action, directories, _) = setup(); + + let launchers = LauncherConfiguration::built_in(); + let cluster = Cluster { + name: "cluster".into(), + identify: IdentificationMethod::Always(false), + scheduler: SchedulerType::Slurm, + partition: vec![Partition { + memory_per_gpu: Some("b".into()), + ..Partition::default() + }], + }; + + let slurm = Slurm::new(cluster, launchers.by_cluster("cluster")); + + action.resources.gpus_per_process = Some(1); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --mem-per-gpu=b")); + } + + #[test] + #[parallel] + fn cpus_per_node() { + let (mut action, directories, _) = setup(); + + let launchers = LauncherConfiguration::built_in(); + let cluster = Cluster { + name: "cluster".into(), + identify: IdentificationMethod::Always(false), + scheduler: SchedulerType::Slurm, + partition: vec![Partition { + cpus_per_node: Some(10), + ..Partition::default() + }], + }; + + let slurm = Slurm::new(cluster, launchers.by_cluster("cluster")); + + action.resources.processes = Processes::PerSubmission(81); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --nodes=9")); + } + + #[test] + #[parallel] + fn gpus_per_node() { + let (mut action, directories, _) = setup(); + + let launchers = LauncherConfiguration::built_in(); + let cluster = Cluster { + name: "cluster".into(), + identify: IdentificationMethod::Always(false), + scheduler: SchedulerType::Slurm, + partition: vec![Partition { + gpus_per_node: Some(5), + ..Partition::default() + }], + }; + + let slurm = Slurm::new(cluster, launchers.by_cluster("cluster")); + + action.resources.processes = Processes::PerSubmission(81); + action.resources.gpus_per_process = Some(1); + + let script = slurm + .make_script(&action, &directories) + .expect("valid script"); + println!("{script}"); + + assert!(script.contains("#SBATCH --nodes=17")); + } +} diff --git a/src/state.rs b/src/state.rs index 88f3f9f..a1a7f25 100644 --- a/src/state.rs +++ b/src/state.rs @@ -11,9 +11,12 @@ use std::path::PathBuf; use crate::workflow::Workflow; use crate::{ progress_styles, workspace, Error, MultiProgressContainer, COMPLETED_CACHE_FILE_NAME, - COMPLETED_DIRECTORY_NAME, DATA_DIRECTORY_NAME, MIN_PROGRESS_BAR_SIZE, VALUE_CACHE_FILE_NAME, + COMPLETED_DIRECTORY_NAME, DATA_DIRECTORY_NAME, MIN_PROGRESS_BAR_SIZE, + SUBMITTED_CACHE_FILE_NAME, VALUE_CACHE_FILE_NAME, }; +type SubmittedJobs = HashMap>; + /// The state of the project. /// /// `State` collects the following information on the workspace and manages cache files @@ -33,14 +36,20 @@ pub struct State { /// Completed directories for each action. completed: HashMap>, + /// Submitted jobs: action -> directory -> (cluster, job ID) + submitted: SubmittedJobs, + /// Completion files read while synchronizing. completed_file_names: Vec, - // TODO: scheduled jobs + /// Set to true when `values` is modified from the on-disk cache. values_modified: bool, /// Set to true when `completed` is modified from the on-disk cache. completed_modified: bool, + + /// Set to true when `submitted` is modified from the on-disk cache. + submitted_modified: bool, } impl State { @@ -54,6 +63,70 @@ impl State { &self.completed } + /// Get the mapping of actions -> directories -> (cluster, submitted job ID) + pub fn submitted(&self) -> &SubmittedJobs { + &self.submitted + } + + /// Test whether a given directory has a submitted job for the given action. + pub fn is_submitted(&self, action_name: &str, directory: &PathBuf) -> bool { + if let Some(submitted_directories) = self.submitted.get(action_name) { + submitted_directories.contains_key(directory) + } else { + false + } + } + + /// Add a submitted job. + pub fn add_submitted( + &mut self, + action_name: &str, + directories: &[PathBuf], + cluster_name: &str, + job_id: u32, + ) { + for directory in directories { + self.submitted + .entry(action_name.into()) + .and_modify(|e| { + e.insert(directory.clone(), (cluster_name.to_string(), job_id)); + }) + .or_insert(HashMap::from([( + directory.clone(), + (cluster_name.to_string(), job_id), + )])); + } + self.submitted_modified = true; + } + + /// Remove inactive jobs on the given cluster. + /// + /// Note: The argument lists the *active* jobs to keep! + /// + pub fn remove_inactive_submitted(&mut self, cluster_name: &str, active_job_ids: &HashSet) { + trace!("Removing inactive jobs from the submitted cache."); + self.submitted_modified = true; + + for directories in self.submitted.values_mut() { + directories.retain(|_, v| v.0 != cluster_name || active_job_ids.contains(&v.1)); + } + } + + /// Get all submitted jobs on a given cluster. + pub fn jobs_submitted_on(&self, cluster_name: &str) -> Vec { + let mut set: HashSet = HashSet::new(); + + for directories in self.submitted.values() { + for (job_cluster, job_id) in directories.values() { + if job_cluster == cluster_name { + set.insert(*job_id); + } + } + } + + Vec::from_iter(set.drain()) + } + /// List all directories in the state. pub fn list_directories(&self) -> Vec { trace!("Listing all directories in project."); @@ -67,9 +140,11 @@ impl State { let mut state = State { values: Self::read_value_cache(workflow)?, completed: Self::read_completed_cache(workflow)?, + submitted: Self::read_submitted_cache(workflow)?, completed_file_names: Vec::new(), values_modified: false, completed_modified: false, + submitted_modified: false, }; // Ensure that completed has keys for all actions in the workflow. @@ -98,7 +173,7 @@ impl State { } Err(error) => match error.kind() { io::ErrorKind::NotFound => { - debug!( + trace!( "'{}' not found, initializing default values.", value_file.display().to_string() ); @@ -127,7 +202,7 @@ impl State { } Err(error) => match error.kind() { io::ErrorKind::NotFound => { - debug!( + trace!( "'{}' not found, initializing empty completions.", completed_file.display().to_string() ); @@ -139,6 +214,33 @@ impl State { } } + /// Read the submitted job cache from disk. + fn read_submitted_cache(workflow: &Workflow) -> Result { + let data_directory = workflow.root.join(DATA_DIRECTORY_NAME); + let submitted_file = data_directory.join(SUBMITTED_CACHE_FILE_NAME); + + match fs::read(&submitted_file) { + Ok(bytes) => { + debug!("Reading cache '{}'.", submitted_file.display().to_string()); + + let result = postcard::from_bytes(&bytes) + .map_err(|e| Error::PostcardParse(submitted_file, e))?; + Ok(result) + } + Err(error) => match error.kind() { + io::ErrorKind::NotFound => { + debug!( + "'{}' not found, assuming no submitted jobs.", + submitted_file.display().to_string() + ); + Ok(HashMap::new()) + } + + _ => Err(Error::FileRead(submitted_file, error)), + }, + } + } + /// Save the state cache to the filesystem. pub fn save_cache( &mut self, @@ -155,6 +257,11 @@ impl State { self.completed_modified = false; } + if self.submitted_modified { + self.save_submitted_cache(workflow)?; + self.submitted_modified = false; + } + Ok(()) } @@ -227,19 +334,45 @@ impl State { Ok(()) } + /// Save the completed cache to the filesystem. + fn save_submitted_cache(&mut self, workflow: &Workflow) -> Result<(), Error> { + let data_directory = workflow.root.join(DATA_DIRECTORY_NAME); + let submitted_file = data_directory.join(SUBMITTED_CACHE_FILE_NAME); + + debug!( + "Saving submitted job cache: '{}'.", + submitted_file.display().to_string() + ); + + let out_bytes: Vec = postcard::to_stdvec(&self.submitted) + .map_err(|e| Error::PostcardSerialize(submitted_file.clone(), e))?; + + let mut file = File::create(&submitted_file) + .map_err(|e| Error::FileWrite(submitted_file.clone(), e))?; + file.write_all(&out_bytes) + .map_err(|e| Error::FileWrite(submitted_file.clone(), e))?; + file.sync_all() + .map_err(|e| Error::FileWrite(submitted_file.clone(), e))?; + drop(file); + + Ok(()) + } + /// Synchronize a workspace on disk with a `State`. /// /// * Remove directories from the state that are no longer present on the filesystem. /// * Make no changes to directories in the state that remain. /// * When new directories are present on the filesystem, add them to the state - /// which includes reading the value file and checking which actions are completed. + /// * Remove actions that are no longer present from the completed and submitted caches. + /// * Remove directories that are no longer present from the completed and submitted caches. /// /// # Errors /// /// * Returns `Error` when there is an I/O error reading the /// workspace directory /// - pub fn synchronize_workspace( + pub(crate) fn synchronize_workspace( &mut self, workflow: &Workflow, io_threads: u16, @@ -325,6 +458,7 @@ impl State { self.insert_staged_completed(new_complete); self.remove_missing_completed(workflow); + self.remove_missing_submitted(workflow); Ok(self) } @@ -374,8 +508,44 @@ impl State { } } + /// Remove missing submitted actions and directories. + fn remove_missing_submitted(&mut self, workflow: &Workflow) { + let current_actions: HashSet = + workflow.action.iter().map(|a| a.name.clone()).collect(); + + let actions_to_remove: Vec = self + .submitted + .keys() + .filter(|a| !current_actions.contains(*a)) + .cloned() + .collect(); + + for action_name in actions_to_remove { + warn!("Removing action '{}' from the submitted cache as it is no longer present in the workflow.", action_name); + self.submitted.remove(&action_name); + self.submitted_modified = true; + } + + for (_, directory_map) in self.submitted.iter_mut() { + let directories_to_remove: Vec = directory_map + .keys() + .filter(|d| !self.values.contains_key(*d)) + .cloned() + .collect(); + + for directory_name in directories_to_remove { + trace!("Removing directory '{}' from the submitted cache as it is no longer present in the workspace.", directory_name.display()); + directory_map.remove(&directory_name); + self.submitted_modified = true; + } + } + + // Note: A separate method takes care of removing submitted job IDs that are + // no longer submitted. + } + /// Synchronize with completion files on the filesystem. - pub fn synchronize_completion_files( + fn synchronize_completion_files( &mut self, workflow: &Workflow, multi_progress: &mut MultiProgressContainer, @@ -466,6 +636,7 @@ mod tests { use assert_fs::prelude::*; use assert_fs::TempDir; use indicatif::{MultiProgress, ProgressDrawTarget}; + use serial_test::parallel; use super::*; @@ -483,7 +654,8 @@ mod tests { } #[test] - fn test_no_workspace() { + #[parallel] + fn no_workspace() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -500,7 +672,8 @@ mod tests { } #[test] - fn test_empty_workspace() { + #[parallel] + fn empty_workspace() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -515,7 +688,8 @@ mod tests { } #[test] - fn test_add_remove() { + #[parallel] + fn add_remove() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -547,7 +721,8 @@ mod tests { } #[test] - fn test_value() { + #[parallel] + fn value() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -601,7 +776,8 @@ products = ["g"] } #[test] - fn test_new_completeions_and_cache() { + #[parallel] + fn new_completeions_and_cache() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -639,7 +815,8 @@ products = ["g"] } #[test] - fn test_completions_not_synced_for_known_directories() { + #[parallel] + fn completions_not_synced_for_known_directories() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -663,7 +840,8 @@ products = ["g"] } #[test] - fn test_completed_removed() { + #[parallel] + fn completed_removed() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -710,4 +888,153 @@ products = ["g"] } } } + + #[test] + #[parallel] + fn new_submitted_and_cache() { + let mut multi_progress = setup(); + + let temp = TempDir::new().unwrap(); + let n = 8; + + let workflow = setup_completion_directories(&temp, n); + let workflow = Workflow::open_str(temp.path(), &workflow).unwrap(); + + let mut state = State::default(); + let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); + assert!(result.is_ok()); + + assert!(state.submitted.is_empty()); + + state.add_submitted("b", &["dir1".into(), "dir5".into()], "cluster1", 11); + state.add_submitted("b", &["dir3".into(), "dir4".into()], "cluster2", 12); + state.add_submitted("e", &["dir6".into(), "dir7".into()], "cluster2", 13); + + assert!(state.is_submitted("b", &"dir1".into())); + assert!(!state.is_submitted("b", &"dir2".into())); + assert!(state.is_submitted("b", &"dir3".into())); + assert!(state.is_submitted("b", &"dir4".into())); + assert!(state.is_submitted("b", &"dir5".into())); + assert!(!state.is_submitted("b", &"dir6".into())); + assert!(!state.is_submitted("b", &"dir7".into())); + + assert!(!state.is_submitted("e", &"dir1".into())); + assert!(!state.is_submitted("e", &"dir2".into())); + assert!(!state.is_submitted("e", &"dir3".into())); + assert!(!state.is_submitted("e", &"dir4".into())); + assert!(!state.is_submitted("e", &"dir5".into())); + assert!(state.is_submitted("e", &"dir6".into())); + assert!(state.is_submitted("e", &"dir7".into())); + + assert_eq!(state.jobs_submitted_on("cluster1"), vec![11]); + let mut jobs_on_cluster2 = state.jobs_submitted_on("cluster2"); + jobs_on_cluster2.sort(); + assert_eq!(jobs_on_cluster2, vec![12, 13]); + + state + .save_cache(&workflow, &mut multi_progress) + .expect("Cache saved."); + + let cached_state = State::from_cache(&workflow).expect("Read state from cache"); + assert_eq!(state, cached_state); + } + + #[test] + #[parallel] + fn remove_submitted_actions_and_dirs() { + let mut multi_progress = setup(); + + let temp = TempDir::new().unwrap(); + let n = 8; + + let workflow = setup_completion_directories(&temp, n); + let workflow = Workflow::open_str(temp.path(), &workflow).unwrap(); + + let mut state = State::default(); + let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); + assert!(result.is_ok()); + + assert!(state.submitted.is_empty()); + + state.add_submitted("b", &["dir25".into(), "dir27".into()], "cluster1", 18); + state.add_submitted("b", &["dir1".into(), "dir2".into()], "cluster1", 19); + state.add_submitted("f", &["dir3".into(), "dir4".into()], "cluster2", 27); + + assert!(state.is_submitted("b", &"dir1".into())); + assert!(state.is_submitted("b", &"dir2".into())); + assert!(state.is_submitted("b", &"dir25".into())); + assert!(state.is_submitted("b", &"dir27".into())); + + assert!(state.is_submitted("f", &"dir3".into())); + assert!(state.is_submitted("f", &"dir4".into())); + + state + .save_cache(&workflow, &mut multi_progress) + .expect("Cache saved."); + + let mut cached_state = State::from_cache(&workflow).expect("Read state from cache"); + assert_eq!(state, cached_state); + + let result = cached_state.synchronize_workspace(&workflow, 2, &mut multi_progress); + assert!(result.is_ok()); + + assert!(!cached_state.submitted.contains_key("f")); + assert!(!cached_state.is_submitted("f", &"dir3".into())); + assert!(!cached_state.is_submitted("f", &"dir4".into())); + + assert!(cached_state.is_submitted("b", &"dir1".into())); + assert!(cached_state.is_submitted("b", &"dir2".into())); + assert!(!cached_state.is_submitted("b", &"dir25".into())); + assert!(!cached_state.is_submitted("b", &"dir27".into())); + } + + #[test] + #[parallel] + fn remove_inactive() { + let mut multi_progress = setup(); + + let temp = TempDir::new().unwrap(); + let n = 8; + + let workflow = setup_completion_directories(&temp, n); + let workflow = Workflow::open_str(temp.path(), &workflow).unwrap(); + + let mut state = State::default(); + let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); + assert!(result.is_ok()); + + assert!(state.submitted.is_empty()); + + state.add_submitted("b", &["dir1".into(), "dir5".into()], "cluster1", 11); + state.add_submitted("b", &["dir3".into(), "dir4".into()], "cluster2", 12); + state.add_submitted("e", &["dir6".into(), "dir7".into()], "cluster2", 13); + + assert!(state.is_submitted("b", &"dir1".into())); + assert!(!state.is_submitted("b", &"dir2".into())); + assert!(state.is_submitted("b", &"dir3".into())); + assert!(state.is_submitted("b", &"dir4".into())); + assert!(state.is_submitted("b", &"dir5".into())); + assert!(!state.is_submitted("b", &"dir6".into())); + assert!(!state.is_submitted("b", &"dir7".into())); + + assert!(!state.is_submitted("e", &"dir1".into())); + assert!(!state.is_submitted("e", &"dir2".into())); + assert!(!state.is_submitted("e", &"dir3".into())); + assert!(!state.is_submitted("e", &"dir4".into())); + assert!(!state.is_submitted("e", &"dir5".into())); + assert!(state.is_submitted("e", &"dir6".into())); + assert!(state.is_submitted("e", &"dir7".into())); + + state.remove_inactive_submitted("cluster2", &HashSet::from([13])); + assert!(state.is_submitted("b", &"dir1".into())); + assert!(state.is_submitted("b", &"dir5".into())); + assert!(!state.is_submitted("b", &"dir3".into())); + assert!(!state.is_submitted("b", &"dir4".into())); + assert!(state.is_submitted("e", &"dir6".into())); + assert!(state.is_submitted("e", &"dir7".into())); + + state.remove_inactive_submitted("cluster1", &HashSet::from([])); + assert!(!state.is_submitted("b", &"dir1".into())); + assert!(!state.is_submitted("b", &"dir5".into())); + } } diff --git a/src/workflow.rs b/src/workflow.rs index 6ef740a..1918a37 100644 --- a/src/workflow.rs +++ b/src/workflow.rs @@ -30,9 +30,9 @@ pub struct Workflow { #[serde(default)] pub workspace: Workspace, - /// The cluster parameters + /// The submission options #[serde(default)] - pub cluster: HashMap, + pub submit_options: HashMap, /// The actions. #[serde(default)] @@ -54,14 +54,14 @@ pub struct Workspace { pub value_file: Option, } -/// The cluster parameters +/// The submission options /// -/// `ClusterParameters` stores the user-provided cluster specific parameters for a workflow or +/// `SubmitOPtions` stores the user-provided cluster specific submission options for a workflow or /// action. /// #[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] -pub struct ClusterParameters { +pub struct SubmitOptions { /// The account. pub account: Option, @@ -70,7 +70,7 @@ pub struct ClusterParameters { /// Custom options. #[serde(default)] - pub options: Vec, + pub custom: Vec, /// The partition. pub partition: Option, @@ -105,9 +105,9 @@ pub struct Action { #[serde(default)] pub resources: Resources, - /// The cluster parameters + /// The cluster specific submission options. #[serde(default)] - pub cluster: HashMap, + pub submit_options: HashMap, /// The group of jobs to submit. #[serde(default)] @@ -289,6 +289,15 @@ impl Resources { self.total_processes(n_directories) * self.threads_per_process.unwrap_or(1) } + /// Determine the total number of GPUs this action will use. + /// + /// # Arguments + /// `n_directories`: Number of directories in the submission. + /// + pub fn total_gpus(&self, n_directories: usize) -> usize { + self.total_processes(n_directories) * self.gpus_per_process.unwrap_or(0) + } + /// Determine the total walltime this action will use. /// /// # Arguments @@ -401,24 +410,28 @@ impl Workflow { return Err(Error::DuplicateAction(action.name.clone())); } - // Populate action's cluster with the global cluster options. - for (name, parameters) in &self.cluster { - if !action.cluster.contains_key(name) { - action.cluster.insert(name.clone(), parameters.clone()); + // Populate each action's submit_options with the global ones. + for (name, global_options) in &self.submit_options { + if !action.submit_options.contains_key(name) { + action + .submit_options + .insert(name.clone(), global_options.clone()); } else { - let action_parameters = - action.cluster.get_mut(name).expect("Key should be present"); - if action_parameters.account.is_none() { - action_parameters.account = parameters.account.clone(); + let action_options = action + .submit_options + .get_mut(name) + .expect("Key should be present"); + if action_options.account.is_none() { + action_options.account = global_options.account.clone(); } - if action_parameters.setup.is_none() { - action_parameters.setup = parameters.setup.clone(); + if action_options.setup.is_none() { + action_options.setup = global_options.setup.clone(); } - if action_parameters.partition.is_none() { - action_parameters.partition = parameters.partition.clone(); + if action_options.partition.is_none() { + action_options.partition = global_options.partition.clone(); } - if action_parameters.options.is_empty() { - action_parameters.options = parameters.options.clone(); + if action_options.custom.is_empty() { + action_options.custom = global_options.custom.clone(); } } } @@ -505,14 +518,14 @@ fn find_and_open_workflow() -> Result<(PathBuf, File), Error> { mod tests { use assert_fs::prelude::*; use assert_fs::TempDir; - use serial_test::serial; + use serial_test::{parallel, serial}; use std::env; use super::*; #[test] - #[serial(set_current_dir)] - fn test_no_workflow() { + #[serial] + fn no_workflow() { let temp = TempDir::new().unwrap(); env::set_current_dir(temp.path()).unwrap(); @@ -530,8 +543,8 @@ mod tests { } #[test] - #[serial(set_current_dir)] - fn test_parent_search() { + #[serial] + fn parent_search() { let temp = TempDir::new().unwrap(); temp.child("workflow.toml").touch().unwrap(); @@ -547,29 +560,27 @@ mod tests { temp.path().canonicalize().unwrap() ); } else { - assert!( - false, - "Expected to find a workflow file, but got {:?}", - result - ); + panic!("Expected to find a workflow file, but got {:?}", result); } } #[test] - fn test_empty_workflow_file() { + #[parallel] + fn empty_workflow_file() { let temp = TempDir::new().unwrap(); let workflow = ""; - let workflow = Workflow::open_str(temp.path().into(), workflow).unwrap(); + let workflow = Workflow::open_str(temp.path(), workflow).unwrap(); assert_eq!(workflow.root, temp.path().canonicalize().unwrap()); assert_eq!(workflow.workspace.path, PathBuf::from("workspace")); assert!(workflow.workspace.value_file.is_none()); - assert!(workflow.cluster.is_empty()); + assert!(workflow.submit_options.is_empty()); assert!(workflow.action.is_empty()); } #[test] - fn test_workspace() { + #[parallel] + fn workspace() { let temp = TempDir::new().unwrap(); let workflow = r#" [workspace] @@ -583,9 +594,10 @@ value_file = "s" } #[test] - fn test_cluster_parameter_defaults() { + #[parallel] + fn submit_options_defaults() { let temp = TempDir::new().unwrap(); - let workflow = "[cluster.a]"; + let workflow = "[submit_options.a]"; let workflow = Workflow::open_str(temp.path(), workflow).unwrap(); assert_eq!( @@ -593,24 +605,25 @@ value_file = "s" temp.path().canonicalize().unwrap() ); - assert_eq!(workflow.cluster.len(), 1); - assert!(workflow.cluster.contains_key("a")); + assert_eq!(workflow.submit_options.len(), 1); + assert!(workflow.submit_options.contains_key("a")); - let cluster_parameters = workflow.cluster.get("a").unwrap(); - assert_eq!(cluster_parameters.account, None); - assert_eq!(cluster_parameters.setup, None); - assert!(cluster_parameters.options.is_empty()); - assert_eq!(cluster_parameters.partition, None); + let submit_options = workflow.submit_options.get("a").unwrap(); + assert_eq!(submit_options.account, None); + assert_eq!(submit_options.setup, None); + assert!(submit_options.custom.is_empty()); + assert_eq!(submit_options.partition, None); } #[test] - fn test_cluster_parameter_nondefault() { + #[parallel] + fn submit_options_nondefault() { let temp = TempDir::new().unwrap(); let workflow = r#" -[cluster.a] +[submit_options.a] account = "my_account" setup = "module load openmpi" -options = ["--option1", "--option2"] +custom = ["--option1", "--option2"] partition = "gpu" "#; let workflow = Workflow::open_str(temp.path(), workflow).unwrap(); @@ -620,21 +633,22 @@ partition = "gpu" temp.path().canonicalize().unwrap() ); - assert_eq!(workflow.cluster.len(), 1); - assert!(workflow.cluster.contains_key("a")); + assert_eq!(workflow.submit_options.len(), 1); + assert!(workflow.submit_options.contains_key("a")); - let cluster_parameters = workflow.cluster.get("a").unwrap(); - assert_eq!(cluster_parameters.account, Some(String::from("my_account"))); + let submit_options = workflow.submit_options.get("a").unwrap(); + assert_eq!(submit_options.account, Some(String::from("my_account"))); assert_eq!( - cluster_parameters.setup, + submit_options.setup, Some(String::from("module load openmpi")) ); - assert_eq!(cluster_parameters.options, vec!["--option1", "--option2"]); - assert_eq!(cluster_parameters.partition, Some(String::from("gpu"))); + assert_eq!(submit_options.custom, vec!["--option1", "--option2"]); + assert_eq!(submit_options.partition, Some(String::from("gpu"))); } #[test] - fn test_action_defaults() { + #[parallel] + fn action_defaults() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -645,7 +659,7 @@ command = "c" assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); + let action = workflow.action.first().unwrap(); assert_eq!(action.name, "b"); assert_eq!(action.command, "c"); assert!(action.previous_actions.is_empty()); @@ -660,17 +674,18 @@ command = "c" Walltime::PerDirectory(Duration::new(true, 0, 3600, 0).unwrap()) ); - assert!(action.cluster.is_empty()); + assert!(action.submit_options.is_empty()); assert!(action.group.include.is_empty()); assert!(action.group.sort_by.is_empty()); - assert_eq!(action.group.split_by_sort_key, false); + assert!(!action.group.split_by_sort_key); assert_eq!(action.group.maximum_size, None); - assert_eq!(action.group.submit_whole, false); - assert_eq!(action.group.reverse_sort, false); + assert!(!action.group.submit_whole); + assert!(!action.group.reverse_sort); } #[test] - fn test_group_defaults() { + #[parallel] + fn group_defaults() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -682,23 +697,24 @@ command = "c" assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); + let action = workflow.action.first().unwrap(); assert_eq!( action.resources.walltime, Walltime::PerDirectory(Duration::new(true, 0, 3600, 0).unwrap()) ); - assert!(action.cluster.is_empty()); + assert!(action.submit_options.is_empty()); assert!(action.group.include.is_empty()); assert!(action.group.sort_by.is_empty()); - assert_eq!(action.group.split_by_sort_key, false); + assert!(!action.group.split_by_sort_key); assert_eq!(action.group.maximum_size, None); - assert_eq!(action.group.submit_whole, false); - assert_eq!(action.group.reverse_sort, false); + assert!(!action.group.submit_whole); + assert!(!action.group.reverse_sort); } #[test] - fn test_action_duplicate() { + #[parallel] + fn action_duplicate() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -723,7 +739,8 @@ command = "d" } #[test] - fn test_action_launchers() { + #[parallel] + fn action_launchers() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -736,12 +753,13 @@ launchers = ["openmp", "mpi"] assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); + let action = workflow.action.first().unwrap(); assert_eq!(action.launchers, vec!["openmp", "mpi"]); } #[test] - fn test_action_previous_actions() { + #[parallel] + fn action_previous_actions() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -771,7 +789,8 @@ previous_actions = ["b"] } #[test] - fn test_previous_action_error() { + #[parallel] + fn previous_action_error() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -793,7 +812,8 @@ previous_actions = ["a"] } #[test] - fn test_action_resources() { + #[parallel] + fn action_resources() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -810,7 +830,7 @@ walltime.per_submission = "4d, 05:32:11" assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); + let action = workflow.action.first().unwrap(); assert_eq!(action.resources.processes, Processes::PerSubmission(12)); assert_eq!(action.resources.threads_per_process, Some(8)); assert_eq!(action.resources.gpus_per_process, Some(1)); @@ -824,7 +844,8 @@ walltime.per_submission = "4d, 05:32:11" } #[test] - fn test_action_resources_per_directory() { + #[parallel] + fn action_resources_per_directory() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -839,7 +860,7 @@ walltime.per_directory = "00:01" assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); + let action = workflow.action.first().unwrap(); assert_eq!(action.resources.processes, Processes::PerDirectory(1)); assert_eq!( @@ -851,7 +872,8 @@ walltime.per_directory = "00:01" } #[test] - fn test_processes_duplicate() { + #[parallel] + fn processes_duplicate() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -877,7 +899,8 @@ processes.per_directory = 2 } #[test] - fn test_walltime_duplicate() { + #[parallel] + fn walltime_duplicate() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -902,7 +925,8 @@ walltime.per_directory = "01:00" ); } #[test] - fn test_action_products() { + #[parallel] + fn action_products() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -915,12 +939,13 @@ products = ["d", "e"] assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); + let action = workflow.action.first().unwrap(); assert_eq!(action.products, vec!["d".to_string(), "e".to_string()]); } #[test] - fn test_action_group() { + #[parallel] + fn action_group() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -939,7 +964,7 @@ reverse_sort = true assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); + let action = workflow.action.first().unwrap(); assert_eq!( action.group.include, vec![ @@ -971,14 +996,15 @@ reverse_sort = true ] ); assert_eq!(action.group.sort_by, vec![String::from("/sort")]); - assert_eq!(action.group.split_by_sort_key, true); + assert!(action.group.split_by_sort_key); assert_eq!(action.group.maximum_size, Some(10)); assert!(action.group.submit_whole); - assert_eq!(action.group.reverse_sort, true); + assert!(action.group.reverse_sort); } #[test] - fn test_action_cluster_none() { + #[parallel] + fn action_submit_options_none() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] @@ -990,48 +1016,50 @@ command = "c" assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); - assert!(action.cluster.is_empty()); + let action = workflow.action.first().unwrap(); + assert!(action.submit_options.is_empty()); } #[test] - fn test_action_cluster_default() { + #[parallel] + fn action_submit_options_default() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] name = "b" command = "c" -[action.cluster.d] +[action.submit_options.d] "#; let workflow = Workflow::open_str(temp.path(), workflow).unwrap(); assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); - assert!(!action.cluster.is_empty()); - assert!(action.cluster.contains_key("d")); + let action = workflow.action.first().unwrap(); + assert!(!action.submit_options.is_empty()); + assert!(action.submit_options.contains_key("d")); - let cluster = action.cluster.get("d").unwrap(); - assert_eq!(cluster.account, None); - assert_eq!(cluster.setup, None); - assert!(cluster.options.is_empty()); - assert_eq!(cluster.partition, None); + let submit_options = action.submit_options.get("d").unwrap(); + assert_eq!(submit_options.account, None); + assert_eq!(submit_options.setup, None); + assert!(submit_options.custom.is_empty()); + assert_eq!(submit_options.partition, None); } #[test] - fn test_action_cluster_nondefault() { + #[parallel] + fn action_submit_options_nondefault() { let temp = TempDir::new().unwrap(); let workflow = r#" [[action]] name = "b" command = "c" -[action.cluster.d] +[action.submit_options.d] account = "e" setup = "f" -options = ["g", "h"] +custom = ["g", "h"] partition = "i" "#; @@ -1039,25 +1067,26 @@ partition = "i" assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); - assert!(!action.cluster.is_empty()); - assert!(action.cluster.contains_key("d")); + let action = workflow.action.first().unwrap(); + assert!(!action.submit_options.is_empty()); + assert!(action.submit_options.contains_key("d")); - let cluster = action.cluster.get("d").unwrap(); - assert_eq!(cluster.account, Some("e".to_string())); - assert_eq!(cluster.setup, Some("f".to_string())); - assert_eq!(cluster.options, vec!["g", "h"]); - assert_eq!(cluster.partition, Some("i".to_string())); + let submit_options = action.submit_options.get("d").unwrap(); + assert_eq!(submit_options.account, Some("e".to_string())); + assert_eq!(submit_options.setup, Some("f".to_string())); + assert_eq!(submit_options.custom, vec!["g", "h"]); + assert_eq!(submit_options.partition, Some("i".to_string())); } #[test] - fn test_action_cluster_global() { + #[parallel] + fn action_submit_options_global() { let temp = TempDir::new().unwrap(); let workflow = r#" -[cluster.d] +[submit_options.d] account = "e" setup = "f" -options = ["g", "h"] +custom = ["g", "h"] partition = "i" [[action]] @@ -1069,35 +1098,36 @@ command = "c" assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); - assert!(!action.cluster.is_empty()); - assert!(action.cluster.contains_key("d")); + let action = workflow.action.first().unwrap(); + assert!(!action.submit_options.is_empty()); + assert!(action.submit_options.contains_key("d")); - let cluster = action.cluster.get("d").unwrap(); - assert_eq!(cluster.account, Some("e".to_string())); - assert_eq!(cluster.setup, Some("f".to_string())); - assert_eq!(cluster.options, vec!["g", "h"]); - assert_eq!(cluster.partition, Some("i".to_string())); + let submit_options = action.submit_options.get("d").unwrap(); + assert_eq!(submit_options.account, Some("e".to_string())); + assert_eq!(submit_options.setup, Some("f".to_string())); + assert_eq!(submit_options.custom, vec!["g", "h"]); + assert_eq!(submit_options.partition, Some("i".to_string())); } #[test] - fn test_action_cluster_no_override() { + #[parallel] + fn action_submit_options_no_override() { let temp = TempDir::new().unwrap(); let workflow = r#" -[cluster.d] +[submit_options.d] account = "e" setup = "f" -options = ["g", "h"] +custom = ["g", "h"] partition = "i" [[action]] name = "b" command = "c" -[action.cluster.d] +[action.submit_options.d] account = "j" setup = "k" -options = ["l", "m"] +custom = ["l", "m"] partition = "n" "#; @@ -1105,51 +1135,53 @@ partition = "n" assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); - assert!(!action.cluster.is_empty()); - assert!(action.cluster.contains_key("d")); + let action = workflow.action.first().unwrap(); + assert!(!action.submit_options.is_empty()); + assert!(action.submit_options.contains_key("d")); - let cluster = action.cluster.get("d").unwrap(); - assert_eq!(cluster.account, Some("j".to_string())); - assert_eq!(cluster.setup, Some("k".to_string())); - assert_eq!(cluster.options, vec!["l", "m"]); - assert_eq!(cluster.partition, Some("n".to_string())); + let submit_options = action.submit_options.get("d").unwrap(); + assert_eq!(submit_options.account, Some("j".to_string())); + assert_eq!(submit_options.setup, Some("k".to_string())); + assert_eq!(submit_options.custom, vec!["l", "m"]); + assert_eq!(submit_options.partition, Some("n".to_string())); } #[test] - fn test_action_cluster_override() { + #[parallel] + fn action_submit_options_override() { let temp = TempDir::new().unwrap(); let workflow = r#" -[cluster.d] +[submit_options.d] account = "e" setup = "f" -options = ["g", "h"] +custom = ["g", "h"] partition = "i" [[action]] name = "b" command = "c" -[action.cluster.d] +[action.submit_options.d] "#; let workflow = Workflow::open_str(temp.path(), workflow).unwrap(); assert_eq!(workflow.action.len(), 1); - let action = workflow.action.get(0).unwrap(); - assert!(!action.cluster.is_empty()); - assert!(action.cluster.contains_key("d")); + let action = workflow.action.first().unwrap(); + assert!(!action.submit_options.is_empty()); + assert!(action.submit_options.contains_key("d")); - let cluster = action.cluster.get("d").unwrap(); - assert_eq!(cluster.account, Some("e".to_string())); - assert_eq!(cluster.setup, Some("f".to_string())); - assert_eq!(cluster.options, vec!["g", "h"]); - assert_eq!(cluster.partition, Some("i".to_string())); + let submit_options = action.submit_options.get("d").unwrap(); + assert_eq!(submit_options.account, Some("e".to_string())); + assert_eq!(submit_options.setup, Some("f".to_string())); + assert_eq!(submit_options.custom, vec!["g", "h"]); + assert_eq!(submit_options.partition, Some("i".to_string())); } #[test] - fn test_total_processes() { + #[parallel] + fn total_processes() { let r = Resources { processes: Processes::PerSubmission(10), ..Resources::default() @@ -1170,7 +1202,8 @@ command = "c" } #[test] - fn test_total_cpus() { + #[parallel] + fn total_cpus() { let r = Resources { processes: Processes::PerSubmission(10), threads_per_process: Some(2), @@ -1187,15 +1220,40 @@ command = "c" ..Resources::default() }; - assert_eq!(r.total_processes(10), 100); - assert_eq!(r.total_processes(100), 1000); - assert_eq!(r.total_processes(1000), 10000); + assert_eq!(r.total_cpus(10), 100); + assert_eq!(r.total_cpus(100), 1000); + assert_eq!(r.total_cpus(1000), 10000); } #[test] - fn test_total_walltime() { + #[parallel] + fn total_gpus() { let r = Resources { - walltime: Walltime::PerDirectory(Duration::new(true, 1, 1 * 3600, 0).unwrap()), + processes: Processes::PerSubmission(10), + gpus_per_process: Some(2), + ..Resources::default() + }; + + assert_eq!(r.total_gpus(10), 20); + assert_eq!(r.total_gpus(100), 20); + assert_eq!(r.total_gpus(1000), 20); + + let r = Resources { + processes: Processes::PerDirectory(10), + gpus_per_process: None, + ..Resources::default() + }; + + assert_eq!(r.total_gpus(10), 0); + assert_eq!(r.total_gpus(100), 0); + assert_eq!(r.total_gpus(1000), 0); + } + + #[test] + #[parallel] + fn total_walltime() { + let r = Resources { + walltime: Walltime::PerDirectory(Duration::new(true, 1, 3600, 0).unwrap()), ..Resources::default() }; @@ -1213,29 +1271,30 @@ command = "c" ); let r = Resources { - walltime: Walltime::PerSubmission(Duration::new(true, 1, 1 * 3600, 0).unwrap()), + walltime: Walltime::PerSubmission(Duration::new(true, 1, 3600, 0).unwrap()), ..Resources::default() }; assert_eq!( r.total_walltime(2), - Duration::new(true, 1, 1 * 3600, 0).unwrap() + Duration::new(true, 1, 3600, 0).unwrap() ); assert_eq!( r.total_walltime(4), - Duration::new(true, 1, 1 * 3600, 0).unwrap() + Duration::new(true, 1, 3600, 0).unwrap() ); assert_eq!( r.total_walltime(8), - Duration::new(true, 1, 1 * 3600, 0).unwrap() + Duration::new(true, 1, 3600, 0).unwrap() ); } #[test] - fn test_resource_cost() { + #[parallel] + fn resource_cost() { let r = Resources { processes: Processes::PerSubmission(10), - walltime: Walltime::PerDirectory(Duration::new(true, 0, 1 * 3600, 0).unwrap()), + walltime: Walltime::PerDirectory(Duration::new(true, 0, 3600, 0).unwrap()), ..Resources::default() }; @@ -1245,7 +1304,7 @@ command = "c" let r = Resources { processes: Processes::PerSubmission(10), - walltime: Walltime::PerDirectory(Duration::new(true, 0, 1 * 3600, 0).unwrap()), + walltime: Walltime::PerDirectory(Duration::new(true, 0, 3600, 0).unwrap()), threads_per_process: Some(4), ..Resources::default() }; @@ -1256,10 +1315,9 @@ command = "c" let r = Resources { processes: Processes::PerSubmission(10), - walltime: Walltime::PerDirectory(Duration::new(true, 0, 1 * 3600, 0).unwrap()), + walltime: Walltime::PerDirectory(Duration::new(true, 0, 3600, 0).unwrap()), threads_per_process: Some(4), gpus_per_process: Some(2), - ..Resources::default() }; assert_eq!(r.cost(1), ResourceCost::with_values(0.0, 20.0)); diff --git a/src/workspace.rs b/src/workspace.rs index 0c2c2ea..dcc4ff2 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -1,5 +1,5 @@ use indicatif::{ProgressBar, ProgressDrawTarget}; -use log::{debug, trace}; +use log::debug; use serde_json::Value; use std::collections::{HashMap, HashSet}; use std::ffi::OsStr; @@ -8,6 +8,7 @@ use std::path::PathBuf; use std::sync::mpsc::{self, Receiver}; use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; +use std::time::Duration; use crate::workflow::Workflow; use crate::{progress_styles, Error, MultiProgressContainer, MIN_PROGRESS_BAR_SIZE}; @@ -25,6 +26,7 @@ pub fn list_directories( .add(ProgressBar::new_spinner().with_message("Listing workspace")); multi_progress.progress_bars.push(progress.clone()); progress.set_style(progress_styles::counted_spinner()); + progress.enable_steady_tick(Duration::from_millis(progress_styles::STEADY_TICK)); let mut directories = Vec::new(); @@ -123,8 +125,6 @@ pub fn find_completed_directories( let progress = progress.clone(); let thread_name = format!("find-completed-{i}"); - trace!("Spawning thread {thread_name}."); - let handle = thread::Builder::new() .name(thread_name) @@ -266,8 +266,6 @@ pub(crate) fn read_values( let value_file = workflow.workspace.value_file.clone(); let thread_name = format!("read-values-{i}"); - trace!("Spawning thread {thread_name}."); - let handle = thread::Builder::new() .name(thread_name) @@ -344,6 +342,7 @@ mod tests { use assert_fs::prelude::*; use assert_fs::TempDir; use indicatif::{MultiProgress, ProgressDrawTarget}; + use serial_test::parallel; use std::path::PathBuf; use super::*; @@ -363,7 +362,8 @@ mod tests { } #[test] - fn test_list_directories() { + #[parallel] + fn list() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -389,7 +389,8 @@ mod tests { } #[test] - fn test_find_completed_directories() { + #[parallel] + fn find_completed() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); @@ -490,7 +491,8 @@ products = ["3", "4"] } #[test] - fn test_read_values() { + #[parallel] + fn read() { let mut multi_progress = setup(); let temp = TempDir::new().unwrap(); diff --git a/tests/cli.rs b/tests/cli.rs index f30ac2c..dff47ce 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2,6 +2,7 @@ use assert_cmd::Command; use assert_fs::prelude::*; use assert_fs::TempDir; use predicates::prelude::*; +use serial_test::parallel; use std::fs; /// Create a sample workflow and workspace to use with the tests. @@ -59,6 +60,7 @@ fn complete_action( } #[test] +#[parallel] fn requires_subcommand() -> Result<(), Box> { let mut cmd = Command::cargo_bin("row")?; @@ -70,12 +72,13 @@ fn requires_subcommand() -> Result<(), Box> { } #[test] +#[parallel] fn no_workflow_file() -> Result<(), Box> { let mut cmd = Command::cargo_bin("row")?; let temp = TempDir::new()?; - cmd.args(&["show", "status"]) - .args(&["--cluster", "none"]) + cmd.args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()); cmd.assert() .failure() @@ -85,6 +88,7 @@ fn no_workflow_file() -> Result<(), Box> { } #[test] +#[parallel] fn help() -> Result<(), Box> { let mut cmd = Command::cargo_bin("row")?; @@ -97,15 +101,17 @@ fn help() -> Result<(), Box> { } #[test] +#[parallel] fn empty_workflow() -> Result<(), Box> { let mut cmd = Command::cargo_bin("row")?; + cmd.env("ROW_HOME", "/not/a/path"); let temp = TempDir::new()?; temp.child("workflow.toml").touch()?; temp.child("workspace").create_dir_all()?; - cmd.args(&["show", "status"]) - .args(&["--cluster", "none"]) + cmd.args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()); cmd.assert() .success() @@ -115,16 +121,18 @@ fn empty_workflow() -> Result<(), Box> { } #[test] +#[parallel] fn status() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +0 +0 +10 +0")?) @@ -134,18 +142,20 @@ fn status() -> Result<(), Box> { } #[test] +#[parallel] fn status_action_selection() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .arg("-a") .arg("one") .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +0 +0 +10 +0")?) @@ -155,19 +165,21 @@ fn status_action_selection() -> Result<(), Box> { } #[test] +#[parallel] fn status_directories() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .arg("dir1") .arg("dir2") .arg("nodir") .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +0 +0 +2 +0")?) @@ -178,18 +190,20 @@ fn status_directories() -> Result<(), Box> { } #[test] +#[parallel] fn status_directories_stdin() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .arg("-") .current_dir(temp.path()) .write_stdin("dir1\ndir2\nnodir\n") .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +0 +0 +2 +0")?) @@ -200,16 +214,18 @@ fn status_directories_stdin() -> Result<(), Box> { } #[test] +#[parallel] fn scan() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +0 +0 +10 +0")?) @@ -219,11 +235,12 @@ fn scan() -> Result<(), Box> { complete_action("two", &temp, 4)?; Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +0 +0 +10 +0")?) @@ -244,11 +261,12 @@ fn scan() -> Result<(), Box> { assert_eq!(fs::read_dir(completed.path())?.count(), 1); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +8 +0 +2 +0")?) @@ -260,16 +278,18 @@ fn scan() -> Result<(), Box> { } #[test] +#[parallel] fn scan_action() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +0 +0 +10 +0")?) @@ -285,15 +305,17 @@ fn scan_action() -> Result<(), Box> { .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success(); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +8 +0 +2 +0")?) @@ -303,16 +325,18 @@ fn scan_action() -> Result<(), Box> { } #[test] +#[parallel] fn scan_directories() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +0 +0 +10 +0")?) @@ -327,15 +351,17 @@ fn scan_directories() -> Result<(), Box> { .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success(); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +1 +0 +9 +0")?) @@ -345,25 +371,28 @@ fn scan_directories() -> Result<(), Box> { } #[test] +#[parallel] fn submit() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? .arg("submit") - .args(&["--cluster", "none"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success(); Command::cargo_bin("row")? - .args(&["show", "status"]) - .args(&["--cluster", "none"]) + .args(["show", "status"]) + .args(["--cluster", "none"]) .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^one +10 +0 +0 +0")?) @@ -373,15 +402,17 @@ fn submit() -> Result<(), Box> { } #[test] +#[parallel] fn directories_no_action() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "directories"]) - .args(&["--cluster", "none"]) + .args(["show", "directories"]) + .args(["--cluster", "none"]) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .current_dir(temp.path()) .assert() .failure() @@ -391,78 +422,84 @@ fn directories_no_action() -> Result<(), Box> { } #[test] +#[parallel] fn directories() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "directories"]) - .args(&["--cluster", "none"]) + .args(["show", "directories"]) + .args(["--cluster", "none"]) .arg("one") .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() - .stdout(predicate::str::is_match("(?m)^Directory Status")?) - .stdout(predicate::str::is_match("(?m)^dir0 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir1 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir2 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir3 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir4 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir5 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir6 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir7 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir8 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir9 *eligible$")?); + .stdout(predicate::str::is_match("(?m)^Directory Status +Job ID")?) + .stdout(predicate::str::is_match("(?m)^dir0 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir1 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir2 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir3 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir4 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir5 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir6 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir7 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir8 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir9 *eligible *$")?); Ok(()) } #[test] +#[parallel] fn directories_select_directories() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "directories"]) - .args(&["--cluster", "none"]) + .args(["show", "directories"]) + .args(["--cluster", "none"]) .arg("one") .arg("dir3") .arg("dir9") .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() - .stdout(predicate::str::is_match("(?m)^Directory Status")?) - .stdout(predicate::str::is_match("(?m)^dir0 *eligible$")?.not()) - .stdout(predicate::str::is_match("(?m)^dir1 *eligible$")?.not()) - .stdout(predicate::str::is_match("(?m)^dir2 *eligible$")?.not()) - .stdout(predicate::str::is_match("(?m)^dir3 *eligible$")?) - .stdout(predicate::str::is_match("(?m)^dir4 *eligible$")?.not()) - .stdout(predicate::str::is_match("(?m)^dir5 *eligible$")?.not()) - .stdout(predicate::str::is_match("(?m)^dir6 *eligible$")?.not()) - .stdout(predicate::str::is_match("(?m)^dir7 *eligible$")?.not()) - .stdout(predicate::str::is_match("(?m)^dir8 *eligible$")?.not()) - .stdout(predicate::str::is_match("(?m)^dir9 *eligible$")?); + .stdout(predicate::str::is_match("(?m)^Directory Status +Job ID")?) + .stdout(predicate::str::is_match("(?m)^dir0 *eligible *$")?.not()) + .stdout(predicate::str::is_match("(?m)^dir1 *eligible *$")?.not()) + .stdout(predicate::str::is_match("(?m)^dir2 *eligible *$")?.not()) + .stdout(predicate::str::is_match("(?m)^dir3 *eligible *$")?) + .stdout(predicate::str::is_match("(?m)^dir4 *eligible *$")?.not()) + .stdout(predicate::str::is_match("(?m)^dir5 *eligible *$")?.not()) + .stdout(predicate::str::is_match("(?m)^dir6 *eligible *$")?.not()) + .stdout(predicate::str::is_match("(?m)^dir7 *eligible *$")?.not()) + .stdout(predicate::str::is_match("(?m)^dir8 *eligible *$")?.not()) + .stdout(predicate::str::is_match("(?m)^dir9 *eligible *$")?); Ok(()) } #[test] +#[parallel] fn directories_no_header() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "directories"]) - .args(&["--cluster", "none"]) + .args(["show", "directories"]) + .args(["--cluster", "none"]) .arg("one") .arg("--no-header") .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() .stdout(predicate::str::is_match("(?m)^Directory Status")?.not()); @@ -471,26 +508,68 @@ fn directories_no_header() -> Result<(), Box> { } #[test] +#[parallel] fn directories_value() -> Result<(), Box> { let temp = TempDir::new()?; let _ = setup_sample_workflow(&temp, 10); Command::cargo_bin("row")? - .args(&["show", "directories"]) - .args(&["--cluster", "none"]) - .args(&["--value", "/v"]) - .args(&["--value", "/v2"]) + .args(["show", "directories"]) + .args(["--cluster", "none"]) + .args(["--value", "/v"]) + .args(["--value", "/v2"]) .arg("one") .arg("dir3") .arg("dir9") .current_dir(temp.path()) .env_remove("ROW_COLOR") .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") .assert() .success() - .stdout(predicate::str::is_match("(?m)^Directory +Status +/v +/v2")?) + .stdout(predicate::str::is_match( + "(?m)^Directory +Status +Job ID +/v +/v2", + )?) .stdout(predicate::str::is_match("(?m)^dir3 +eligible +3 +1$")?) .stdout(predicate::str::is_match("(?m)^dir9 +eligible +9 +4$")?); Ok(()) } + +#[test] +#[parallel] +fn show_cluster() -> Result<(), Box> { + let temp = TempDir::new()?; + + Command::cargo_bin("row")? + .args(["show", "cluster"]) + .args(["--cluster", "none"]) + .current_dir(temp.path()) + .env_remove("ROW_COLOR") + .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") + .assert() + .success() + .stdout(predicate::str::contains(r#"name = "none""#)); + + Ok(()) +} + +#[test] +#[parallel] +fn show_launchers() -> Result<(), Box> { + let temp = TempDir::new()?; + + Command::cargo_bin("row")? + .args(["show", "launchers"]) + .args(["--cluster", "none"]) + .current_dir(temp.path()) + .env_remove("ROW_COLOR") + .env_remove("CLICOLOR") + .env("ROW_HOME", "/not/a/path") + .assert() + .success() + .stdout(predicate::str::contains(r#"executable = "mpirun""#)); + + Ok(()) +} diff --git a/validate/.gitignore b/validate/.gitignore new file mode 100644 index 0000000..7ba7d82 --- /dev/null +++ b/validate/.gitignore @@ -0,0 +1,3 @@ +/workflow.toml +/*/ +/*.out diff --git a/validate/validate.py b/validate/validate.py new file mode 100644 index 0000000..7462d4e --- /dev/null +++ b/validate/validate.py @@ -0,0 +1,488 @@ +"""Validate that row jobs correctly submit to supported clusters. + +Create a row project directory that validates cluster job submissions. + +To test a built-in cluster, run: +* `python validate.py init` (add `--account=` if needed). +* `row submit` +* Wait for jobs to complete.... +* `cat *.out` +* `cat /output/*.out` + +The submitted jobs check serial, threaded, MPI, MPI+threads, GPU, and +MPI+GPU jobs to ensure that they run sucessfully and are scheduled to the +selected resources. Check `*.out` for any error messages. Then check +`/output/*.out` for `ERROR`, `WARN`, and `PASSED` lines. +`validate.py` prints: `ERROR` when the launched job has a more restrictive +binding than requested; `WARN` when the binding is less restrictive; and +'PASSED' when there are at least enough avaialble resources to execute. + +To test a non-built-in cluster: +* Configure your cluster in `cluster.toml`. +* If the default `srun` launcher is not sufficient, configure your MPI + launcher in `launchers.toml. +* Add a key to the `CLUSTERS` dictionary in `validate.py` that describes + your cluster. +* Then follow the steps above. +""" + +import argparse +import collections +import os +import re +import socket +import subprocess +import textwrap +from pathlib import Path + +# import numpy to ensure that it does not improperly modify the cpuset +# https://stackoverflow.com/questions/15639779/ +import numpy # noqa: F401 + +# Set the number of cpus and gpus per node in the *default* partitions that row selects. +# Testing non-default partitions is beyond the scope of this script. Set to 0 to prevent +# CPU and/or GPU jobs from executing. +Cluster = collections.namedtuple('Cluster', ['cpus_per_node', 'gpus_per_node', 'gpu_arch']) +CLUSTERS = { + 'greatlakes': Cluster(cpus_per_node=36, gpus_per_node=2, gpu_arch='nvidia'), + 'anvil': Cluster(cpus_per_node=128, gpus_per_node=0, gpu_arch='nvidia'), + 'delta': Cluster(cpus_per_node=128, gpus_per_node=4, gpu_arch='nvidia'), +} + +N_THREADS = 4 +N_GPUS = 2 +N_PROCESSES = 4 +N_NODES = 2 + + +def get_cluster_name(): + """Get the current cluster name.""" + result = subprocess.run( + ['row', 'show', 'cluster', '--name'], capture_output=True, check=True, text=True + ) + return result.stdout.strip() + + +def get_nvidia_gpus(): + """Get the assigned NVIDIA GPUs.""" + result = subprocess.run( + ['nvidia-smi', '--list-gpus'], capture_output=True, check=True, text=True + ) + + gpus = [] + migs = [] + pattern = re.compile(r'.*\(UUID: (GPU|MIG)-(.*)\)') + + for line in result.stdout.splitlines(): + match = pattern.match(line) + if not match: + message = f'Unexpected output from nvidia_smi: {line}.' + raise RuntimeError(message) + + if match.group(1) == 'GPU': + gpus.append(match.group(2)) + elif match.group(1) == 'MIG': + migs.append(match.group(2)) + else: + message = 'Unexpected match {match.group(1)}.' + raise RuntimeError(message) + + if len(migs): + return migs + + return gpus + + +def init(account, setup): + """Initialize the project.""" + cluster_name = get_cluster_name() + if cluster_name not in CLUSTERS: + message = f'Unsupported cluster {cluster_name}.' + raise RuntimeError(message) + cluster = CLUSTERS.get(cluster_name) + + # Create the workspace + workspace = Path(cluster_name) + workspace.mkdir(exist_ok=True) + output = workspace / Path('output') + output.mkdir(exist_ok=True) + + # Create workflow.toml + with open(file='workflow.toml', mode='w', encoding='utf-8') as workflow: + workflow.write( + textwrap.dedent(f"""\ + [workspace] + path = "{cluster_name}" + + [submit_options.{cluster_name}] + """) + ) + + if account is not None: + workflow.write( + textwrap.dedent(f"""\ + account = "{account}" + """) + ) + + if setup is not None: + workflow.write( + textwrap.dedent(f"""\ + setup = "{setup}" + """) + ) + + if cluster.cpus_per_node >= 1: + workflow.write( + textwrap.dedent(""" + [[action]] + name = "serial" + command = "python validate.py execute serial {directory}" + products = ["serial.out"] + [action.resources] + processes.per_submission = 1 + walltime.per_submission = "00:05:00" + """) + ) + + if cluster.cpus_per_node >= N_THREADS: + workflow.write( + textwrap.dedent(f""" + [[action]] + name = "threads" + command = "python validate.py execute threads {{directory}}" + products = ["threads.out"] + [action.resources] + processes.per_submission = 1 + threads_per_process = {N_THREADS} + walltime.per_submission = "00:05:00" + """) + ) + + if cluster.cpus_per_node >= N_PROCESSES: + workflow.write( + textwrap.dedent(f""" + [[action]] + name = "mpi_subnode" + command = "python validate.py execute mpi_subnode {{directory}}" + products = ["mpi_subnode.out"] + launchers = ["mpi"] + [action.resources] + processes.per_submission = {N_PROCESSES} + walltime.per_submission = "00:05:00" + """) + ) + + if cluster.cpus_per_node >= N_PROCESSES * N_THREADS: + workflow.write( + textwrap.dedent(f""" + [[action]] + name = "mpi_threads_subnode" + command = "python validate.py execute mpi_threads_subnode {{directory}}" + products = ["mpi_threads_subnode.out"] + launchers = ["mpi"] + [action.resources] + processes.per_submission = {N_PROCESSES} + threads_per_process = {N_THREADS} + walltime.per_submission = "00:05:00" + """) + ) + + if cluster.cpus_per_node >= 1: + workflow.write( + textwrap.dedent(f""" + [[action]] + name = "mpi_multinode" + command = "python validate.py execute mpi_multinode {{directory}}" + products = ["mpi_multinode.out"] + launchers = ["mpi"] + [action.resources] + processes.per_submission = {N_NODES * cluster.cpus_per_node} + walltime.per_submission = "00:05:00" + """) + ) + + if cluster.cpus_per_node >= 1 and (cluster.cpus_per_node % N_THREADS) == 0: + workflow.write( + textwrap.dedent(f""" + [[action]] + name = "mpi_threads_multinode" + command = "python validate.py execute mpi_threads_multinode {{directory}}" + products = ["mpi_threads_multinode.out"] + launchers = ["mpi"] + [action.resources] + processes.per_submission = {N_NODES * cluster.cpus_per_node // N_THREADS} + threads_per_process = {N_THREADS} + walltime.per_submission = "00:05:00" + """) + ) + + if cluster.gpus_per_node >= 1 and cluster.gpu_arch == 'nvidia': + workflow.write( + textwrap.dedent(""" + [[action]] + name = "nvidia_gpu" + command = "python validate.py execute nvidia_gpu {directory}" + products = ["nvidia_gpu.out"] + [action.resources] + processes.per_submission = 1 + gpus_per_process = 1 + walltime.per_submission = "00:05:00" + """) + ) + + if cluster.gpus_per_node >= N_GPUS and cluster.gpu_arch == 'nvidia': + workflow.write( + textwrap.dedent(f""" + [[action]] + name = "nvidia_gpus" + command = "python validate.py execute nvidia_gpus {{directory}}" + products = ["nvidia_gpus.out"] + [action.resources] + processes.per_submission = 1 + gpus_per_process = {N_GPUS} + walltime.per_submission = "00:05:00" + """) + ) + + if cluster.gpus_per_node >= 1 and cluster.gpu_arch == 'nvidia': + workflow.write( + textwrap.dedent(f""" + [[action]] + name = "mpi_nvidia_gpus" + command = "python validate.py execute mpi_nvidia_gpus {{directory}}" + products = ["mpi_nvidia_gpus.out"] + launchers = ["mpi"] + [action.resources] + processes.per_submission = {N_PROCESSES} + gpus_per_process = 1 + walltime.per_submission = "00:05:00" + """) + ) + + +def serial(directory): + """Validate serial jobs.""" + action_cluster = os.environ['ACTION_CLUSTER'] + + output_path = Path(action_cluster) / Path(directory) / Path('serial.out') + with output_path.open(mode='w', encoding='utf-8') as output: + row_cluster = get_cluster_name() + if action_cluster != row_cluster: + print( + 'ERROR: `row cluster --name` does not match at submission ' + f'({action_cluster}) and execution ({row_cluster})', + file=output, + ) + + cpuset = os.sched_getaffinity(0) + if len(cpuset) > 1: + print( + f'WARN: Allowed to run on more cpus than requested: {cpuset}.', + file=output, + ) + elif len(cpuset) == 1: + print(f'PASSED: {cpuset}', file=output) + else: + print('ERROR: unknown.', file=output) + + +def threads(directory): + """Validate threaded jobs.""" + action_cluster = os.environ['ACTION_CLUSTER'] + + output_path = Path(action_cluster) / Path(directory) / Path('threads.out') + with output_path.open(mode='w', encoding='utf-8') as output: + cpuset = os.sched_getaffinity(0) + if len(cpuset) > N_THREADS: + print( + f'WARN: Allowed to run on more cpus than requested: {cpuset}.', + file=output, + ) + + if len(cpuset) < N_THREADS: + print(f'ERROR: Not allowed to run on requested cpus: {cpuset}.', file=output) + elif len(cpuset) == N_THREADS: + print(f'PASSED: {cpuset}', file=output) + + +def check_mpi(directory, n_processes, n_threads, n_hosts, name, n_gpus=0, gpu_arch='nvidia'): + """Validate that MPI jobs run on less than a whole node. + + Ensure that each process has n_threads threads. + """ + from mpi4py import MPI + + action_cluster = os.environ['ACTION_CLUSTER'] + + comm = MPI.COMM_WORLD + + if comm.Get_size() != n_processes: + message = f'ERROR: incorrect number of processes {comm.Get_size()}.' + raise RuntimeError(message) + + cpusets = comm.gather(os.sched_getaffinity(0), root=0) + hostnames = comm.gather(socket.gethostname(), root=0) + gpus = [] + if n_gpus > 0 and gpu_arch == 'nvidia': + gpus = comm.gather(get_nvidia_gpus(), root=0) + + if comm.Get_rank() == 0: + cpuset_sizes = [len(s) for s in cpusets] + gpu_sizes = [len(g) for g in gpus] + + output_path = Path(action_cluster) / Path(directory) / Path(name + '.out') + with output_path.open(mode='w', encoding='utf-8') as output: + if len(set(hostnames)) > n_hosts: + print( + f'WARN: Executing on more than {n_hosts} host(s): {set(hostnames)}.', + file=output, + ) + + if len(set(cpuset_sizes)) != 1: + print(f'WARN: cpusets have different sizes: {cpusets}.', file=output) + + if max(cpuset_sizes) > n_threads: + print( + f'WARN: Allowed to run on more cpus than requested: {cpusets}.', + file=output, + ) + + if n_gpus > 0: + if len(set(gpu_sizes)) != 1: + print(f'WARN: gpus have different sizes: {gpus}.', file=output) + + if max(gpu_sizes) > n_gpus: + print( + f'WARN: Allowed to run on more GPUs than requested: {gpus}.', + file=output, + ) + + if min(cpuset_sizes) < n_threads: + print( + f'ERROR: Not allowed to run on requested cpus: {cpusets}.', + file=output, + ) + elif len(set(hostnames)) < n_hosts: + print( + f'ERROR: Executing on fewer than {n_hosts} hosts: {set(hostnames)}.', + file=output, + ) + elif n_gpus > 0 and min(gpu_sizes) < n_gpus: + print( + f'ERROR: Not allowed to run on requested GPUs: {gpus}.', + file=output, + ) + else: + print(f'PASSED: {set(hostnames)} {cpusets} {gpus}', file=output) + + +def mpi_subnode(directory): + """Check that MPI allocates processes correctly on one node.""" + check_mpi(directory, n_processes=N_PROCESSES, n_threads=1, n_hosts=1, name='mpi_subnode') + + +def mpi_threads_subnode(directory): + """Check that MPI allocates processes and threads correctly on one node.""" + check_mpi( + directory, + n_processes=N_PROCESSES, + n_threads=N_THREADS, + n_hosts=1, + name='mpi_threads_subnode', + ) + + +def mpi_nvidia_gpus(directory): + """Check that MPI allocates GPUs correctly.""" + check_mpi( + directory, + n_processes=N_PROCESSES, + n_threads=1, + n_hosts=1, + name='mpi_nvidia_gpus', + n_gpus=1, + gpu_arch='nvidia', + ) + + +def mpi_multinode(directory): + """Check that MPI allocates processes correctly on multiple nodes.""" + cluster_name = get_cluster_name() + cluster = CLUSTERS.get(cluster_name) + + check_mpi( + directory, + n_processes=cluster.cpus_per_node * N_NODES, + n_threads=1, + n_hosts=N_NODES, + name='mpi_multinode', + ) + + +def mpi_threads_multinode(directory): + """Check that MPI allocates processes and threads correctly on multiple nodes.""" + cluster_name = get_cluster_name() + cluster = CLUSTERS.get(cluster_name) + + check_mpi( + directory, + n_processes=cluster.cpus_per_node * N_NODES // N_THREADS, + n_threads=N_THREADS, + n_hosts=N_NODES, + name='mpi_threads_multinode', + ) + + +def check_nvidia_gpu(directory, n_gpus, name): + """Validate threaded GPU jobs.""" + action_cluster = os.environ['ACTION_CLUSTER'] + + output_path = Path(action_cluster) / Path(directory) / Path(name + '.out') + with output_path.open(mode='w', encoding='utf-8') as output: + gpus = get_nvidia_gpus() + if len(gpus) > n_gpus: + print( + f'WARN: Allowed to run on more GPUs than requested: {gpus}.', + file=output, + ) + + if len(gpus) < n_gpus: + print(f'ERROR: Not allowed to run on requested GPUs: {gpus}.', file=output) + elif len(gpus) == n_gpus: + print(f'PASSED: {gpus}', file=output) + + +def nvidia_gpu(directory): + """Validate single GPU jobs.""" + check_nvidia_gpu(directory, n_gpus=1, name='nvidia_gpu') + + +def nvidia_gpus(directory): + """Validate multi-GPU jobs.""" + check_nvidia_gpu(directory, n_gpus=N_GPUS, name='nvidia_gpus') + + +if __name__ == '__main__': + # Parse the command line arguments: + # * `python execute [DIRECTORIES]` + # * `python init --account ` + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest='subparser_name', required=True) + + execute_parser = subparsers.add_parser('execute') + execute_parser.add_argument('action') + execute_parser.add_argument('directories', nargs='+') + + init_parser = subparsers.add_parser('init') + init_parser.add_argument('--account') + init_parser.add_argument('--setup') + args = parser.parse_args() + + if args.subparser_name == 'init': + init(account=args.account, setup=args.setup) + elif args.subparser_name == 'execute': + globals()[args.action](*args.directories) + else: + message = f'Unknown subcommand {args.subparser_name}' + raise ValueError(message)