diff --git a/docs/benchpark-setup.rst b/docs/benchpark-setup.rst index e22555272..00db52f08 100644 --- a/docs/benchpark-setup.rst +++ b/docs/benchpark-setup.rst @@ -1,46 +1,79 @@ -.. Copyright 2023 Lawrence Livermore National Security, LLC and other +.. + Copyright 2023 Lawrence Livermore National Security, LLC and other Benchpark Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: Apache-2.0 -================== -Set up a Workspace -================== +#################### + Set up a Workspace +#################### -To setup an experiment workspace you must first initialize the system you will be running on. -Next, initialize the experiment you plan to run with the appropriate programming model. -Finally, set up the workspace in a directory for your experiment.:: +To setup an experiment workspace you must first initialize the system you will be +running on. Next, initialize the experiment you plan to run with the appropriate +programming model. Finally, set up the workspace in a directory for your experiment.: - benchpark system init --dest= compiler= - benchpark experiment init --dest= +/~ = --system= - benchpark setup +.. code:: + + benchpark system init --dest= compiler= + benchpark experiment init --dest= +/~ = --system= + benchpark setup where: -- ````: amg2023 | saxpy | etc. (specified choices in :doc:`benchmark-list`) -- ````: Cts | Tioga | etc. (specified systems in :doc:`system-list`) +- ````: amg2023 | saxpy | etc. (specified choices in :doc:`benchmark-list`) +- ````: Cts | Tioga | etc. (specified systems in :doc:`system-list`) + +This command will assemble a Ramble workspace per experiment with a configuration for +the specified benchmark and system with the following directory structure: + +.. code:: -This command will assemble a Ramble workspace per experiment -with a configuration for the specified benchmark and system -with the following directory structure:: + experiments_root/ + ramble/ + spack/ + / + / + workspace/ + configs/ + (everything from system_def_dir) + (everything from experiment_def_dir>) - experiments_root/ - ramble/ - spack/ - / - / - workspace/ - configs/ - (everything from system_def_dir) - (everything from experiment_def_dir>) +``benchpark setup`` will output instructions to follow: -``benchpark setup`` will output instructions to follow:: +.. code:: . /setup.sh -The ``setup.sh`` script calls the Spack and Ramble setup scripts. It optionally accepts +The ``setup.sh`` script calls the Spack and Ramble setup scripts. It optionally accepts parameters to ``ramble workspace setup`` as `documented in Ramble `_, including ``--dry-run`` and ``--phases make_experiments``. Now you are ready to compile your experiments as described in :doc:`build-experiment`. + +************************************* + Built-in System/Experiment Variants +************************************* + +There are benchpark system and experiment variants that you can change, without needing +to define them in your ``system.py`` and ``experiment.py``. + +For Systems +=========== + + - ``timeout`` - Job timeout limit in minutes. + +For Experiments +=============== + + - ``package_manager`` - Specify this variant to use a ramble package manager other + than ``spack``. See :doc:`run-binary` to see an example. + + - ``append_path`` - Append to environment PATH during experiment execution. + + - ``prepend_path`` - Prepend to environment PATH during experiment execution. + + - ``n_repeats`` - number of times your experiment will be repeated (think of + trials). ``n_repeats=5`` will repeat your experiment 5 times, resulting in 5 + trials. These will be separate job submission scripts, so separate resource + allocations. For combining these into the same allocation, see TBD. diff --git a/lib/benchpark/experiment.py b/lib/benchpark/experiment.py index 57013f34c..56c07e924 100644 --- a/lib/benchpark/experiment.py +++ b/lib/benchpark/experiment.py @@ -218,6 +218,12 @@ class Experiment(ExperimentSystemBase, ExecMode, Affinity, Hwloc): description="Prepend to environment PATH during experiment execution", ) + variant( + "n_repeats", + default="0", + description="Number of experiment repetitions", + ) + def __init__(self, spec): self.spec: "benchpark.spec.ConcreteExperimentSpec" = spec # Device type must be set before super with absence of mpionly experiment type @@ -349,6 +355,7 @@ def compute_config_section(self): } # default configs for all experiments default_config = { + "n_repeats": self.spec.variants["n_repeats"][0], "deprecated": True, "benchpark_experiment_command": "benchpark " + " ".join(sys.argv[1:]), "system": system_dict, diff --git a/lib/benchpark/test/caliper.py b/lib/benchpark/test/caliper.py index 2a8c0a015..16d17b252 100644 --- a/lib/benchpark/test/caliper.py +++ b/lib/benchpark/test/caliper.py @@ -35,6 +35,7 @@ def test_experiment_compute_variables_section_caliper(monkeypatch): "experiment_name": "{experiment_name}", "n_nodes": "{n_nodes}", "n_ranks": "{n_ranks}", + "n_repeats": "0", "n_threads_per_proc": "{n_threads_per_proc}", "benchpark_spec": ["~cuda+mpi~openmp~rocm"], "append_path": "'", @@ -109,6 +110,7 @@ def test_caliper_modifier(monkeypatch): "experiment_name": "{experiment_name}", "n_nodes": "{n_nodes}", "n_ranks": "{n_ranks}", + "n_repeats": "0", "n_threads_per_proc": "{n_threads_per_proc}", "n_resources": "{n_resources}", "process_problem_size": "{process_problem_size}", diff --git a/lib/benchpark/test/experiment.py b/lib/benchpark/test/experiment.py index 0d1f3b7d6..98306cca0 100644 --- a/lib/benchpark/test/experiment.py +++ b/lib/benchpark/test/experiment.py @@ -121,6 +121,7 @@ def test_default_config_section(): "benchpark_experiment_command": "benchpark " + " ".join(sys.argv[1:]), # Not applicable here "deprecated": True, + "n_repeats": "0", "spack_flags": { "install": "--add --keep-stage", "concretize": "-U -f",