Expose n_repeats as experiment variant

docs/benchpark-setup.rst

@@ -1,46 +1,79 @@
-.. Copyright 2023 Lawrence Livermore National Security, LLC and other
+..
+   Copyright 2023 Lawrence Livermore National Security, LLC and other
    Benchpark Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: Apache-2.0
 
-==================
-Set up a Workspace
-==================
+####################
+ Set up a Workspace
+####################
 
-To setup an experiment workspace you must first initialize the system you will be running on.
-Next, initialize the experiment you plan to run with the appropriate programming model.
-Finally, set up the workspace in a directory for your experiment.::
+To setup an experiment workspace you must first initialize the system you will be
+running on. Next, initialize the experiment you plan to run with the appropriate
+programming model. Finally, set up the workspace in a directory for your experiment.:
 
-    benchpark system init --dest=</output/path/to/system_def_dir> <SystemName> compiler=<Compiler>
-    benchpark experiment init --dest=</output/path/to/experiment_def_dir> <Benchmark> +/~<Boolean Variant> <String Variant>=<value> --system=</output/path/to/system_def_dir>
-    benchpark setup </output/path/to/experiment_def> </output/path/to/workspace>
+.. code::
+
+   benchpark system init --dest=</output/path/to/system_def_dir> <SystemName> compiler=<Compiler>
+   benchpark experiment init --dest=</output/path/to/experiment_def_dir> <Benchmark> +/~<Boolean Variant> <String Variant>=<value> --system=</output/path/to/system_def_dir>
+   benchpark setup </output/path/to/experiment_def> </output/path/to/workspace>
 
 where:
 
-- ``<Benchmark>``: amg2023 | saxpy | etc. (specified choices in :doc:`benchmark-list`)
-- ``<System>``: Cts | Tioga | etc. (specified systems in :doc:`system-list`)
+-  ``<Benchmark>``: amg2023 | saxpy | etc. (specified choices in :doc:`benchmark-list`)
+-  ``<System>``: Cts | Tioga | etc. (specified systems in :doc:`system-list`)
+
+This command will assemble a Ramble workspace per experiment with a configuration for
+the specified benchmark and system with the following directory structure:
+
+.. code::
 
-This command will assemble a Ramble workspace per experiment
-with a configuration for the specified benchmark and system
-with the following directory structure::
+   experiments_root/
+       ramble/
+       spack/
+       <Benchmark/ProgrammingModel>/
+           <System>/
+               workspace/
+                   configs/
+                       (everything from system_def_dir)
+                       (everything from experiment_def_dir>)
 
-    experiments_root/
-        ramble/
-        spack/
-        <Benchmark/ProgrammingModel>/
-            <System>/
-                workspace/
-                    configs/
-                        (everything from system_def_dir)
-                        (everything from experiment_def_dir>)
+``benchpark setup`` will output instructions to follow:
 
-``benchpark setup`` will output instructions to follow::
+.. code::
 
    . <experiments_root>/setup.sh
 
-The ``setup.sh`` script calls the Spack and Ramble setup scripts.  It optionally accepts
+The ``setup.sh`` script calls the Spack and Ramble setup scripts. It optionally accepts
 parameters to ``ramble workspace setup`` as `documented in Ramble
 <https://ramble.readthedocs.io/en/latest/getting_started.html#setting-up-a-workspace>`_,
 including ``--dry-run`` and ``--phases make_experiments``.
 
 Now you are ready to compile your experiments as described in :doc:`build-experiment`.
+
+*************************************
+ Built-in System/Experiment Variants
+*************************************
+
+There are benchpark system and experiment variants that you can change, without needing
+to define them in your ``system.py`` and ``experiment.py``.
+
+For Systems
+===========
+
+   -  ``timeout`` - Job timeout limit in minutes.
+
+For Experiments
+===============
+
+   -  ``package_manager`` - Specify this variant to use a ramble package manager other
+      than ``spack``. See :doc:`run-binary` to see an example.
+
+   -  ``append_path`` - Append to environment PATH during experiment execution.
+
+   -  ``prepend_path`` - Prepend to environment PATH during experiment execution.
+
+   -  ``n_repeats`` - number of times your experiment will be repeated (think of
+      trials). ``n_repeats=5`` will repeat your experiment 5 times, resulting in 5
+      trials. These will be separate job submission scripts, so separate resource
+      allocations. For combining these into the same allocation, see TBD.

lib/benchpark/experiment.py

@@ -218,6 +218,12 @@ class Experiment(ExperimentSystemBase, ExecMode, Affinity, Hwloc):
         description="Prepend to environment PATH during experiment execution",
     )
 
+    variant(
+        "n_repeats",
+        default="0",
+        description="Number of experiment repetitions",
+    )
+
     def __init__(self, spec):
         self.spec: "benchpark.spec.ConcreteExperimentSpec" = spec
         # Device type must be set before super with absence of mpionly experiment type
@@ -349,6 +355,7 @@ def compute_config_section(self):
             }
         # default configs for all experiments
         default_config = {
+            "n_repeats": self.spec.variants["n_repeats"][0],
             "deprecated": True,
             "benchpark_experiment_command": "benchpark " + " ".join(sys.argv[1:]),
             "system": system_dict,

lib/benchpark/test/caliper.py

@@ -35,6 +35,7 @@ def test_experiment_compute_variables_section_caliper(monkeypatch):
             "experiment_name": "{experiment_name}",
             "n_nodes": "{n_nodes}",
             "n_ranks": "{n_ranks}",
+            "n_repeats": "0",
             "n_threads_per_proc": "{n_threads_per_proc}",
             "benchpark_spec": ["~cuda+mpi~openmp~rocm"],
             "append_path": "'",
@@ -109,6 +110,7 @@ def test_caliper_modifier(monkeypatch):
         "experiment_name": "{experiment_name}",
         "n_nodes": "{n_nodes}",
         "n_ranks": "{n_ranks}",
+        "n_repeats": "0",
         "n_threads_per_proc": "{n_threads_per_proc}",
         "n_resources": "{n_resources}",
         "process_problem_size": "{process_problem_size}",

lib/benchpark/test/experiment.py

@@ -121,6 +121,7 @@ def test_default_config_section():
         "benchpark_experiment_command": "benchpark "
         + " ".join(sys.argv[1:]),  # Not applicable here
         "deprecated": True,
+        "n_repeats": "0",
         "spack_flags": {
             "install": "--add --keep-stage",
             "concretize": "-U -f",