simultaneous per testset (#756)

* Made minor changes

* Will not use simultaneous from test_set if it exists

* typo

* It should now use the test_set.simultaneous key if it exists

* accidently removed a blank line

* Better documentation

* style fix

* I can't spell

* removed whitespaces and added comments

* Added a unit test for test_set.simultaneous; untested

* Verified the functionality of the unit test

---------

Co-authored-by: Paul Ferrell <51765748+Paul-Ferrell@users.noreply.github.com>

Author: smehta99

lib/pavilion/series/series.py

@@ -270,6 +270,13 @@ def _create_test_sets(self, iteration=0):
         # create all TestSet objects
         universal_modes = self.config['modes']
         for set_name, set_info in self.config['test_sets'].items():
+            # Checks if the simultaneous key is set inside the test_set, if so, simultaneous limit
+            # set in the test_set will override the simultaneous key at the full series level
+            if set_info['simultaneous']:
+                _simultaneous = set_info['simultaneous']
+            else:
+                _simultaneous = self.simultaneous
+
             set_obj = TestSet(
                 pav_cfg=self.pav_cfg,
                 name=set_name,
@@ -283,7 +290,7 @@ def _create_test_sets(self, iteration=0):
                 parents_must_pass=set_info['depends_pass'],
                 overrides=self.config.get('overrides', []),
                 status=self.status,
-                simultaneous=self.simultaneous,
+                simultaneous= _simultaneous,
                 outfile=self.outfile,
                 verbosity=self.verbosity,
                 ignore_errors=self.ignore_errors,
@@ -510,9 +517,10 @@ def _run_set(self, test_set: TestSet, build_only: bool, rebuild: bool, local_bui
                                          "in series {}."
                                          .format(ktests, len(started_tests),
                                                  test_set.name, self.sid))
-
+            # If simultaneous is set in the test_set, use that.
+            _simultaneous = test_set.simultaneous if test_set.simultaneous else self.simultaneous
             # Wait for jobs until enough have finished to start a new batch.
-            while tests_running + self.batch_size > self.simultaneous:
+            while tests_running + self.batch_size > _simultaneous:
                 tests_running -= test_set.wait()
 
 
@@ -583,6 +591,7 @@ def add_test_set_config(
             self, name, test_names: List[str], modes: List[str] = None,
             only_if: Dict[str, List[str]] = None,
             not_if: Dict[str, List[str]] = None,
+            simultaneous: int = None,
             save: bool = True,
             _depends_on: List[str] = None, _depends_pass: bool = False):
         """Manually add a test set to this series. The set will be added to the
@@ -594,6 +603,7 @@ def add_test_set_config(
         :param modes: A List of modes to add.
         :param only_if: Only if conditions
         :param not_if:  Not if conditions
+        :param simultaneous: Number of tests within the test set to run simultaneously.
         :param save: Save the series config after adding the test set. Setting this
             to false is useful if you want to add multiple configs before saving.
         :param _depends_on: A list of test names that this test depends on. For
@@ -613,6 +623,7 @@ def add_test_set_config(
             'modes': modes or [],
             'only_if': only_if or {},
             'not_if': not_if or {},
+            'simultaneous': simultaneous,
         }
 
         if save:

lib/pavilion/series_config/file_format.py

@@ -18,6 +18,7 @@ class SeriesConfigLoader(yc.YamlConfigLoader):
                     yc.BoolElem('depends_pass', default=False),
                     yc.ListElem('depends_on', sub_elem=yc.StrElem()),
                     yc.ListElem('modes', sub_elem=yc.StrElem()),
+                    yc.IntElem('simultaneous', default=None),
                     CondCategoryElem(
                         'only_if', sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
                         key_case=EnvCatElem.KC_MIXED
@@ -62,7 +63,9 @@ class SeriesConfigLoader(yc.YamlConfigLoader):
         ),
         yc.IntElem(
             'simultaneous', default=0,
-            help_text="The maximum number of tests to run simultaneously."
+            help_text="The maximum number of tests to run simultaneously. This can be"
+                      " set at the full series level and/or at each test_set level. If"
+                      " set at both levels then test_set will take precedence."
         ),
         yc.BoolElem(
             'ordered', default=False,

test/tests/series_tests.py

@@ -111,6 +111,34 @@ def test_series_simultaneous(self):
                 self.assertLessEqual(last_ended, started)
             last_ended = ended
 
+    def test_series_test_set_simultaneous(self):
+        """Test to see if simultaneous in the test_set overrides the simultaneous at full series"""
+        series_sec_cfg = OrderedDict()
+        series_sec_cfg['set1'] = {
+                'tests': ['echo_test.b', 'echo_test.b', 'echo_test.b'],
+                'simultaneous': 1
+                }
+
+        series_cfg = series_config.make_config({
+                'test_sets': series_sec_cfg,
+                'modes':        ['smode2'],
+                'simultaneous': '3',
+                'ignore_errors': False,
+            })
+
+        test_series_obj = series.TestSeries(self.pav_cfg, series_cfg=series_cfg)
+        test_series_obj.run()
+        test_series_obj.wait(timeout=10)
+
+        last_ended = None
+        for test_id in sorted(test_series_obj.tests):
+            test_obj = test_series_obj.tests[test_id]
+            started = test_obj.results['started']
+            ended = test_obj.results['finished']
+            if last_ended is not None:
+                self.assertLessEqual(last_ended, started)
+            last_ended = ended
+
     def test_series_modes(self):
         """Test if modes and host are applied correctly."""