diff --git a/projects/jump_ci/testing/config.yaml b/projects/jump_ci/testing/config.yaml index afeafb2665..cf2aad03da 100644 --- a/projects/jump_ci/testing/config.yaml +++ b/projects/jump_ci/testing/config.yaml @@ -33,3 +33,6 @@ exec_list: prepare_ci: null test_ci: null post_cleanup_ci: null +multi_run: + args: [] + stop_on_error: false diff --git a/projects/jump_ci/testing/test.py b/projects/jump_ci/testing/test.py index 7fda79fefe..bd5f25df0e 100755 --- a/projects/jump_ci/testing/test.py +++ b/projects/jump_ci/testing/test.py @@ -26,6 +26,7 @@ def rewrite_variables_overrides(variable_overrides_dict): logging.info(f"New args to execute on the jump host: {new_args_str}") for idx, value in enumerate(new_args): new_variable_overrides[f"PR_POSITIONAL_ARG_{idx+1}"] = value + next_pr_positional_arg_count = idx + 2 for k, v in variable_overrides_dict.items(): if k.startswith("PR_POSITIONAL_ARG"): continue @@ -44,7 +45,7 @@ def rewrite_variables_overrides(variable_overrides_dict): new_variable_overrides[k] = v logging.info(f"Passing '{k}: {v}' to the new variables overrides") - return new_variable_overrides + return new_variable_overrides, next_pr_positional_arg_count def jump_ci(command): @@ -72,8 +73,6 @@ def do_jump_ci(cluster=None, project=None, test_args=None): TOPSAIL_JUMP_CI="true", TOPSAIL_JUMP_CI_INSIDE_JUMP_HOST="true", ) - if step_dir := os.environ.get("TOPSAIL_OPENSHIFT_CI_STEP_DIR"): - extra_env["TOPSAIL_OPENSHIFT_CI_STEP_DIR"] = f"{step_dir}/test-artifacts" # see "jump_ci retrieve_artifacts" below env_pass_lists = config.project.get_config("env.pass_lists", print=False) @@ -112,7 +111,7 @@ def do_jump_ci(cluster=None, project=None, test_args=None): variables_overrides_dict[f"PR_POSITIONAL_ARG_{idx+1}"] = arg config.project.set_config("overrides", variables_overrides_dict) - + next_pr_positional_arg_count = idx + 2 else: if not os.environ.get("OPENSHIFT_CI") == "true": logging.fatal("Not running in OpenShift CI. Don't know how to rewrite the variable_overrides_file. Aborting.") @@ -120,43 +119,62 @@ def do_jump_ci(cluster=None, project=None, test_args=None): project = config.project.get_config("overrides.PR_POSITIONAL_ARG_2") - variables_overrides_dict = rewrite_variables_overrides( + variables_overrides_dict, next_pr_positional_arg_count = rewrite_variables_overrides( config.project.get_config("overrides") ) - run.run_toolbox( - "jump_ci", "prepare_step", - cluster=cluster, - lock_owner=utils.get_lock_owner(), - project=project, - step=command, - env_file=env_fd_path, - variables_overrides_dict=variables_overrides_dict, - secrets_path_env_key=secrets_path_env_key, - ) + for idx, multi_run_args in enumerate((config.project.get_config("multi_run.args") or [...])): + test_artifacts_dirname = "test-artifacts" + multi_run_args_dict = {} + if multi_run_args is not ...: + test_artifacts_dirname = f"test-artifacts-{idx:03d}" + with open(env.ARTIFACTS_DIR / "multi_run_args.list", "a+") as f: + print(f"{test_artifacts_dirname}: {multi_run_args}") + for idx, multi_run_arg in enumerate((multi_run_args if isinstance(multi_run_args, list) else [multi_run_args])): + variables_overrides_dict[f"PR_POSITIONAL_ARG_{next_pr_positional_arg_count+idx}"] = multi_run_arg + else: + test_artifacts_dirname = "test-artifacts" + + if step_dir := os.environ.get("TOPSAIL_OPENSHIFT_CI_STEP_DIR"): + # see "jump_ci retrieve_artifacts" below + extra_env["TOPSAIL_OPENSHIFT_CI_STEP_DIR"] = f"{step_dir}/{test_artifacts_dirname}" + + run.run_toolbox( + "jump_ci", "prepare_step", + cluster=cluster, + lock_owner=utils.get_lock_owner(), + project=project, + step=command, + env_file=env_fd_path, + variables_overrides_dict=(variables_overrides_dict | multi_run_args_dict), + secrets_path_env_key=secrets_path_env_key, + ) - try: - tunnelling.run_with_ansible_ssh_conf(f"bash {cluster_lock_dir}/test/{command}/entrypoint.sh") - logging.info(f"Test step '{command}' on cluster '{cluster}' succeeded.") - failed = False - except subprocess.CalledProcessError as e: - logging.fatal(f"Test step '{command}' on cluster '{cluster}' FAILED.") - failed = True - except run.SignalError as e: - logging.error(f"Caught signal {e.sig}. Aborting.") - raise - finally: - # always run the cleanup to be sure that the container doesn't stay running - tunnelling.run_with_ansible_ssh_conf(f"bash {cluster_lock_dir}/test/{command}/entrypoint.sh cleanup") - - run.run_toolbox( - "jump_ci", "retrieve_artifacts", - cluster=cluster, - lock_owner=utils.get_lock_owner(), - remote_dir=f"test/{command}/artifacts", - local_dir=f"../test-artifacts", # copy to the main artifact directory - mute_stdout=True, - ) + try: + tunnelling.run_with_ansible_ssh_conf(f"bash {cluster_lock_dir}/test/{command}/entrypoint.sh") + logging.info(f"Test step '{command}' on cluster '{cluster}' succeeded.") + failed = False + except subprocess.CalledProcessError as e: + logging.fatal(f"Test step '{command}' on cluster '{cluster}' FAILED.") + failed = True + except run.SignalError as e: + logging.error(f"Caught signal {e.sig}. Aborting.") + raise + finally: + # always run the cleanup to be sure that the container doesn't stay running + tunnelling.run_with_ansible_ssh_conf(f"bash {cluster_lock_dir}/test/{command}/entrypoint.sh cleanup") + + run.run_toolbox( + "jump_ci", "retrieve_artifacts", + cluster=cluster, + lock_owner=utils.get_lock_owner(), + remote_dir=f"test/{command}/artifacts", + local_dir=f"../{test_artifacts_dirname}", # copy to the main artifact directory + mute_stdout=True, + ) + + if failed and config.project.get_config("multi_run.stop_on_error"): + break jump_ci_artifacts = env.ARTIFACT_DIR / "jump-ci-artifacts" jump_ci_artifacts.mkdir(parents=True, exist_ok=True)