[jump-ci] Allow testing multiple presets (#646)

Author: kpouget

projects/jump_ci/testing/config.yaml

@@ -33,3 +33,6 @@ exec_list:
   prepare_ci: null
   test_ci: null
   post_cleanup_ci: null
+multi_run:
+  args: []
+  stop_on_error: false

projects/jump_ci/testing/test.py

@@ -24,8 +24,11 @@ def rewrite_variables_overrides(variable_overrides_dict):
     new_args_str = new_variable_overrides[f"PR_POSITIONAL_ARGS"] = " ".join(new_args)
     new_variable_overrides["PR_POSITIONAL_ARGS"] = variable_overrides_dict["PR_POSITIONAL_ARG_0"]
     logging.info(f"New args to execute on the jump host: {new_args_str}")
+
+    idx = 0
     for idx, value in enumerate(new_args):
         new_variable_overrides[f"PR_POSITIONAL_ARG_{idx+1}"] = value
+    next_pr_positional_arg_count = idx + 2
 
     for k, v in variable_overrides_dict.items():
         if k.startswith("PR_POSITIONAL_ARG"): continue
@@ -44,7 +47,7 @@ def rewrite_variables_overrides(variable_overrides_dict):
         new_variable_overrides[k] = v
         logging.info(f"Passing '{k}: {v}' to the new variables overrides")
 
-    return new_variable_overrides
+    return new_variable_overrides, next_pr_positional_arg_count
 
 
 def jump_ci(command):
@@ -67,26 +70,28 @@ def do_jump_ci(cluster=None, project=None, test_args=None):
 
         secrets_path_env_key = config.project.get_config("secrets.dir.env_key")
 
-        env_fd_path, env_file = utils.get_tmp_fd()
         extra_env = dict(
             TOPSAIL_JUMP_CI="true",
             TOPSAIL_JUMP_CI_INSIDE_JUMP_HOST="true",
         )
-        if step_dir := os.environ.get("TOPSAIL_OPENSHIFT_CI_STEP_DIR"):
-            extra_env["TOPSAIL_OPENSHIFT_CI_STEP_DIR"] = f"{step_dir}/test-artifacts" # see "jump_ci retrieve_artifacts" below
 
-        env_pass_lists = config.project.get_config("env.pass_lists", print=False)
+        def prepare_env_file(_extra_env):
+            env_fd_path, env_file = utils.get_tmp_fd()
+
+            env_pass_lists = config.project.get_config("env.pass_lists", print=False)
 
-        env_pass_list = set()
-        for _, pass_list in (env_pass_lists or {}).items():
-            env_pass_list.update(pass_list)
+            env_pass_list = set()
+            for _, pass_list in (env_pass_lists or {}).items():
+                env_pass_list.update(pass_list)
 
-        for k, v in (os.environ | extra_env).items():
-            if k not in (env_pass_list | extra_env.keys()): continue
+            for k, v in (os.environ | _extra_env).items():
+                if k not in (env_pass_list | _extra_env.keys()): continue
 
-            print(f"{k}={shlex.quote(v)}", file=env_file)
+                print(f"{k}={shlex.quote(v)}", file=env_file)
 
-        env_file.flush()
+            env_file.flush()
+
+            return env_fd_path, env_file
 
         variable_overrides_file = pathlib.Path(os.environ.get("ARTIFACT_DIR")) / "variable_overrides.yaml"
 
@@ -100,6 +105,8 @@ def do_jump_ci(cluster=None, project=None, test_args=None):
 
         run.run_toolbox("jump_ci", "ensure_lock", cluster=cluster, owner=utils.get_lock_owner())
 
+        cluster_lock_dir = f" /tmp/topsail_{cluster}"
+
         if test_args is not None:
             variables_overrides_dict = dict(
                 PR_POSITIONAL_ARGS=test_args,
@@ -110,55 +117,86 @@ def do_jump_ci(cluster=None, project=None, test_args=None):
                 variables_overrides_dict[f"PR_POSITIONAL_ARG_{idx+1}"] = arg
 
             config.project.set_config("overrides", variables_overrides_dict)
-
+            next_pr_positional_arg_count = idx + 2
         else:
             if not os.environ.get("OPENSHIFT_CI") == "true":
                 logging.fatal("Not running in OpenShift CI. Don't know how to rewrite the variable_overrides_file. Aborting.")
                 raise SystemExit(1)
 
             project = config.project.get_config("overrides.PR_POSITIONAL_ARG_2")
 
-            variables_overrides_dict = rewrite_variables_overrides(
+            variables_overrides_dict, next_pr_positional_arg_count = rewrite_variables_overrides(
                 config.project.get_config("overrides")
             )
 
-        run.run_toolbox(
-            "jump_ci", "prepare_step",
-            cluster=cluster,
-            lock_owner=utils.get_lock_owner(),
-            project=project,
-            step=command,
-            env_file=env_fd_path,
-            variables_overrides_dict=variables_overrides_dict,
-            secrets_path_env_key=secrets_path_env_key,
-        )
-
-        try:
-            tunnelling.run_with_ansible_ssh_conf(f"bash /tmp/{cluster}/test/{command}/entrypoint.sh")
-            logging.info(f"Test step '{command}' on cluster '{cluster}' succeeded.")
-            failed = False
-        except subprocess.CalledProcessError as e:
-            logging.fatal(f"Test step '{command}' on cluster '{cluster}' FAILED.")
-            failed = True
-        except run.SignalError as e:
-            logging.error(f"Caught signal {e.sig}. Aborting.")
-            raise
-        finally:
-            # always run the cleanup to be sure that the container doesn't stay running
-            tunnelling.run_with_ansible_ssh_conf(f"bash /tmp/{cluster}/test/{command}/entrypoint.sh cleanup")
-
-        run.run_toolbox(
-            "jump_ci", "retrieve_artifacts",
-            cluster=cluster,
-            lock_owner=utils.get_lock_owner(),
-            remote_dir=f"test/{command}/artifacts",
-            local_dir=f"../test-artifacts", # copy to the main artifact directory
-            mute_stdout=True,
-        )
+        for idx, multi_run_args in enumerate((config.project.get_config("multi_run.args") or [...])):
+            multi_run_args_dict = {}
+            multi_run_dirname = None
+            test_artifacts_dirname = "test-artifacts"
+
+            if multi_run_args is not ...:
+
+                multi_run_args_lst = multi_run_args if isinstance(multi_run_args, list) else [multi_run_args]
+                multi_run_dirname = f"multi_run__{'_'.join(multi_run_args_lst)}"
+
+                with open(env.ARTIFACT_DIR / "multi_run_args.list", "a+") as f:
+                    print(f"{multi_run_dirname}: {multi_run_args}")
+
+                for idx, multi_run_arg in enumerate(multi_run_args_lst):
+                    variables_overrides_dict[f"PR_POSITIONAL_ARG_{next_pr_positional_arg_count+idx}"] = multi_run_arg
+
+            with env.NextArtifactDir(multi_run_dirname) if multi_run_dirname else open("/dev/null"):
+
+                if multi_run_dirname:
+                    test_artifacts_dirname = f"{env.ARTIFACT_DIR.name}/{test_artifacts_dirname}"
+
+                if step_dir := os.environ.get("TOPSAIL_OPENSHIFT_CI_STEP_DIR"):
+                    # see "jump_ci retrieve_artifacts" below
+                    extra_env["TOPSAIL_OPENSHIFT_CI_STEP_DIR"] = f"{step_dir}/{test_artifacts_dirname}"
+
+                env_fd_path, env_file = prepare_env_file(extra_env)
+
+                run.run_toolbox(
+                    "jump_ci", "prepare_step",
+                    cluster=cluster,
+                    lock_owner=utils.get_lock_owner(),
+                    project=project,
+                    step=command,
+                    env_file=env_fd_path,
+                    variables_overrides_dict=(variables_overrides_dict | multi_run_args_dict),
+                    secrets_path_env_key=secrets_path_env_key,
+                )
+                env_file.close()
+
+                try:
+                    tunnelling.run_with_ansible_ssh_conf(f"bash {cluster_lock_dir}/test/{command}/entrypoint.sh")
+                    logging.info(f"Test step '{command}' on cluster '{cluster}' succeeded.")
+                    failed = False
+                except subprocess.CalledProcessError as e:
+                    logging.fatal(f"Test step '{command}' on cluster '{cluster}' FAILED.")
+                    failed = True
+                except run.SignalError as e:
+                    logging.error(f"Caught signal {e.sig}. Aborting.")
+                    raise
+                finally:
+                    # always run the cleanup to be sure that the container doesn't stay running
+                    tunnelling.run_with_ansible_ssh_conf(f"bash {cluster_lock_dir}/test/{command}/entrypoint.sh cleanup")
+
+                run.run_toolbox(
+                    "jump_ci", "retrieve_artifacts",
+                    cluster=cluster,
+                    lock_owner=utils.get_lock_owner(),
+                    remote_dir=f"test/{command}/artifacts",
+                    local_dir=f"../{pathlib.Path(test_artifacts_dirname).name}", # copy to the main artifact directory
+                    mute_stdout=True,
+                )
+
+            if failed and config.project.get_config("multi_run.stop_on_error"):
+                break
 
         jump_ci_artifacts = env.ARTIFACT_DIR / "jump-ci-artifacts"
         jump_ci_artifacts.mkdir(parents=True, exist_ok=True)
-        run.run(f'mv {env.ARTIFACT_DIR}/0* {jump_ci_artifacts}/')
+        run.run(f'mv {env.ARTIFACT_DIR}/*__jump_ci_* {jump_ci_artifacts}/')
 
         if failed:
             raise SystemExit(1)

projects/jump_ci/toolbox/jump_ci_ensure_lock/tasks/main.yml

@@ -23,18 +23,22 @@
     fail: msg="The cluster KUBECONFIG ({{ cluster_kubeconfig }}) does not exists"
     when: not cluster_kubeconfig_stat.stat.exists
 
+- name: Save the name of the cluster lock directory
+  set_fact:
+    cluster_lock_dir: "/tmp/topsail_{{ jump_ci_ensure_lock_cluster }}"
+
 - name: "Get the stats of the lock directory: {{ jump_ci_ensure_lock_cluster }}"
   stat:
-    path: "/tmp/{{ jump_ci_ensure_lock_cluster }}"
+    path: "{{ cluster_lock_dir }}"
   register: lock_directory_stat
 
 - name: Fail if the lock directory doesn't exist
-  fail: msg="The lock directory '/tmp/{{ jump_ci_ensure_lock_cluster }}' does not exists"
+  fail: msg="The lock directory '{{ cluster_lock_dir }}' does not exists"
   when: not lock_directory_stat.stat.exists
 
 - name: Get the owner of the lock
   command:
-    cat "/tmp/{{ jump_ci_ensure_lock_cluster }}/lock_owner"
+    cat "{{ cluster_lock_dir }}/lock_owner"
   register: lock_owner_content_cmd
 
 - name: Fail if the owner is different

projects/jump_ci/toolbox/jump_ci_prepare_step/tasks/main.yml

@@ -9,7 +9,7 @@
 
 - name: Fetch the name of the image to use
   shell:
-    cat "/tmp/{{ jump_ci_prepare_step_cluster }}/pod.image"
+    cat "{{ cluster_lock_dir }}/pod.image"
   register: pod_image_cmd
   failed_when: false
 
@@ -19,13 +19,14 @@
 
 - name: Generate the step artifacts dirname
   set_fact:
-    step_artifact_dir: "/tmp/{{ jump_ci_prepare_step_cluster }}/test/{{ jump_ci_prepare_step_step }}"
+    step_artifact_dir: "{{ cluster_lock_dir }}/test/{{ jump_ci_prepare_step_step }}"
 
 - name: Ensure that the step artifacts dirname directory is empty
   ansible.builtin.file:
     state: absent
     path: "{{ step_artifact_dir }}"
     mode: '0755'
+  no_log: true # can be too chatty when cleaning an existing artifacts directly
 
 - name: Ensure that the step test dirname directory exists
   ansible.builtin.file:
@@ -68,22 +69,22 @@
 - name: Ensure that the secrets directory exists
   ansible.builtin.file:
     state: directory
-    path: "/tmp/{{ jump_ci_prepare_step_cluster }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}"
+    path: "{{ cluster_lock_dir }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}"
     mode: '0755'
 
 - name: Copy the secrets to the jump host
   ansible.posix.synchronize:
     src: "{{ lookup('env', jump_ci_prepare_step_secrets_path_env_key) }}/"
-    dest: "/tmp/{{ jump_ci_prepare_step_cluster }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}/"
+    dest: "{{ cluster_lock_dir }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}/"
     mode: "push"
   no_log: true # not necessary, and leaks the hostnames and secret filenames (they aren't secret)
   when: jump_ci_prepare_step_secrets_path_env_key
 
 - name: Ensure that the secret dirs have the right permission
-  command: find "/tmp/{{ jump_ci_prepare_step_cluster }}/secrets" -type d -exec chmod 0755 {} \;
+  command: find "{{ cluster_lock_dir }}/secrets" -type d -exec chmod 0755 {} \;
 
 - name: Ensure that the secret files have the right permission
-  command: find "/tmp/{{ jump_ci_prepare_step_cluster }}/secrets/" -type f -exec chmod 0644 {} \;
+  command: find "{{ cluster_lock_dir }}/secrets/" -type f -exec chmod 0644 {} \;
 
 - name: Prepare the entrypoint script
   template:

projects/jump_ci/toolbox/jump_ci_prepare_step/templates/entrypoint.sh.j2

@@ -16,7 +16,7 @@ cleanup() {
 
     # Not cleaning up the secrets for now. They are safe anyway
     # echo Cleaning up the secrets ..
-    # rm -rf /tmp/{{ jump_ci_prepare_step_cluster }}/secrets
+    # rm -rf {{ cluster_lock_dir }}/secrets
 }
 
 # ---
@@ -49,7 +49,7 @@ podman run \
   --volume "{{ step_artifact_dir }}/artifacts:/logs/artifacts" \
   --env ARTIFACT_DIR="/logs/artifacts" \
   \
-  --volume /tmp/{{ jump_ci_prepare_step_cluster }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}:/run/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}:ro \
+  --volume {{ cluster_lock_dir }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}:/run/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}:ro \
   --env {{ jump_ci_prepare_step_secrets_path_env_key }}="/run/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}" \
   \
   --volume {{ cluster_kubeconfig }}:/run/secrets/kubeconfig:ro \

projects/jump_ci/toolbox/jump_ci_prepare_topsail/tasks/main.yml

@@ -8,7 +8,7 @@
 
 - name: Generate the artifact dirname
   set_fact:
-    artifact_dir: "/tmp/{{ jump_ci_prepare_topsail_cluster }}/jump_ci_artifacts"
+    artifact_dir: "{{ cluster_lock_dir }}/jump_ci_artifacts"
 
 - name: Ensure that the logs directory exists
   ansible.builtin.file:
@@ -49,7 +49,7 @@
 
 - name: Set TOPSAIL's directory
   set_fact:
-    topsail_home: "/tmp/{{ jump_ci_prepare_topsail_cluster }}/topsail"
+    topsail_home: "{{ cluster_lock_dir }}/topsail"
 
 - name: Set git command
   set_fact:
@@ -154,7 +154,7 @@
 
 - name: "Build image as an update {{ image_full_name }}"
   shell: |
-    cat > /tmp/overlay.containerfile <<EOF
+    cat > {{ cluster_lock_dir }}/overlay.containerfile <<EOF
     FROM hell
     ENTRYPOINT []
     CMD []
@@ -167,7 +167,7 @@
 
     time podman build \
       --tag "{{ image_full_name }}" \
-      --file /tmp/overlay.containerfile \
+      --file {{ cluster_lock_dir }}/overlay.containerfile \
       --from "{{ jump_ci_prepare_topsail_image_name }}:{{ jump_ci_prepare_topsail_update_from_imagetag }}"
   when:
   - has_image.rc != 0
@@ -186,4 +186,4 @@
 
 - name: Save the name of the image to use
   shell:
-    echo "{{ image_full_name }}" > "/tmp/{{ jump_ci_prepare_topsail_cluster }}/pod.image"
+    echo "{{ image_full_name }}" > "{{ cluster_lock_dir }}/pod.image"

projects/jump_ci/toolbox/jump_ci_release_lock/tasks/main.yml

@@ -9,4 +9,4 @@
 
 - name: Release the lock directory
   command:
-    rm -rf "/tmp/{{ jump_ci_release_lock_cluster }}"
+    rm -rf "{{ cluster_lock_dir }}"

projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/tasks/main.yml

@@ -9,7 +9,7 @@
 
 - name: Compute the path of the directory to retrieve
   set_fact:
-    remote_dir_path: "/tmp/{{ jump_ci_retrieve_artifacts_cluster }}/{{ jump_ci_retrieve_artifacts_remote_dir }}"
+    remote_dir_path: "{% if cluster_lock_dir is defined %}{{ cluster_lock_dir }}{% else %}{{ '/tmp/topsail_'+jump_ci_retrieve_artifacts_cluster}}{% endif %}/{{ jump_ci_retrieve_artifacts_remote_dir }}"
 
 - name: Get the size of the directory to retrieve
   command:

projects/jump_ci/toolbox/jump_ci_take_lock/tasks/main.yml

@@ -3,11 +3,15 @@
   shell:
     whoami
 
+- name: Save the name of the cluster lock directory
+  set_fact:
+    cluster_lock_dir: "/tmp/topsail_{{ jump_ci_take_lock_cluster }}"
+
 - name: Create the lock directory
   shell: |
-    if ! mkdir "/tmp/{{ jump_ci_take_lock_cluster }}" 2>/dev/null; then
+    if ! mkdir "{{ cluster_lock_dir }}" 2>/dev/null; then
       echo "$(date) Lock '{{ jump_ci_take_lock_cluster }}' already taken ..."
-      cat "/tmp/{{ jump_ci_take_lock_cluster }}/lock_owner 2>/dev/null" || true
+      cat "{{ cluster_lock_dir }}/lock_owner 2>/dev/null" || true
       exit 1
     else
       echo "$(date) Lock '{{ jump_ci_take_lock_cluster }}' acquired.";
@@ -19,7 +23,7 @@
 
 - name: Log info about the lock owner
   shell:
-    echo "{{ jump_ci_take_lock_owner }}" > "/tmp/{{ jump_ci_take_lock_cluster }}/lock_owner"
+    echo "{{ jump_ci_take_lock_owner }}" > "{{ cluster_lock_dir }}/lock_owner"
 
 - name: Ensure that the lock is owned
   include_role: