Update async benchmarks execution

pyproject.toml

@@ -41,6 +41,7 @@ slack-sdk = "^3.15.2"
 slack-bolt = "^1.13.0"
 certifi = "^2021.10.8"
 pygithub = "^1.57"
+codecov = "^2.1.13"
 
 [tool.poetry.dev-dependencies]
 pytest = "^4.6"

redisbench_admin/export/args.py

@@ -52,6 +52,7 @@ def create_export_arguments(parser):
     parser.add_argument("--github_repo", type=str, default=None)
     parser.add_argument("--github_org", type=str, default=None)
     parser.add_argument("--github_branch", type=str, default=None, nargs="?", const="")
+    parser.add_argument("--circle_pull_request", type=str, default=None, nargs="?", const="")
     parser.add_argument("--triggering_env", type=str, default="ci")
     parser.add_argument(
         "--exporter",

redisbench_admin/run_async/render_files.py

@@ -1,79 +1,88 @@
-import jinja2
+from argparse import Namespace
 
 
-def renderServiceFile(access_key, region, secret_key, gh_token, job_name, args, argv):
-    environment = jinja2.Environment()
-    template = environment.from_string(
-        """[Unit]
-Description=Redisbench-admin run service
-
-[Service]
-Type=oneshot
-Environment="PATH=/home/ubuntu/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-Environment="AWS_ACCESS_KEY_ID={{access_key}}"
-Environment="AWS_DEFAULT_REGION={{region}}"
-Environment="AWS_SECRET_ACCESS_KEY={{secret_key}}"
-{% if gh_token != None %}Environment="GH_TOKEN={{gh_token}}"{% endif %}
-{% if job_name != None %}Environment="CIRCLE_JOB={{job_name}}"{% endif %}
-WorkingDirectory=/home/ubuntu/work_dir/tests/benchmarks
-Type=simple
-User=ubuntu
-ExecStart=/home/ubuntu/work_dir/redisbench-admin/.venv/bin/python /home/ubuntu/work_dir/redisbench-admin/run.py run-remote {{args}}
+def render_run_script(
+    access_key: str,
+    region: str,
+    secret_key: str,
+    gh_token: str,
+    gh_branch: str,
+    job_name: str,
+    circle_pull_request: str,
+    args: Namespace,
+    argv: list,
+) -> None:
+    """
+    Generate a bash script for running a remote RedisBench job.
 
-[Install]
-WantedBy=multi-user.target
+    Args:
+        access_key (str): AWS access key ID.
+        region (str): AWS region.
+        secret_key (str): AWS secret access key.
+        gh_token (str): GitHub token.
+        job_name (str): Name of the job.
+        args (Namespace): Command-line arguments for the RedisBench job.
+        argv (list): List of additional command-line arguments.
     """
-    )
+    gh_branch = args.gh_branch if args.gh_branch else None
+    circle_pull_request = args.circle_pull_request if args.circle_pull_request else None
+
     if "--private_key" not in argv:
-        argv.append("--private_key")
-        argv.append("/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem")
+        argv.extend(
+            [
+                "--private_key",
+                "/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem",
+            ]
+        )
     else:
+        private_key_index = argv.index("--private_key")
         argv[
-            argv.index(args.private_key)
+            private_key_index + 1
         ] = "/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem"
-    if len(args.module_path) != 0:
-        argv[argv.index(args.module_path[0])] = (
-            "/home/ubuntu/work_dir/tests/benchmarks/"
-            + args.module_path[0].split("/")[-1]
-        )
-    argv_str = " ".join(argv)
-    with open("redisbench-admin.service", mode="w", encoding="utf-8") as results:
-        results.write(
-            template.render(
-                access_key=access_key,
-                region=region,
-                secret_key=secret_key,
-                gh_token=gh_token,
-                job_name=job_name,
-                args=argv_str,
-            )
-        )
 
+    if args.module_path:
+        module_path_index = argv.index(args.module_path[0])
+        module_filename = args.module_path[0].split("/")[-1]
+        argv[
+            module_path_index
+        ] = f"/home/ubuntu/work_dir/tests/benchmarks/{module_filename}"
+
+    argv_str = " ".join(argv)
+    script_lines = [
+        f'export AWS_ACCESS_KEY_ID="{access_key}"',
+        f'export AWS_DEFAULT_REGION="{region}"',
+        f'export AWS_SECRET_ACCESS_KEY="{secret_key}"',
+        f'export GH_TOKEN="{gh_token}"',
+        f'export CIRCLE_JOB="{job_name}"',
+        f'export GH_TOKEN="{gh_token}"' if gh_token is not None else "",
+        f'export CIRCLE_JOB="{job_name}"' if job_name is not None else "",
+        f"redisbench-admin run-remote {argv_str}",
+        f"""redisbench-admin compare  \
+                  --defaults_filename ./tests/benchmarks/defaults.yml \
+                  --comparison-branch {gh_branch} \
+                  --auto-approve \
+                  --pull-request {circle_pull_request}
+         """
+        if circle_pull_request is not "None"
+        else "",
+    ]
+    with open("runbenchmark", mode="w", encoding="utf-8") as results:
+        results.write("\n".join(script_lines))
 
-def renderRunFile():
-    with open("run.py", mode="w", encoding="utf-8") as run_file:
-        run_file.write(
-            """#!/usr/local/bin/python
-# -*- coding: utf-8 -*-
-import re
-import sys
-from redisbench_admin.cli import main
 
-if __name__ == "__main__":
-    sys.argv[0] = re.sub(r"(-script\\.pyw|\\.exe)?$", "", sys.argv[0])
-    sys.exit(main())
-"""
-        )
+def save_pem_file(pem_data: str) -> None:
+    """
+    Save PEM-encoded data to a file after processing.
 
+    Args:
+        pem_data (str): The PEM-encoded data to be processed and saved.
+    """
+    pem_data = pem_data.replace("-----BEGIN RSA PRIVATE KEY-----", "")
+    pem_data = pem_data.replace("-----END RSA PRIVATE KEY-----", "")
+    pem_data = pem_data.replace(" ", "\n")
+    pem_data = (
+        "-----BEGIN RSA PRIVATE KEY-----" + pem_data + "-----END RSA PRIVATE KEY-----"
+    )
 
-def savePemFile(pem_data):
     with open("benchmarks.redislabs.pem", mode="w", encoding="utf-8") as pem_file:
-        pem_data = pem_data.replace("-----BEGIN RSA PRIVATE KEY-----", "")
-        pem_data = pem_data.replace("-----END RSA PRIVATE KEY-----", "")
-        pem_data = pem_data.replace(" ", "\n")
-        pem_data = (
-            "-----BEGIN RSA PRIVATE KEY-----"
-            + pem_data
-            + "-----END RSA PRIVATE KEY-----"
-        )
         pem_file.write(pem_data)

redisbench_admin/run_async/run_async.py

@@ -22,9 +22,8 @@
 from redisbench_admin.run_async.benchmark import BenchmarkClass
 from redisbench_admin.run_remote.notifications import generate_failure_notification
 from redisbench_admin.run_async.render_files import (
-    renderServiceFile,
-    renderRunFile,
-    savePemFile,
+    render_run_script,
+    save_pem_file,
 )
 from redisbench_admin.run_async.terraform import (
     TerraformClass,
@@ -204,8 +203,8 @@ def run_async_command_logic(argv, args, project_name, project_version):
         benchmark.benchmark_definitions, benchmark.default_specs
     )
 
-    # create service file
-    renderServiceFile(
+    # create a script to run benchmarks
+    render_run_script(
         access_key=EC2_ACCESS_KEY,
         secret_key=EC2_SECRET_KEY,
         region=EC2_REGION,
@@ -215,9 +214,8 @@ def run_async_command_logic(argv, args, project_name, project_version):
         argv=argv,
     )
 
-    # create run.py file for running redisbench-cli
-    renderRunFile()
-    savePemFile(EC2_PRIVATE_PEM)
+    # save ssh key for ec2 machine that later on will be used to run commands
+    save_pem_file(EC2_PRIVATE_PEM)
 
     # copy module file
     if len(args.module_path) != 0:
@@ -260,14 +258,8 @@ def run_async_command_logic(argv, args, project_name, project_version):
             "/etc/apt/sources.list.d/hashicorp.list",
             "sudo apt update",
             "sudo apt install terraform",
-            "curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.2.2 python3 -",
-            'cd work_dir/redisbench-admin && PATH="/home/ubuntu/.local/bin:$PATH" poetry config '
-            "virtualenvs.in-project true",
-            'cd work_dir/redisbench-admin && PATH="/home/ubuntu/.local/bin:$PATH" poetry install',
-            "./work_dir/deps/readies/bin/getdocker",
-            "cd work_dir && sudo cp tests/benchmarks/redisbench-admin.service /etc/systemd/system",
-            "sudo systemctl daemon-reload",
-            "sudo systemctl start redisbench-admin.service",
+            "pip3 install redisbench-admin -update",
+            "./runbenchmark &",
         ],
         "22",
         get_pty=True,