Allow to skip module artifact check/errors (#417)

redis-performance · Apr 30, 2023 · 8df9082 · 8df9082
1 parent 4960bf0
commit 8df9082
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 26 deletions.
diff --git a/redisbench_admin/run_remote/args.py b/redisbench_admin/run_remote/args.py
@@ -119,5 +119,11 @@ def create_run_remote_arguments(parser):
         action="store_true",
         help="skip environment variables check",
     )
+    parser.add_argument(
+        "--continue-on-module-check-error",
+        default=False,
+        action="store_true",
+        help="Continue running benchmarks even if module check failed",
+    )
 
     return parser
diff --git a/redisbench_admin/run_remote/remote_db.py b/redisbench_admin/run_remote/remote_db.py
@@ -102,6 +102,7 @@ def remote_db_spin(
     redis_password=None,
     flushall_on_every_test_start=False,
     ignore_keyspace_errors=False,
+    continue_on_module_check_error=False,
 ):
     (
         _,
@@ -131,6 +132,7 @@ def remote_db_spin(
             remote_module_file_dir,
             server_public_ip,
             username,
+            continue_on_module_check_error,
         )
     # setup Redis
     redis_setup_result = True
@@ -229,6 +231,11 @@ def remote_db_spin(
                 True,
                 username,
             )
+            raise Exception(
+                "A error occurred while spinning DB: {}. Aborting...".format(
+                    e.__str__()
+                )
+            )
 
     if cluster_enabled and skip_redis_setup is False:
         setup_redis_cluster_from_conns(

diff --git a/redisbench_admin/run_remote/run_remote.py b/redisbench_admin/run_remote/run_remote.py
@@ -178,21 +178,29 @@ def run_remote_command_logic(args, project_name, project_version):
             logging.critical("{}. Exiting right away!".format(failure_reason))
             exit(1)
 
+    continue_on_module_check_error = args.continue_on_module_check_error
     module_check_status, error_message = redis_modules_check(local_module_files)
     if module_check_status is False:
-        if webhook_notifications_active:
-            failure_reason = error_message
-            generate_failure_notification(
-                webhook_client_slack,
-                ci_job_name,
-                ci_job_link,
-                failure_reason,
-                tf_github_org,
-                tf_github_repo,
-                tf_github_branch,
-                None,
+        if continue_on_module_check_error is False:
+            if webhook_notifications_active:
+                failure_reason = error_message
+                generate_failure_notification(
+                    webhook_client_slack,
+                    ci_job_name,
+                    ci_job_link,
+                    failure_reason,
+                    tf_github_org,
+                    tf_github_repo,
+                    tf_github_branch,
+                    None,
+                )
+            exit(1)
+        else:
+            logging.error(
+                "the module check failed with the following message {} but you've decided to continue anyway.".format(
+                    error_message
+                )
             )
-        exit(1)
 
     common_properties_log(
         tf_bin_path,
@@ -505,6 +513,7 @@ def run_remote_command_logic(args, project_name, project_version):
                                             redis_password,
                                             flushall_on_every_test_start,
                                             ignore_keyspace_errors,
+                                            continue_on_module_check_error,
                                         )
                                         if benchmark_type == "read-only":
                                             ro_benchmark_set(

diff --git a/redisbench_admin/run_remote/standalone.py b/redisbench_admin/run_remote/standalone.py
@@ -72,6 +72,7 @@ def remote_module_files_cp(
     remote_module_file_dir,
     server_public_ip,
     username,
+    continue_on_module_check_error=False,
 ):
     remote_module_files = []
     if local_module_files is not None:
@@ -99,22 +100,24 @@ def remote_module_files_cp(
                     os.path.basename(local_module_file_and_plugin),
                 )
                 # copy the module to the DB machine
-                copy_file_to_remote_setup(
+                cp_res = copy_file_to_remote_setup(
                     server_public_ip,
                     username,
                     private_key,
                     local_module_file_and_plugin,
                     remote_module_file,
                     None,
                     port,
+                    continue_on_module_check_error,
                 )
-                execute_remote_commands(
-                    server_public_ip,
-                    username,
-                    private_key,
-                    ["chmod 755 {}".format(remote_module_file)],
-                    port,
-                )
+                if cp_res:
+                    execute_remote_commands(
+                        server_public_ip,
+                        username,
+                        private_key,
+                        ["chmod 755 {}".format(remote_module_file)],
+                        port,
+                    )
                 if pos > 1:
                     remote_module_files_in = remote_module_files_in + " "
                 remote_module_files_in = remote_module_files_in + remote_module_file

diff --git a/redisbench_admin/utils/remote.py b/redisbench_admin/utils/remote.py
@@ -63,8 +63,10 @@ def copy_file_to_remote_setup(
     remote_file,
     dirname=None,
     port=22,
+    continue_on_module_check_error=False,
 ):
     full_local_path = local_file
+    res = False
     if dirname is not None:
         full_local_path = "{}/{}".format(dirname, local_file)
     logging.info(
@@ -94,12 +96,20 @@ def copy_file_to_remote_setup(
         )
         res = True
     else:
-        logging.error(
-            "Local file {} does not exists. aborting...".format(full_local_path)
-        )
-        raise Exception(
-            "Local file {} does not exists. aborting...".format(full_local_path)
-        )
+        if continue_on_module_check_error:
+            logging.warning(
+                "Continuing running benchmarks after module check failed on file: {}. Full path {}".format(
+                    local_file, full_local_path
+                )
+            )
+        else:
+            logging.error(
+                "Local file {} does not exists. aborting...".format(full_local_path)
+            )
+
+            raise Exception(
+                "Local file {} does not exists. aborting...".format(full_local_path)
+            )
     return res