From 63cad39bf55ad0e8063f8d57f6e5c0b7cfa11dee Mon Sep 17 00:00:00 2001 From: filipe oliveira Date: Fri, 21 Jul 2023 00:56:11 +0100 Subject: [PATCH] In case of connection error at end of benchmark. collect all logs and fail (#419) --- pyproject.toml | 2 +- redisbench_admin/run_remote/run_remote.py | 137 ++++++++++++---------- 2 files changed, 79 insertions(+), 60 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c5b5aca..ca69bed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisbench-admin" -version = "0.10.15" +version = "0.10.16" description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )." authors = ["filipecosta90 ","Redis Performance Group "] readme = "README.md" diff --git a/redisbench_admin/run_remote/run_remote.py b/redisbench_admin/run_remote/run_remote.py index 065cfd8..e203f84 100644 --- a/redisbench_admin/run_remote/run_remote.py +++ b/redisbench_admin/run_remote/run_remote.py @@ -768,52 +768,30 @@ def run_remote_command_logic(args, project_name, project_version): tf_github_branch, artifact_version ) ): - ( - end_time_ms, - _, - overall_end_time_metrics, - ) = collect_redis_metrics( - redis_conns, - ["memory"], - { - "memory": [ - "used_memory", - "used_memory_dataset", - ] - }, - ) - if total_shards_cpu_usage is not None: - overall_end_time_metrics[ - "total_shards_used_cpu_pct" - ] = total_shards_cpu_usage - expire_ms = 7 * 24 * 60 * 60 * 1000 - export_redis_metrics( - artifact_version, - end_time_ms, - overall_end_time_metrics, - rts, - setup_name, - setup_type, - test_name, - tf_github_branch, - tf_github_org, - tf_github_repo, - tf_triggering_env, - {"metric-type": "redis-metrics"}, - expire_ms, - ) - if collect_commandstats: + try: ( end_time_ms, _, - overall_commandstats_metrics, + overall_end_time_metrics, ) = collect_redis_metrics( - redis_conns, ["commandstats"] + redis_conns, + ["memory"], + { + "memory": [ + "used_memory", + "used_memory_dataset", + ] + }, ) + if total_shards_cpu_usage is not None: + overall_end_time_metrics[ + "total_shards_used_cpu_pct" + ] = total_shards_cpu_usage + expire_ms = 7 * 24 * 60 * 60 * 1000 export_redis_metrics( artifact_version, end_time_ms, - overall_commandstats_metrics, + overall_end_time_metrics, rts, setup_name, setup_type, @@ -822,30 +800,71 @@ def run_remote_command_logic(args, project_name, project_version): tf_github_org, tf_github_repo, tf_triggering_env, - {"metric-type": "commandstats"}, + {"metric-type": "redis-metrics"}, expire_ms, ) - ( - end_time_ms, - _, - overall_commandstats_metrics, - ) = collect_redis_metrics( - redis_conns, ["latencystats"] + if collect_commandstats: + ( + end_time_ms, + _, + overall_commandstats_metrics, + ) = collect_redis_metrics( + redis_conns, ["commandstats"] + ) + export_redis_metrics( + artifact_version, + end_time_ms, + overall_commandstats_metrics, + rts, + setup_name, + setup_type, + test_name, + tf_github_branch, + tf_github_org, + tf_github_repo, + tf_triggering_env, + {"metric-type": "commandstats"}, + expire_ms, + ) + ( + end_time_ms, + _, + overall_commandstats_metrics, + ) = collect_redis_metrics( + redis_conns, ["latencystats"] + ) + export_redis_metrics( + artifact_version, + end_time_ms, + overall_commandstats_metrics, + rts, + setup_name, + setup_type, + test_name, + tf_github_branch, + tf_github_org, + tf_github_repo, + tf_triggering_env, + {"metric-type": "latencystats"}, + expire_ms, + ) + except redis.exceptions.ConnectionError as e: + db_error_artifacts( + db_ssh_port, + dirname, + full_logfiles, + logname, + private_key, + s3_bucket_name, + s3_bucket_path, + server_public_ip, + temporary_dir, + args.upload_results_s3, + username, ) - export_redis_metrics( - artifact_version, - end_time_ms, - overall_commandstats_metrics, - rts, - setup_name, - setup_type, - test_name, - tf_github_branch, - tf_github_org, - tf_github_repo, - tf_triggering_env, - {"metric-type": "latencystats"}, - expire_ms, + return_code |= 1 + raise Exception( + "Failed to run remote benchmark." ) if setup_details["env"] is None: