diff --git a/pyproject.toml b/pyproject.toml index a0813a7..046dd35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisbench-admin" -version = "0.10.19" +version = "0.10.24" description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )." authors = ["filipecosta90 ","Redis Performance Group "] readme = "README.md" diff --git a/redisbench_admin/compare/compare.py b/redisbench_admin/compare/compare.py index e1743f8..d18b357 100644 --- a/redisbench_admin/compare/compare.py +++ b/redisbench_admin/compare/compare.py @@ -722,6 +722,7 @@ def from_rts_to_regression_table( total_comparison_points = 0 noise_waterline = 3 progress = tqdm(unit="benchmark time-series", total=len(test_names)) + at_comparison = 0 for test_name in test_names: multi_value_baseline = check_multi_value_filter(baseline_str) multi_value_comparison = check_multi_value_filter(comparison_str) diff --git a/redisbench_admin/run/args.py b/redisbench_admin/run/args.py index 18c2e8e..7e2316c 100644 --- a/redisbench_admin/run/args.py +++ b/redisbench_admin/run/args.py @@ -40,9 +40,10 @@ PROFILERS = os.getenv("PROFILERS", PROFILERS_DEFAULT) MAX_PROFILERS_PER_TYPE = int(os.getenv("MAX_PROFILERS", 1)) PROFILE_FREQ = os.getenv("PROFILE_FREQ", PROFILE_FREQ_DEFAULT) -KEEP_ENV = bool(os.getenv("KEEP_ENV", False)) +KEEP_ENV = bool(int(os.getenv("KEEP_ENV", "0"))) ALLOWED_TOOLS_DEFAULT = "memtier_benchmark,redis-benchmark,redisgraph-benchmark-go,ycsb,go-ycsb,tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries,ftsb_redisearch,aibench_run_inference_redisai_vision,ann-benchmarks" ALLOWED_BENCH_TOOLS = os.getenv("ALLOWED_BENCH_TOOLS", ALLOWED_TOOLS_DEFAULT) +SKIP_DB_SETUP = bool(int(os.getenv("SKIP_DB_SETUP", "0"))) def common_run_args(parser): @@ -53,6 +54,12 @@ def common_run_args(parser): action="store_true", help="Keep environment and topology up after benchmark.", ) + parser.add_argument( + "--skip-db-setup", + type=bool, + default=SKIP_DB_SETUP, + help="skip db setup/teardown steps. Usefull when you want to target an existing DB", + ) parser.add_argument( "--fail_fast", required=False, diff --git a/redisbench_admin/run/common.py b/redisbench_admin/run/common.py index 28a6e72..f3d593d 100644 --- a/redisbench_admin/run/common.py +++ b/redisbench_admin/run/common.py @@ -699,6 +699,8 @@ def print_results_table_stdout( setup_name, test_name, cpu_usage=None, + kv_overall={}, + metric_names=[], ): # check which metrics to extract (_, metrics,) = merge_default_and_config_metrics( @@ -714,6 +716,11 @@ def print_results_table_stdout( results_matrix = extract_results_table(metrics, results_dict) if cpu_usage is not None: results_matrix.append(["Total shards CPU usage %", "", "", cpu_usage]) + for metric_name in metric_names: + if metric_name in kv_overall: + metric_value = kv_overall[metric_name] + results_matrix.append([f"Total shards {metric_name}", "", "", metric_value]) + results_matrix = [[x[0], "{:.3f}".format(x[3])] for x in results_matrix] writer = MarkdownTableWriter( table_name=table_name, diff --git a/redisbench_admin/run_async/async_terraform.py b/redisbench_admin/run_async/async_terraform.py index d7c9d06..6ff7c41 100644 --- a/redisbench_admin/run_async/async_terraform.py +++ b/redisbench_admin/run_async/async_terraform.py @@ -189,6 +189,10 @@ def setup_remote_environment( "github_repo": tf_github_repo, "triggering_env": tf_triggering_env, "timeout_secs": tf_timeout_secs, + "Project": tf_github_org, + "project": tf_github_org, + "Environment": tf_github_org, + "environment": tf_github_org, }, ) return self.retrieve_tf_connection_vars(return_code, tf) diff --git a/redisbench_admin/run_local/args.py b/redisbench_admin/run_local/args.py index 6b5153b..0a05c8b 100644 --- a/redisbench_admin/run_local/args.py +++ b/redisbench_admin/run_local/args.py @@ -3,13 +3,29 @@ # Copyright (c) 2021., Redis Labs Modules # All rights reserved. # +import os from redisbench_admin.run.args import common_run_args from redisbench_admin.run.common import REDIS_BINARY +FLUSHALL_AT_START = bool(int(os.getenv("FLUSHALL_AT_START", "0"))) +IGNORE_KEYSPACE_ERRORS = bool(int(os.getenv("IGNORE_KEYSPACE_ERRORS", "0"))) + def create_run_local_arguments(parser): parser = common_run_args(parser) parser.add_argument("--port", type=int, default=6379) parser.add_argument("--redis-binary", type=str, default=REDIS_BINARY) + parser.add_argument( + "--flushall_on_every_test_start", + type=bool, + default=FLUSHALL_AT_START, + help="At the start of every test send a FLUSHALL", + ) + parser.add_argument( + "--ignore_keyspace_errors", + type=bool, + default=IGNORE_KEYSPACE_ERRORS, + help="Ignore keyspace check errors. Will still log them as errors", + ) return parser diff --git a/redisbench_admin/run_local/local_db.py b/redisbench_admin/run_local/local_db.py index 7d3e78a..8d34d50 100644 --- a/redisbench_admin/run_local/local_db.py +++ b/redisbench_admin/run_local/local_db.py @@ -46,59 +46,16 @@ def local_db_spin( required_modules, setup_type, shard_count, + flushall_on_every_test_start=False, + ignore_keyspace_errors=False, ): - # setup Redis - # copy the rdb to DB machine + redis_conns = [] + artifact_version = "n/a" + result = True temporary_dir = tempfile.mkdtemp() - redis_7 = args.redis_7 - logging.info( - "Using local temporary dir to spin up Redis Instance. Path: {}".format( - temporary_dir - ) - ) - if dbdir_folder is not None: - from distutils.dir_util import copy_tree - - copy_tree(dbdir_folder, temporary_dir) - logging.info( - "Copied entire content of {} into temporary path: {}".format( - dbdir_folder, temporary_dir - ) - ) - ( - _, - _, - redis_configuration_parameters, - dataset_load_timeout_secs, - modules_configuration_parameters_map, - ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") cluster_api_enabled = False - logging.info( - "Using a dataset load timeout of {} seconds.".format(dataset_load_timeout_secs) - ) - redis_conns = [] if setup_type == "oss-cluster": cluster_api_enabled = True - shard_host = "127.0.0.1" - redis_processes, redis_conns = spin_up_local_redis_cluster( - binary, - temporary_dir, - shard_count, - shard_host, - args.port, - local_module_file, - redis_configuration_parameters, - dataset_load_timeout_secs, - modules_configuration_parameters_map, - redis_7, - ) - - status = setup_redis_cluster_from_conns( - redis_conns, shard_count, shard_host, args.port - ) - if status is False: - raise Exception("Redis cluster setup failed. Failing test.") - dataset, dataset_name, _, _ = check_dataset_local_requirements( benchmark_config, temporary_dir, @@ -108,35 +65,108 @@ def local_db_spin( shard_count, cluster_api_enabled, ) - if setup_type == "oss-standalone": - redis_processes = spin_up_local_redis( - binary, - args.port, - temporary_dir, - local_module_file, + + if args.skip_db_setup: + logging.info("Skipping DB Setup...") + if dataset is not None: + logging.info("Given this benchmark requires an RDB load will skip it...") + result = False + return ( + result, + artifact_version, + cluster_api_enabled, + redis_conns, + redis_processes, + ) + else: + # setup Redis + # copy the rdb to DB machine + redis_7 = args.redis_7 + logging.info( + "Using local temporary dir to spin up Redis Instance. Path: {}".format( + temporary_dir + ) + ) + if dbdir_folder is not None: + from distutils.dir_util import copy_tree + + copy_tree(dbdir_folder, temporary_dir) + logging.info( + "Copied entire content of {} into temporary path: {}".format( + dbdir_folder, temporary_dir + ) + ) + ( + _, + _, redis_configuration_parameters, - dbdir_folder, dataset_load_timeout_secs, modules_configuration_parameters_map, - redis_7, + ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") + + logging.info( + "Using a dataset load timeout of {} seconds.".format( + dataset_load_timeout_secs + ) ) + if setup_type == "oss-cluster": + cluster_api_enabled = True + shard_host = "127.0.0.1" + redis_processes, redis_conns = spin_up_local_redis_cluster( + binary, + temporary_dir, + shard_count, + shard_host, + args.port, + local_module_file, + redis_configuration_parameters, + dataset_load_timeout_secs, + modules_configuration_parameters_map, + redis_7, + ) + + status = setup_redis_cluster_from_conns( + redis_conns, shard_count, shard_host, args.port + ) + if status is False: + raise Exception("Redis cluster setup failed. Failing test.") + + if setup_type == "oss-standalone": + redis_processes = spin_up_local_redis( + binary, + args.port, + temporary_dir, + local_module_file, + redis_configuration_parameters, + dbdir_folder, + dataset_load_timeout_secs, + modules_configuration_parameters_map, + redis_7, + ) + if setup_type == "oss-cluster": + for shardn, redis_process in enumerate(redis_processes): + logging.info( + "Checking if shard #{} process with pid={} is alive".format( + shardn + 1, redis_process.pid + ) + ) + if is_process_alive(redis_process) is False: + raise Exception("Redis process is not alive. Failing test.") + cluster_init_steps(clusterconfig, redis_conns, local_module_file) + + if setup_type == "oss-standalone": r = redis.Redis(port=args.port) r.ping() - r.client_setname("redisbench-admin-stadalone") + r.client_setname("redisbench-admin-standalone") redis_conns.append(r) - if setup_type == "oss-cluster": - for shardn, redis_process in enumerate(redis_processes): - logging.info( - "Checking if shard #{} process with pid={} is alive".format( - shardn + 1, redis_process.pid - ) - ) - if is_process_alive(redis_process) is False: - raise Exception("Redis process is not alive. Failing test.") - if setup_type == "oss-cluster": - cluster_init_steps(clusterconfig, redis_conns, local_module_file) + if dataset is None: + if flushall_on_every_test_start: + logging.info("Will flush all data at test start...") + for shard_n, shard_conn in enumerate(redis_conns): + logging.info(f"Flushing all in shard {shard_n}...") + shard_conn.flushall() if check_dbconfig_tool_requirement(benchmark_config): logging.info("Detected the requirements to load data via client tool") @@ -175,11 +205,10 @@ def local_db_spin( ) ) - dbconfig_keyspacelen_check( - benchmark_config, - redis_conns, - ) + dbconfig_keyspacelen_check(benchmark_config, redis_conns, ignore_keyspace_errors) - run_redis_pre_steps(benchmark_config, redis_conns[0], required_modules) + artifact_version = run_redis_pre_steps( + benchmark_config, redis_conns[0], required_modules + ) - return cluster_api_enabled, redis_conns, redis_processes + return result, artifact_version, cluster_api_enabled, redis_conns, redis_processes diff --git a/redisbench_admin/run_local/run_local.py b/redisbench_admin/run_local/run_local.py index d2c5d9f..2b30a81 100644 --- a/redisbench_admin/run_local/run_local.py +++ b/redisbench_admin/run_local/run_local.py @@ -11,6 +11,7 @@ import datetime import traceback import redis +from redisbench_admin.run.git import git_vars_crosscheck import redisbench_admin.run.metrics from redisbench_admin.profilers.perf import PERF_CALLGRAPH_MODE @@ -28,9 +29,14 @@ ) from redisbench_admin.run.metrics import ( from_info_to_overall_shard_cpu, + collect_redis_metrics, collect_cpu_data, ) -from redisbench_admin.run.redistimeseries import datasink_profile_tabular_data + +from redisbench_admin.run.redistimeseries import ( + datasink_profile_tabular_data, + timeseries_test_sucess_flow, +) from redisbench_admin.run.run import ( calculate_client_tool_duration_and_check, define_benchmark_plan, @@ -49,6 +55,7 @@ from redisbench_admin.utils.benchmark_config import ( prepare_benchmark_definitions, results_dict_kpi_check, + get_metadata_tags, ) from redisbench_admin.utils.local import ( get_local_run_full_filename, @@ -67,20 +74,29 @@ def run_local_command_logic(args, project_name, project_version): project_name=project_name, project_version=project_version ) ) + tf_github_org = args.github_org + tf_github_actor = args.github_actor + tf_github_repo = args.github_repo + tf_github_sha = args.github_sha + tf_github_branch = args.github_branch + ( + github_actor, + github_branch, github_org_name, github_repo_name, github_sha, - github_actor, - github_branch, - github_branch_detached, - ) = extract_git_vars() + ) = git_vars_crosscheck( + tf_github_actor, tf_github_branch, tf_github_org, tf_github_repo, tf_github_sha + ) dbdir_folder = args.dbdir_folder os.path.abspath(".") required_modules = args.required_module profilers_enabled = args.enable_profilers s3_bucket_name = args.s3_bucket_name + flushall_on_every_test_start = args.flushall_on_every_test_start + ignore_keyspace_errors = args.ignore_keyspace_errors profilers_list = [] if profilers_enabled: profilers_list = args.profilers.split(",") @@ -125,7 +141,7 @@ def run_local_command_logic(args, project_name, project_version): _, benchmark_definitions, default_metrics, - _, + exporter_timemetric_path, default_specs, clusterconfig, ) = prepare_benchmark_definitions(args) @@ -189,6 +205,8 @@ def run_local_command_logic(args, project_name, project_version): if " " in binary: binary = binary.split(" ") ( + result_db_spin, + artifact_version, cluster_api_enabled, redis_conns, redis_processes, @@ -204,7 +222,14 @@ def run_local_command_logic(args, project_name, project_version): required_modules, setup_type, shard_count, + flushall_on_every_test_start, + ignore_keyspace_errors, ) + if result_db_spin is False: + logging.warning( + "Skipping this test given DB spin stage failed..." + ) + continue if benchmark_type == "read-only": logging.info( "Given the benchmark for this setup is ready-only we will prepare to reuse it on the next read-only benchmarks (if any )." @@ -352,6 +377,22 @@ def run_local_command_logic(args, project_name, project_version): s3_bucket_name, test_name, ) + + ( + end_time_ms, + _, + overall_end_time_metrics, + ) = collect_redis_metrics( + redis_conns, + ["memory"], + { + "memory": [ + "used_memory", + "used_memory_dataset", + ] + }, + ) + if ( profilers_enabled and args.push_results_redistimeseries @@ -377,7 +418,7 @@ def run_local_command_logic(args, project_name, project_version): start_time_str, stdout, ) - + results_dict = {} with open( local_benchmark_output_filename, "r" ) as json_file: @@ -389,12 +430,42 @@ def run_local_command_logic(args, project_name, project_version): setup_name, test_name, total_shards_cpu_usage, + overall_end_time_metrics, + [ + "memory_used_memory", + "memory_used_memory_dataset", + ], ) # check KPIs return_code = results_dict_kpi_check( benchmark_config, results_dict, return_code ) + + metadata_tags = get_metadata_tags(benchmark_config) + ( + _, + branch_target_tables, + ) = timeseries_test_sucess_flow( + args.push_results_redistimeseries, + artifact_version, + benchmark_config, + benchmark_duration_seconds, + 0, + default_metrics, + setup_name, + setup_type, + exporter_timemetric_path, + results_dict, + rts, + start_time_ms, + test_name, + github_branch, + github_org_name, + github_repo_name, + tf_triggering_env, + ) + if setup_details["env"] is None: if args.keep_env_and_topo is False: for conn in redis_conns: diff --git a/redisbench_admin/run_remote/args.py b/redisbench_admin/run_remote/args.py index 1da36f2..0b22382 100644 --- a/redisbench_admin/run_remote/args.py +++ b/redisbench_admin/run_remote/args.py @@ -21,7 +21,6 @@ REMOTE_DB_PORT = int(os.getenv("REMOTE_DB_PORT", "6379")) REMOTE_DB_PASS = os.getenv("REMOTE_DB_PASS", None) REMOTE_PRIVATE_KEYNAME = os.getenv("REMOTE_PRIVATE_KEYNAME", DEFAULT_PRIVATE_KEY) -REMOTE_SKIP_DB_SETUP = bool(int(os.getenv("REMOTE_SKIP_DB_SETUP", "0"))) FLUSHALL_AT_START = bool(int(os.getenv("FLUSHALL_AT_START", "0"))) FLUSHALL_AT_END = bool(int(os.getenv("FLUSHALL_AT_END", "0"))) IGNORE_KEYSPACE_ERRORS = bool(int(os.getenv("IGNORE_KEYSPACE_ERRORS", "0"))) @@ -60,12 +59,6 @@ def create_run_remote_arguments(parser): ) parser.add_argument("--db_port", type=int, default=REMOTE_DB_PORT) parser.add_argument("--db_pass", type=str, default=REMOTE_DB_PASS) - parser.add_argument( - "--skip-db-setup", - type=bool, - default=REMOTE_SKIP_DB_SETUP, - help="skip db setup/teardown steps. Usefull when you want to target an existing DB", - ) parser.add_argument( "--flushall_on_every_test_start", type=bool, diff --git a/redisbench_admin/utils/remote.py b/redisbench_admin/utils/remote.py index 5911b9b..47e5683 100644 --- a/redisbench_admin/utils/remote.py +++ b/redisbench_admin/utils/remote.py @@ -30,7 +30,7 @@ ) # environment variables -PERFORMANCE_RTS_PUSH = bool(os.getenv("PUSH_RTS", False)) +PERFORMANCE_RTS_PUSH = bool(int(os.getenv("PUSH_RTS", "0"))) PERFORMANCE_RTS_AUTH = os.getenv("PERFORMANCE_RTS_AUTH", None) PERFORMANCE_RTS_USER = os.getenv("PERFORMANCE_RTS_USER", None) PERFORMANCE_RTS_HOST = os.getenv("PERFORMANCE_RTS_HOST", "localhost") @@ -51,7 +51,7 @@ def get_git_root(path): def view_bar_simple(a, b): res = a / int(b) * 100 - sys.stdout.write("\r Complete precent: %.2f %%" % res) + sys.stdout.write("\r Complete percent: %.2f %%" % res) sys.stdout.flush() @@ -296,6 +296,10 @@ def setup_remote_environment( "github_actor": tf_github_actor, "setup_name": tf_setup_name, "github_org": tf_github_org, + "Project": tf_github_org, + "project": tf_github_org, + "Environment": tf_github_org, + "environment": tf_github_org, "github_repo": tf_github_repo, "triggering_env": tf_triggering_env, "timeout_secs": tf_timeout_secs, @@ -969,6 +973,10 @@ def get_project_ts_tags( tags = { "github_org": tf_github_org, "github_repo": tf_github_repo, + "Project": tf_github_org, + "project": tf_github_org, + "Environment": tf_github_org, + "environment": tf_github_org, "github_org/github_repo": "{}/{}".format(tf_github_org, tf_github_repo), "deployment_type": deployment_type, "deployment_name": deployment_name,