diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index da8b620..ef373d4 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -11,6 +11,7 @@ jobs: strategy: matrix: python-version: [ '3.10', '3.11', '3.12' ] + os: [ "ubuntu-latest", "ubuntu-24.04" ] fail-fast: false env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true diff --git a/pyproject.toml b/pyproject.toml index d8c19dd..8805bd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.235" +version = "0.1.240" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__cli__/args.py b/redis_benchmarks_specification/__cli__/args.py index b685455..d2ff724 100644 --- a/redis_benchmarks_specification/__cli__/args.py +++ b/redis_benchmarks_specification/__cli__/args.py @@ -5,7 +5,7 @@ # import datetime import os - +from distutils.util import strtobool from redis_benchmarks_specification.__common__.env import ( GH_REDIS_SERVER_HOST, GH_TOKEN, @@ -71,6 +71,12 @@ def spec_cli_args(parser): action="store_true", help="Include modules statistics on commandstats.", ) + parser.add_argument( + "--use-git-timestamp", + type=lambda x: bool(strtobool(x)), + default=True, + help="Use git timestamp", + ) parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN) parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="") parser.add_argument( diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index 906ccbc..829cf37 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -445,6 +445,15 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): git_branch = cdict["git_branch"] commit_datetime = cdict["commit_datetime"] commit_summary = cdict["commit_summary"] + reply_fields = {} + use_git_timestamp = args.use_git_timestamp + if use_git_timestamp is False: + reply_fields["use_git_timestamp"] = str(use_git_timestamp) + + logging.info( + f"Setting use use_git_timestamp={use_git_timestamp}. ({args.use_git_timestamp})" + ) + if result is True: stream_id = "n/a" if args.dry_run is False: @@ -455,7 +464,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): ) = request_build_from_commit_info( conn, commit_dict, - {}, + reply_fields, binary_key, binary_value, REDIS_BINS_EXPIRE_SECS, diff --git a/redis_benchmarks_specification/__common__/github.py b/redis_benchmarks_specification/__common__/github.py index 01d9c81..52cb9af 100644 --- a/redis_benchmarks_specification/__common__/github.py +++ b/redis_benchmarks_specification/__common__/github.py @@ -107,32 +107,37 @@ def check_github_available_and_actionable( logging.info("Detected github token") g = Github(github_token) if pull_request is not None and pull_request != "": - pull_request_n = int(pull_request) - github_pr = ( - g.get_user(tf_github_org) - .get_repo(tf_github_repo) - .get_issue(pull_request_n) - ) - comments = github_pr.get_comments() - pr_link = github_pr.html_url - logging.info("Working on github PR already: {}".format(pr_link)) - is_actionable_pr = True - contains_regression_comment, pos = fn(comments) - if contains_regression_comment: - regression_comment = comments[pos] - old_regression_comment_body = regression_comment.body - logging.info( - "Already contains PR comment. Link: {}".format( - regression_comment.html_url + try: + pull_request_n = int(pull_request) + github_pr = ( + g.get_user(tf_github_org) + .get_repo(tf_github_repo) + .get_issue(pull_request_n) + ) + comments = github_pr.get_comments() + pr_link = github_pr.html_url + logging.info("Working on github PR already: {}".format(pr_link)) + is_actionable_pr = True + contains_regression_comment, pos = fn(comments) + if contains_regression_comment: + regression_comment = comments[pos] + old_regression_comment_body = regression_comment.body + logging.info( + "Already contains PR comment. Link: {}".format( + regression_comment.html_url + ) ) + if verbose: + logging.info("Printing old PR comment:") + print("".join(["-" for x in range(1, 80)])) + print(regression_comment.body) + print("".join(["-" for x in range(1, 80)])) + else: + logging.info("Does not contain PR comment") + except Exception as e: + logging.error( + f"an error occured when checking github info. {e.__str__()}. proceeding..." ) - if verbose: - logging.info("Printing old PR comment:") - print("".join(["-" for x in range(1, 80)])) - print(regression_comment.body) - print("".join(["-" for x in range(1, 80)])) - else: - logging.info("Does not contain PR comment") logging.info( f"contains_regression_comment: {contains_regression_comment}, is_actionable_pr: {is_actionable_pr}, pr_link: {pr_link}" ) diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index e69960c..399f4bc 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -664,6 +664,10 @@ def compute_regression_table( total_comparison_points, regressions_list, improvements_list, + unstable_list, + baseline_only_list, + comparison_only_list, + no_datapoints_list, ) = from_rts_to_regression_table( baseline_deployment_name, comparison_deployment_name, @@ -708,6 +712,29 @@ def compute_regression_table( baseline_deployment_name, ) + if total_unstable > 0: + old_stdout = sys.stdout + sys.stdout = mystdout = StringIO() + table_output += "#### Unstable Table\n\n" + writer_regressions = MarkdownTableWriter( + table_name="", + headers=[ + "Test Case", + f"Baseline {baseline_github_org}/{baseline_github_repo} {baseline_str} (median obs. +- std.dev)", + f"Comparison {comparison_github_org}/{comparison_github_repo} {comparison_str} (median obs. +- std.dev)", + "% change ({})".format(metric_mode), + "Note", + ], + value_matrix=table_unstable, + ) + writer_regressions.dump(mystdout, False) + table_output += mystdout.getvalue() + table_output += "\n\n" + test_names_str = "|".join([l[0] for l in unstable_list]) + table_output += f"Unstable test regexp names: {test_names_str}\n\n" + mystdout.close() + sys.stdout = old_stdout + if total_regressions > 0: old_stdout = sys.stdout sys.stdout = mystdout = StringIO() @@ -774,6 +801,27 @@ def compute_regression_table( sys.stdout = old_stdout table_output += mystdout.getvalue() table_output += "\n\n" + len_baseline_only_list = len(baseline_only_list) + if len_baseline_only_list > 0: + table_output += f"\n WARNING: There were {len_baseline_only_list} benchmarks with datapoints only on baseline.\n\n" + baseline_only_test_names_str = "|".join([l[0] for l in baseline_only_list]) + table_output += ( + f" Baseline only test regexp names: {baseline_only_test_names_str}\n\n" + ) + len_comparison_only_list = len(comparison_only_list) + if len_comparison_only_list > 0: + table_output += f"\n WARNING: There were {len_comparison_only_list} benchmarks with datapoints only on comparison.\n\n" + comparison_only_test_names_str = "|".join([l[0] for l in comparison_only_list]) + table_output += ( + f" Comparison only test regexp names: {comparison_only_test_names_str}\n\n" + ) + len_no_datapoints = len(no_datapoints_list) + if len_no_datapoints > 0: + table_output += f"\n WARNING: There were {len_no_datapoints} benchmarks with NO datapoints for both baseline and comparison.\n\n" + no_datapoints_test_names_str = "|".join([l[0] for l in no_datapoints_list]) + table_output += ( + f" NO DATAPOINTS test regexp names: {no_datapoints_test_names_str}\n\n" + ) return ( detected_regressions, @@ -967,6 +1015,10 @@ def from_rts_to_regression_table( progress = tqdm(unit="benchmark time-series", total=len(test_names)) regressions_list = [] improvements_list = [] + unstable_list = [] + baseline_only_list = [] + comparison_only_list = [] + no_datapoints_list = [] for test_name in test_names: compare_version = "main" github_link = "https://github.com/redis/redis-benchmarks-specification/blob" @@ -1110,6 +1162,17 @@ def from_rts_to_regression_table( logging.error("Detected a ZeroDivisionError. {}".format(e.__str__())) pass unstable = False + + if baseline_v != "N/A" and comparison_v == "N/A": + logging.warning( + "Baseline contains datapoints but comparison not for test: {test_name}" + ) + baseline_only_list.append(test_name) + if comparison_v != "N/A" and baseline_v == "N/A": + logging.warning( + "Comparison contains datapoints but baseline not for test: {test_name}" + ) + comparison_only_list.append(test_name) if ( baseline_v != "N/A" and comparison_pct_change != "N/A" @@ -1119,6 +1182,7 @@ def from_rts_to_regression_table( if comparison_pct_change > 10.0 or baseline_pct_change > 10.0: note = "UNSTABLE (very high variance)" unstable = True + unstable_list.append([test_name, "n/a"]) baseline_v_str = prepare_value_str( baseline_pct_change, baseline_v, baseline_values, simplify_table @@ -1212,6 +1276,21 @@ def from_rts_to_regression_table( if should_add_line: total_comparison_points = total_comparison_points + 1 table_full.append(line) + else: + logging.warning( + "There were no datapoints both for baseline and comparison for test: {test_name}" + ) + no_datapoints_list.append(test_name) + logging.warning( + f"There is a total of {len(no_datapoints_list)} tests without datapoints for baseline AND comparison" + ) + logging.info( + f"There is a total of {len(comparison_only_list)} tests without datapoints for baseline" + ) + logging.info( + f"There is a total of {len(baseline_only_list)} tests without datapoints for comparison" + ) + logging.info(f"There is a total of {len(unstable_list)} UNSTABLE tests") return ( detected_regressions, table_full, @@ -1226,6 +1305,10 @@ def from_rts_to_regression_table( total_comparison_points, regressions_list, improvements_list, + unstable_list, + baseline_only_list, + comparison_only_list, + no_datapoints_list, ) diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values.yml new file mode 100644 index 0000000..3c9c984 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values +description: Runs memtier_benchmark, for a keyspace length of 100K keys loading HASHES with 50 fields each. Each field value has a data size of 1000 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 6g +tested-groups: +- hash +tested-commands: +- hset +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --distinct-client-seed "--data-size" "1000" --command "HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__ field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__ field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14 __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__ field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23 __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__ field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32 __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__ field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41 __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__ field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 100000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 5 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values.yml new file mode 100644 index 0000000..56fa321 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values +description: Runs memtier_benchmark, for a keyspace length of 100K keys loading HASHES with 50 fields each. Each field value has a data size of 100 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 6g +tested-groups: +- hash +tested-commands: +- hset +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --distinct-client-seed "--data-size" "100" --command "HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__ field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__ field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14 __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__ field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23 __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__ field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32 __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__ field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41 __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__ field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 100000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 5 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-10B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-10B-values.yml new file mode 100644 index 0000000..e7ab9c4 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-10B-values.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-100Kkeys-load-hash-50-fields-with-10B-values +description: Runs memtier_benchmark, for a keyspace length of 100K keys loading HASHES with 50 fields each. Each field value has a data size of 10 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 6g +tested-groups: +- hash +tested-commands: +- hset +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --distinct-client-seed "--data-size" "10" --command "HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__ field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__ field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14 __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__ field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23 __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__ field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32 __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__ field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41 __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__ field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 100000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 5 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-10Kkeys-load-hash-50-fields-with-10000B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-10Kkeys-load-hash-50-fields-with-10000B-values.yml new file mode 100644 index 0000000..1dc2943 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-10Kkeys-load-hash-50-fields-with-10000B-values.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-10Kkeys-load-hash-50-fields-with-10000B-values +description: Runs memtier_benchmark, for a keyspace length of 100K keys loading HASHES with 50 fields each. Each field value has a data size of 1000 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 6g +tested-groups: +- hash +tested-commands: +- hset +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --distinct-client-seed "--data-size" "10000" --command "HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__ field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__ field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14 __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__ field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23 __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__ field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32 __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__ field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41 __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__ field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 10000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 5 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-hash-hscan-50-fields-10B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-hash-hscan-50-fields-10B-values.yml new file mode 100644 index 0000000..61f6045 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-hash-hscan-50-fields-10B-values.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1key-hash-hscan-50-fields-10B-values +description: Runs memtier_benchmark, for a keyspace length of 1 keys pre-loading HASHes in which the value has a data size of 10 Bytes. After pre-loading the data it issues HSCAN command. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" --command "HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__ field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__ field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14 __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__ field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23 __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__ field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32 __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__ field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41 __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__ field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50 __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 -n allkeys -c 1 -t 1 --hide-histogram' + resources: + requests: + memory: 2g +tested-groups: +- hash +tested-commands: +- hscan +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: ' --command "HSCAN __key__ 0" --command-key-pattern="R" --key-minimum=1 --key-maximum 1 --test-time 120 -c 50 -t 4 --hide-histogram' + resources: + requests: + cpus: '4' + memory: 2g + +priority: 96 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-lindex-integer.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-lindex-integer.yml new file mode 100644 index 0000000..cef6fb8 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-lindex-integer.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1key-list-10K-elements-lindex-integer +description: 'Runs memtier_benchmark, for a keyspace length of 1 LIST key. The LIST contains 10000 integer elements in it and we query it using LINDEX of a element that we know it is in the middle of the list. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --hide-histogram --command "RPUSH intlist __key__" --key-maximum 10000 --key-minimum 1 -n allkeys --key-prefix "" --command-key-pattern S -c 1 -t 1 --pipeline 10 + resources: + requests: + cpus: '2' + memory: 1g +tested-groups: +- list +tested-commands: +- lindex +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="LINDEX intlist 5000" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g +priority: 34 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-lindex-string.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-lindex-string.yml new file mode 100644 index 0000000..53a12f0 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-lindex-string.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1key-list-10K-elements-lindex-string +description: 'Runs memtier_benchmark, for a keyspace length of 1 LIST key. The LIST contains 10000 string elements in it and we query it using LPOS of a element that we know it is in the middle of the list. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --hide-histogram --command "RPUSH strlist __key__" --key-maximum 10000 --key-minimum 1 -n allkeys --key-prefix "hello" --command-key-pattern S -c 1 -t 1 --pipeline 10 + resources: + requests: + cpus: '2' + memory: 1g +tested-groups: +- list +tested-commands: +- lindex +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="LINDEX strlist 5000" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g +priority: 34 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-linsert-lrem-integer.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-linsert-lrem-integer.yml new file mode 100644 index 0000000..6480e00 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-linsert-lrem-integer.yml @@ -0,0 +1,35 @@ +version: 0.4 +name: memtier_benchmark-1key-list-10K-elements-linsert-lrem-integer +description: 'Runs memtier_benchmark, for a keyspace length of 1 LIST key. The LIST contains 10000 integer elements in it and we query it using LINSERT / LREM in the middle of the list. Given we add and delete right after the performance and list lenght over the test should be constant (meaning running for 60secs or 600secs wont change the list size) ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --hide-histogram --command "RPUSH intlist __key__" --key-maximum 10000 --key-minimum 1 -n allkeys --key-prefix "" --command-key-pattern S -c 1 -t 1 --pipeline 10 + resources: + requests: + cpus: '2' + memory: 1g +tested-groups: +- list +tested-commands: +- linsert +- lrem +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="LINSERT intlist AFTER 5000 5001" --command="LREM intlist 1 5001" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g +priority: 34 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-linsert-lrem-string.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-linsert-lrem-string.yml new file mode 100644 index 0000000..d666574 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-10K-elements-linsert-lrem-string.yml @@ -0,0 +1,35 @@ +version: 0.4 +name: memtier_benchmark-1key-list-10K-elements-linsert-lrem-string +description: 'Runs memtier_benchmark, for a keyspace length of 1 LIST key. The LIST contains 10000 string elements in it and we query it using LINSERT / LREM in the middle of the list. Given we add and delete right after the performance and list lenght over the test should be constant (meaning running for 60secs or 600secs wont change the list size) ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --hide-histogram --command "RPUSH strlist __key__" --key-maximum 10000 --key-minimum 1 -n allkeys --key-prefix "hello" --command-key-pattern S -c 1 -t 1 --pipeline 10 + resources: + requests: + cpus: '2' + memory: 1g +tested-groups: +- list +tested-commands: +- linsert +- lrem +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="LINSERT strlist AFTER hello5000 hello5000-new" --command="LREM strlist 1 hello5000-new" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g +priority: 34 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sscan.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sscan.yml new file mode 100644 index 0000000..7243515 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sscan.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-set-100-elements-sscan +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SMEMBERS that returns all elements. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"' +tested-groups: +- set +tested-commands: +- sscan +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SSCAN set:100 0" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 23 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-100-elements-zscan.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-100-elements-zscan.yml new file mode 100644 index 0000000..1e6033b --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-100-elements-zscan.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-zset-100-elements-zscan +description: 'Runs memtier_benchmark, for a keyspace length of 1 SORTED SET key. The SORTED SET contains 100 elements in it and we query it using ZRANGE BYSCORE with a range of all elements. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"ZADD" "zset:100" "0.306275" "lysbgqqfqw" "0.486004" "mtccjerdon" "0.941626" "jekkafodvk" "0.602656" "nmgxcctxpn" "0.630771" "vyqqkuszzh" "0.406379" "pytrnqdhvs" "0.521814" "oguwnmniig" "0.182854" "gekntrykfh" "0.657658" "nhfnbxqgol" "0.218066" "cgoeihlnei" "0.366328" "kgqrovsxce" "0.520723" "imyizmhzjk" "0.426630" "urvgxwbiiz" "0.878001" "onzjrteqmu" "0.126097" "alqlzsvuuz" "0.562698" "lgrkndhekf" "0.357487" "tcjvjofxtd" "0.395563" "ouuybhtvyb" "0.745796" "owbbdezgxn" "0.165356" "ksqpdywgdd" "0.757214" "qrosceblyo" "0.346153" "agsdggdghc" "0.297342" "gfdqdrondm" "0.668749" "oapbvnisyq" "0.281552" "htvbwmfyic" "0.013874" "vyoomgwuzv" "0.052023" "pcxdhdjwnf" "0.648518" "ibhohmfxzt" "0.967245" "erxulvebrj" "0.971791" "qwxrsgafzy" "0.154577" "whmotknaff" "0.404082" "ueyrvldzwl" "0.324634" "nsxaigrnje" "0.342459" "xsepfhdizi" "0.988925" "exvhmqxvvp" "0.345154" "owtkxlzaci" "0.728413" "ypfeltycpy" "0.406783" "qbiwbqiubb" "0.714527" "wqiwdbncmt" "0.760975" "dapacykoah" "0.212614" "hhjclfbbka" "0.445330" "ujrxcmpvsq" "0.576866" "lbtpbknelm" "0.764225" "wvcnqbvlnf" "0.281427" "mibvtmqxcy" "0.596978" "aovfsvbwjg" "0.650632" "qbyhorvill" "0.948063" "ypekeuutef" "0.028779" "xamjodnbpf" "0.678338" "hhniikmtsx" "0.757806" "sahqeskveq" "0.671949" "bqntlsaqjb" "0.190314" "luemuetmia" "0.219219" "zwnitejtpg" "0.212301" "ncjfckgkcl" "0.730613" "qjyeqcfhjj" "0.657992" "wkbasfyzqz" "0.556851" "ccagtnjilc" "0.543822" "pqyurbvifk" "0.101854" "djetcyfxuc" "0.878471" "vpmpffdoqz" "0.696931" "hlpdstpvzw" "0.662898" "imdqxmkzdj" "0.887214" "eouliovvra" "0.193298" "gxephxbdru" "0.978115" "dlsjwmqzhx" "0.743150" "uapsgmizgh" "0.181609" "gorgpnnqwr" "0.638726" "rojeolnopp" "0.561627" "mbxohpancs" "0.223001" "upodnpqenn" "0.957952" "dlbqcynhrn" "0.586659" "ichqzmiyot" "0.813623" "ulbrotlhze" "0.111318" "licotqplim" "0.525834" "mxwgdcutnb" "0.405858" "vpbkvwgxsf" "0.129768" "ijsmoyesvd" "0.591274" "mbgehjiauu" "0.718370" "vymjzlzqcn" "0.189343" "rjkknoigmu" "0.566307" "jxoxtnzujs" "0.563082" "eqlgkwosie" "0.040883" "ljcgdooafo" "0.326765" "cwaveajmcz" "0.423381" "ssjqrsluod" "0.865798" "fhuvzpxjbc" "0.784445" "perfwnpvkl" "0.659790" "mjjuylgssm" "0.918901" "vxcbagyymm" "0.282908" "nxcdcaqgit" "0.030110" "ewomnmugfa" "0.164237" "rkaznetutk" "0.281729" "rqvryfvlie" "0.877790" "holjcdpijr" "0.878293" "pquewclxuy" "0.674968" "ocrcszcznp" "0.520069" "xczfmrxrja" "0.314841" "nxzsnkmxvi" "0.719988" "hhjchwjlmc"' +tested-groups: +- sorted-set +tested-commands: +- zscan +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="ZSCAN zset:100 0" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 53