From f221c4825322a8a14c7da250a9f85557da4a83d5 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 20 Aug 2024 01:22:25 +0100 Subject: [PATCH 01/12] Enabled running forks source built benchmarks --- pyproject.toml | 2 +- .../__builder__/builder.py | 35 +++ .../__cli__/args.py | 15 + redis_benchmarks_specification/__cli__/cli.py | 16 ++ .../__common__/runner.py | 2 +- .../self_contained_coordinator.py | 5 + utils/tests/test_builder.py | 256 ++++++++++++++++++ utils/tests/test_data/api_builder_common.py | 34 ++- ...test_self_contained_coordinator_memtier.py | 196 +++++++++++++- 9 files changed, 549 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 39e20f8..8e1b547 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.216" +version = "0.1.217" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index 678a71d..70c0132 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -234,6 +234,7 @@ def builder_process_stream( if b"git_hash" in testDetails: git_hash = testDetails[b"git_hash"] logging.info("Received commit hash specifier {}.".format(git_hash)) + logging.info(f"Received the following build stream: {testDetails}.") binary_zip_key = testDetails[b"zip_archive_key"] logging.info( "Retriving zipped source from key {}.".format( @@ -271,6 +272,16 @@ def builder_process_stream( if b"tests_groups_regexp" in testDetails: tests_groups_regexp = testDetails[b"tests_groups_regexp"].decode() + github_org = "redis" + if b"github_org" in testDetails: + github_org = testDetails[b"github_org"].decode() + logging.info(f"detected github_org info on build stream {github_org}") + + github_repo = "redis" + if b"github_repo" in testDetails: + github_repo = testDetails[b"github_repo"].decode() + logging.info(f"detected github_repo info on build stream {github_repo}") + # github updates is_actionable_pr = False contains_regression_comment = False @@ -321,6 +332,14 @@ def builder_process_stream( build_artifacts = ["redis-server"] if "build_artifacts" in build_config: build_artifacts = build_config["build_artifacts"] + if b"build_artifacts" in testDetails: + new_build_artifacts = ( + testDetails[b"build_artifacts"].decode().split(",") + ) + logging.info( + f"overriding default build artifacts {build_artifacts} by {new_build_artifacts}" + ) + build_artifacts = new_build_artifacts build_vars_str = "" if "env" in build_config: if build_config["env"] is not None: @@ -361,6 +380,12 @@ def builder_process_stream( "redis-server", build_vars_str, ) + if b"build_command" in testDetails: + build_command = testDetails[b"build_command"].decode() + server_name = "redis" + if b"server_name" in testDetails: + server_name = testDetails[b"server_name"].decode() + build_start_datetime = datetime.datetime.utcnow() logging.info( "Using the following build command {}.".format(build_command) @@ -435,6 +460,9 @@ def builder_process_stream( tests_priority_upper_limit, tests_regexp, use_git_timestamp, + server_name, + github_org, + github_repo, ) if result is True: benchmark_stream_id = conn.xadd( @@ -572,6 +600,9 @@ def generate_benchmark_stream_request( tests_priority_upper_limit=10000, tests_regexp=".*", use_git_timestamp=False, + server_name="redis", + github_org="redis", + github_repo="redis", ): build_stream_fields = { "id": id, @@ -584,6 +615,9 @@ def generate_benchmark_stream_request( "tests_priority_upper_limit": tests_priority_upper_limit, "tests_priority_lower_limit": tests_priority_lower_limit, "tests_groups_regexp": tests_groups_regexp, + "server_name": server_name, + "github_org": github_org, + "github_repo": github_repo, } if build_config_metadata is not None: build_stream_fields["metadata"] = json.dumps(build_config_metadata) @@ -594,6 +628,7 @@ def generate_benchmark_stream_request( if build_vars_str is not None: build_stream_fields["build_vars"] = build_vars_str if build_command is not None: + logging.info(f"adding build_command: {build_command}") build_stream_fields["build_command"] = build_command if build_image is not None: build_stream_fields["build_image"] = build_image diff --git a/redis_benchmarks_specification/__cli__/args.py b/redis_benchmarks_specification/__cli__/args.py index 44f78e9..b685455 100644 --- a/redis_benchmarks_specification/__cli__/args.py +++ b/redis_benchmarks_specification/__cli__/args.py @@ -166,6 +166,21 @@ def spec_cli_args(parser): action="store_true", help="Iterate over the git commits.", ) + parser.add_argument( + "--build_artifacts", + type=str, + default="", + ) + parser.add_argument( + "--build_command", + type=str, + default="", + ) + parser.add_argument( + "--git_hash", + type=str, + default="", + ) parser.add_argument( "--dry-run", default=False, diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index 2baddd4..f4ef0e7 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -355,6 +355,14 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): filtered_hash_commits = [] for cdict in commits: commit_hash = cdict["git_hash"] + if args.git_hash != "": + if args.git_hash != commit_hash: + logging.info( + "Skipping {} given it does not match commit hash {}".format( + commit_hash, args.git_hash + ) + ) + continue commit_summary = cdict["commit_summary"] commit_datetime = cdict["commit_datetime"] match_obj = re.search(hash_regexp_string, commit_hash) @@ -412,6 +420,14 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): commit_dict["tests_priority_lower_limit"] = tests_priority_lower_limit commit_dict["tests_regexp"] = tests_regexp commit_dict["tests_groups_regexp"] = tests_groups_regexp + commit_dict["github_org"] = args.gh_org + commit_dict["github_repo"] = args.gh_repo + if args.server_name != "": + commit_dict["server_name"] = args.server_name + if args.build_artifacts != "": + commit_dict["build_artifacts"] = args.build_artifacts + if args.build_command != "": + commit_dict["build_command"] = args.build_command if pull_request is not None: logging.info( f"Have a pull request info to include in build request {pull_request}" diff --git a/redis_benchmarks_specification/__common__/runner.py b/redis_benchmarks_specification/__common__/runner.py index 39f3046..64667e2 100644 --- a/redis_benchmarks_specification/__common__/runner.py +++ b/redis_benchmarks_specification/__common__/runner.py @@ -150,7 +150,7 @@ def exporter_datasink_common( git_hash=None, ): logging.info( - f"Using datapoint_time_ms: {datapoint_time_ms}. git_has={git_hash}, git_branch={git_branch}, git_version={git_version}" + f"Using datapoint_time_ms: {datapoint_time_ms}. git_hash={git_hash}, git_branch={git_branch}, git_version={git_version}. gh_org={tf_github_org}, gh_repo={tf_github_repo}" ) timeseries_test_sucess_flow( datasink_push_results_redistimeseries, diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 42d77bf..50a985b 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -547,6 +547,11 @@ def process_self_contained_coordinator_stream( logging.info( f"detected a server_name definition on the streamdata: {server_name}." ) + new_executable = f"{mnt_point}{server_name}-server" + logging.info( + "changing executable from {executable} to {new_executable}" + ) + executable = new_executable if b"restore_build_artifacts" in testDetails: restore_build_artifacts = bool( diff --git a/utils/tests/test_builder.py b/utils/tests/test_builder.py index 9f826fb..4d70035 100644 --- a/utils/tests/test_builder.py +++ b/utils/tests/test_builder.py @@ -4,7 +4,13 @@ # All rights reserved. # import os +import logging +from pathlib import Path +import docker + +from redis_benchmarks_specification.__cli__.args import spec_cli_args +from redis_benchmarks_specification.__cli__.cli import trigger_tests_cli_command_logic from redis_benchmarks_specification.__common__.builder_schema import ( commit_schema_to_stream, get_branch_version_from_test_details, @@ -20,6 +26,17 @@ STREAM_KEYNAME_GH_EVENTS_COMMIT, STREAM_KEYNAME_NEW_BUILD_EVENTS, ) +from redis_benchmarks_specification.__common__.timeseries import ( + get_ts_metric_name, + get_overall_dashboard_keynames, +) +from redis_benchmarks_specification.__self_contained_coordinator__.runners import ( + build_runners_consumer_group_create, +) +from redis_benchmarks_specification.__self_contained_coordinator__.self_contained_coordinator import ( + self_contained_coordinator_blocking_read, +) +from redis_benchmarks_specification.__setups__.topologies import get_topologies def test_build_spec_image_prefetch(): @@ -133,3 +150,242 @@ def test_get_branch_version_from_test_details(): testDetails = {b"git_version": "555.555.555"} _, git_version = get_branch_version_from_test_details(testDetails) assert git_version == "555.555.555" + + +def test_cli_build(): + try: + # if should_run_builder(): + if True: + + db_port = int(os.getenv("DATASINK_PORT", "6379")) + conn = redis.StrictRedis(port=db_port) + conn.ping() + conn.flushall() + + builder_consumer_group_create(conn, "0") + + import argparse + + run_image = "debian:buster" + github_org = "valkey-io" + github_repo = "valkey" + git_hash = "7795152fff06f8200f5e4239ff612b240f638e14" + git_branch = "unstable" + + # should error due to missing --use-tags or --use-branch + parser = argparse.ArgumentParser( + description="test", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser = spec_cli_args(parser) + run_args = [ + "--docker-dont-air-gap", + "--server_name", + "valkey", + "--run_image", + run_image, + "--gh_org", + github_org, + "--gh_repo", + github_repo, + "--redis_port", + "{}".format(db_port), + "--use-branch", + "--build_artifacts", + "valkey-server", + "--build_command", + "sh -c 'make -j'", + "--git_hash", + git_hash, + "--branch", + git_branch, + ] + logging.info("running with args: {}".format(" ".join(run_args))) + args = parser.parse_args( + args=run_args, + ) + try: + trigger_tests_cli_command_logic(args, "tool", "v0") + except SystemExit as e: + assert e.code == 0 + assert STREAM_KEYNAME_GH_EVENTS_COMMIT.encode() in conn.keys() + events_in_pipe = conn.xlen(STREAM_KEYNAME_GH_EVENTS_COMMIT) + assert events_in_pipe > 0 + builders_folder = "./redis_benchmarks_specification/setups/builders" + different_build_specs = ["gcc:8.5.0-amd64-debian-buster-default.yml"] + previous_id = ">" + previous_id, new_builds_count, _ = builder_process_stream( + builders_folder, conn, different_build_specs, previous_id + ) + assert new_builds_count == 1 + assert conn.exists(STREAM_KEYNAME_NEW_BUILD_EVENTS) + + running_platform = "fco-ThinkPad-T490" + + build_runners_consumer_group_create(conn, running_platform, "0") + datasink_conn = redis.StrictRedis(port=db_port) + docker_client = docker.from_env() + home = str(Path.home()) + stream_id = ">" + topologies_map = get_topologies( + "./redis_benchmarks_specification/setups/topologies/topologies.yml" + ) + # we use a benchmark spec with smaller CPU limit for client given github machines only contain 2 cores + # and we need 1 core for DB and another for CLIENT + testsuite_spec_files = [ + "./utils/tests/test_data/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case.yml" + ] + ( + result, + stream_id, + number_processed_streams, + _, + ) = self_contained_coordinator_blocking_read( + conn, + True, + docker_client, + home, + stream_id, + datasink_conn, + testsuite_spec_files, + topologies_map, + running_platform, + False, + [], + "", + 0, + 6399, + 1, + False, + 1, + None, + "amd64", + None, + 0, + 10000, + "unstable", + "", + True, + ) + assert result == True + assert number_processed_streams == 1 + + test_name = "memtier_benchmark-1Mkeys-100B-expire-use-case" + tf_triggering_env = "ci" + deployment_name = "oss-standalone" + deployment_type = "oss-standalone" + use_metric_context_path = False + metric_context_path = None + gh_org = github_org + gh_repo = github_repo + build_variant_name = "gcc:8.5.0-amd64-debian-buster-default" + for metric_name in ["ALL_STATS.Totals.Latency", "ALL_STATS.Totals.Ops/sec"]: + ts_key_name = get_ts_metric_name( + "by.branch", + "unstable", + gh_org, + gh_repo, + deployment_name, + deployment_type, + test_name, + tf_triggering_env, + metric_name, + metric_context_path, + use_metric_context_path, + build_variant_name, + running_platform, + ) + rts = datasink_conn.ts() + assert ts_key_name.encode() in conn.keys() + assert len(rts.range(ts_key_name, 0, "+")) == 1 + ( + prefix, + testcases_setname, + deployment_name_setname, + tsname_project_total_failures, + tsname_project_total_success, + running_platforms_setname, + build_variant_setname, + testcases_metric_context_path_setname, + testcases_and_metric_context_path_setname, + project_archs_setname, + project_oss_setname, + project_branches_setname, + project_versions_setname, + project_compilers_setname, + ) = get_overall_dashboard_keynames( + gh_org, + gh_repo, + tf_triggering_env, + build_variant_name, + running_platform, + test_name, + ) + + assert datasink_conn.exists(testcases_setname) + assert datasink_conn.exists(running_platforms_setname) + assert datasink_conn.exists(build_variant_setname) + assert datasink_conn.exists(testcases_and_metric_context_path_setname) + assert datasink_conn.exists(testcases_metric_context_path_setname) + assert build_variant_name.encode() in datasink_conn.smembers( + build_variant_setname + ) + assert test_name.encode() in datasink_conn.smembers(testcases_setname) + assert running_platform.encode() in datasink_conn.smembers( + running_platforms_setname + ) + testcases_and_metric_context_path_members = [ + x.decode() + for x in datasink_conn.smembers( + testcases_and_metric_context_path_setname + ) + ] + metric_context_path_members = [ + x.decode() + for x in datasink_conn.smembers(testcases_metric_context_path_setname) + ] + assert len(testcases_and_metric_context_path_members) == len( + metric_context_path_members + ) + + assert [x.decode() for x in datasink_conn.smembers(testcases_setname)] == [ + test_name + ] + + assert "amd64".encode() in datasink_conn.smembers(project_archs_setname) + assert "debian-buster".encode() in datasink_conn.smembers( + project_oss_setname + ) + assert "gcc".encode() in datasink_conn.smembers(project_compilers_setname) + assert build_variant_name.encode() in datasink_conn.smembers( + build_variant_setname + ) + assert running_platform.encode() in datasink_conn.smembers( + running_platforms_setname + ) + + assert len(datasink_conn.smembers(project_archs_setname)) == 1 + assert len(datasink_conn.smembers(project_oss_setname)) == 1 + assert len(datasink_conn.smembers(project_compilers_setname)) == 1 + assert len(datasink_conn.smembers(build_variant_setname)) == 1 + assert len(datasink_conn.smembers(running_platforms_setname)) == 1 + assert len(datasink_conn.smembers(testcases_setname)) == 1 + assert len(datasink_conn.smembers(project_branches_setname)) == 1 + assert len(datasink_conn.smembers(project_versions_setname)) == 1 + + # get all keys + all_keys = datasink_conn.keys("*") + by_hash_keys = [] + for key in all_keys: + if "/by.hash/" in key.decode(): + by_hash_keys.append(key) + + # ensure we have by hash keys + assert len(by_hash_keys) > 0 + for hash_key in by_hash_keys: + # ensure we have both version and hash info on the key + assert "branch" in rts.info(hash_key).labels + assert "hash" in rts.info(hash_key).labels + + except redis.exceptions.ConnectionError: + pass diff --git a/utils/tests/test_data/api_builder_common.py b/utils/tests/test_data/api_builder_common.py index 2017930..b43333b 100644 --- a/utils/tests/test_data/api_builder_common.py +++ b/utils/tests/test_data/api_builder_common.py @@ -13,19 +13,37 @@ def flow_1_and_2_api_builder_checks( - conn, build_spec_name="gcc:8.5.0-amd64-debian-buster-default" + conn, + build_spec_name="gcc:8.5.0-amd64-debian-buster-default", + gh_org="redis", + gh_repo="redis", + git_hash="0cf2df84d4b27af4bffd2bf3543838f09e10f874", + git_branch="unstable", + build_command=None, + build_artifacts=None, + server_name=None, ): builder_consumer_group_create(conn) assert conn.xlen(STREAM_KEYNAME_GH_EVENTS_COMMIT) == 0 + fields = { + "git_hash": git_hash, + "git_branch": git_branch, + "github_org": gh_org, + "github_repo": gh_repo, + "use_git_timestamp": True, + } + if server_name is not None: + fields["server_name"] = server_name + if build_command is not None: + fields["build_command"] = build_command + if build_artifacts is not None: + fields["build_artifacts"] = ",".join(build_artifacts) + result, reply_fields, error_msg = commit_schema_to_stream( - { - "git_hash": "0cf2df84d4b27af4bffd2bf3543838f09e10f874", - "git_branch": "unstable", - "use_git_timestamp": True, - }, + fields, conn, - "redis", - "redis", + gh_org, + gh_repo, GH_TOKEN, ) diff --git a/utils/tests/test_self_contained_coordinator_memtier.py b/utils/tests/test_self_contained_coordinator_memtier.py index ec4794b..b66b179 100644 --- a/utils/tests/test_self_contained_coordinator_memtier.py +++ b/utils/tests/test_self_contained_coordinator_memtier.py @@ -102,8 +102,6 @@ def test_self_contained_coordinator_blocking_read(): ) assert result == True assert number_processed_streams == 1 - tf_github_org = "redis" - tf_github_repo = "redis" test_name = "memtier_benchmark-1Mkeys-100B-expire-use-case" tf_triggering_env = "ci" deployment_name = "oss-standalone" @@ -628,11 +626,16 @@ def test_self_contained_coordinator_dockerhub_valkey(): build_arch, testDetails, build_os, + [], + "sudo bash -c 'make -j'", ) build_stream_fields["github_repo"] = github_repo build_stream_fields["github_org"] = github_org build_stream_fields["server_name"] = github_repo build_stream_fields["mnt_point"] = "" + logging.info( + f"requesting stream with following info: {build_stream_fields}" + ) if result is True: benchmark_stream_id = conn.xadd( STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields @@ -1091,3 +1094,192 @@ def test_prepare_memtier_benchmark_parameters(): benchmark_command_str == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) + + +def test_self_contained_coordinator_blocking_read_valkey(): + try: + if run_coordinator_tests(): + db_port = int(os.getenv("DATASINK_PORT", "6379")) + conn = redis.StrictRedis(port=db_port) + conn.ping() + expected_datapoint_ts = None + conn.flushall() + gh_org = "valkey-io" + gh_repo = "valkey" + build_spec_name = "gcc:8.5.0-amd64-debian-buster-default" + git_hash = "7795152fff06f8200f5e4239ff612b240f638e14" + git_branch = "unstable" + build_artifacts = ["valkey-server"] + + build_variant_name, reply_fields = flow_1_and_2_api_builder_checks( + conn, + build_spec_name, + gh_org, + gh_repo, + git_hash, + git_branch, + "sh -c 'make -j'", + build_artifacts, + "valkey", + ) + if b"git_timestamp_ms" in reply_fields: + expected_datapoint_ts = int(reply_fields[b"git_timestamp_ms"].decode()) + if "git_timestamp_ms" in reply_fields: + expected_datapoint_ts = int(reply_fields["git_timestamp_ms"]) + + assert conn.exists(STREAM_KEYNAME_NEW_BUILD_EVENTS) + assert conn.xlen(STREAM_KEYNAME_NEW_BUILD_EVENTS) > 0 + running_platform = "fco-ThinkPad-T490" + + build_runners_consumer_group_create(conn, running_platform, "0") + datasink_conn = redis.StrictRedis(port=db_port) + docker_client = docker.from_env() + home = str(Path.home()) + stream_id = ">" + topologies_map = get_topologies( + "./redis_benchmarks_specification/setups/topologies/topologies.yml" + ) + # we use a benchmark spec with smaller CPU limit for client given github machines only contain 2 cores + # and we need 1 core for DB and another for CLIENT + testsuite_spec_files = [ + "./utils/tests/test_data/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case.yml" + ] + ( + result, + stream_id, + number_processed_streams, + _, + ) = self_contained_coordinator_blocking_read( + conn, + True, + docker_client, + home, + stream_id, + datasink_conn, + testsuite_spec_files, + topologies_map, + running_platform, + False, + [], + "", + 0, + 6399, + 1, + False, + 1, + None, + "amd64", + None, + 0, + 10000, + "unstable", + "", + True, + ) + assert result == True + assert number_processed_streams == 1 + + test_name = "memtier_benchmark-1Mkeys-100B-expire-use-case" + tf_triggering_env = "ci" + deployment_name = "oss-standalone" + deployment_type = "oss-standalone" + use_metric_context_path = False + metric_context_path = None + for metric_name in ["ALL_STATS.Totals.Latency", "ALL_STATS.Totals.Ops/sec"]: + ts_key_name = get_ts_metric_name( + "by.branch", + "unstable", + gh_org, + gh_repo, + deployment_name, + deployment_type, + test_name, + tf_triggering_env, + metric_name, + metric_context_path, + use_metric_context_path, + build_variant_name, + running_platform, + ) + rts = datasink_conn.ts() + assert ts_key_name.encode() in conn.keys() + assert len(rts.range(ts_key_name, 0, "+")) == 1 + if expected_datapoint_ts is not None: + assert rts.range(ts_key_name, 0, "+")[0][0] == expected_datapoint_ts + ( + prefix, + testcases_setname, + deployment_name_setname, + tsname_project_total_failures, + tsname_project_total_success, + running_platforms_setname, + build_variant_setname, + testcases_metric_context_path_setname, + testcases_and_metric_context_path_setname, + project_archs_setname, + project_oss_setname, + project_branches_setname, + project_versions_setname, + project_compilers_setname, + ) = get_overall_dashboard_keynames( + gh_org, + gh_repo, + tf_triggering_env, + build_variant_name, + running_platform, + test_name, + ) + + assert datasink_conn.exists(testcases_setname) + assert datasink_conn.exists(running_platforms_setname) + assert datasink_conn.exists(build_variant_setname) + assert datasink_conn.exists(testcases_and_metric_context_path_setname) + assert datasink_conn.exists(testcases_metric_context_path_setname) + assert build_variant_name.encode() in datasink_conn.smembers( + build_variant_setname + ) + assert test_name.encode() in datasink_conn.smembers(testcases_setname) + assert running_platform.encode() in datasink_conn.smembers( + running_platforms_setname + ) + testcases_and_metric_context_path_members = [ + x.decode() + for x in datasink_conn.smembers( + testcases_and_metric_context_path_setname + ) + ] + metric_context_path_members = [ + x.decode() + for x in datasink_conn.smembers(testcases_metric_context_path_setname) + ] + assert len(testcases_and_metric_context_path_members) == len( + metric_context_path_members + ) + + assert [x.decode() for x in datasink_conn.smembers(testcases_setname)] == [ + test_name + ] + + assert "amd64".encode() in datasink_conn.smembers(project_archs_setname) + assert "debian-buster".encode() in datasink_conn.smembers( + project_oss_setname + ) + assert "gcc".encode() in datasink_conn.smembers(project_compilers_setname) + assert build_variant_name.encode() in datasink_conn.smembers( + build_variant_setname + ) + assert running_platform.encode() in datasink_conn.smembers( + running_platforms_setname + ) + + assert len(datasink_conn.smembers(project_archs_setname)) == 1 + assert len(datasink_conn.smembers(project_oss_setname)) == 1 + assert len(datasink_conn.smembers(project_compilers_setname)) == 1 + assert len(datasink_conn.smembers(build_variant_setname)) == 1 + assert len(datasink_conn.smembers(running_platforms_setname)) == 1 + assert len(datasink_conn.smembers(testcases_setname)) == 1 + assert len(datasink_conn.smembers(project_branches_setname)) == 1 + assert len(datasink_conn.smembers(project_versions_setname)) == 1 + + except redis.exceptions.ConnectionError: + pass From c39fa0d9952f1c3aa5fa32bea1c34016c4e4f4d4 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 20 Aug 2024 01:31:35 +0100 Subject: [PATCH 02/12] Fixed server_name Null check --- redis_benchmarks_specification/__cli__/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index f4ef0e7..0b6eaf4 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -422,7 +422,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): commit_dict["tests_groups_regexp"] = tests_groups_regexp commit_dict["github_org"] = args.gh_org commit_dict["github_repo"] = args.gh_repo - if args.server_name != "": + if args.server_name is not None and args.server_name != "": commit_dict["server_name"] = args.server_name if args.build_artifacts != "": commit_dict["build_artifacts"] = args.build_artifacts From f188bc460d35b61304f767b5ba7efdee7e60ef18 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 20 Aug 2024 14:15:27 +0100 Subject: [PATCH 03/12] Enabled passing baseline/comparison hash and github_repo to ensure proper data filtering on compare. Removed refs/heads/ usage from builder --- pyproject.toml | 2 +- redis_benchmarks_specification/__cli__/cli.py | 16 +++-- .../__common__/builder_schema.py | 4 ++ .../__compare__/args.py | 8 +++ .../__compare__/compare.py | 64 ++++++++++++++++++- 5 files changed, 85 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8e1b547..2041577 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.217" +version = "0.1.218" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index 0b6eaf4..906ccbc 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -351,16 +351,22 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): hash_regexp ) ) + enable_hash_filtering = False + hash_filters = [] + if args.git_hash != "": + enable_hash_filtering = True + hash_filters = args.git_hash.split(",") + logging.info( + f"There is a total of {len(hash_filters)} commit hash fitlers: {hash_filters}" + ) hash_regexp_string = re.compile(hash_regexp) filtered_hash_commits = [] for cdict in commits: commit_hash = cdict["git_hash"] - if args.git_hash != "": - if args.git_hash != commit_hash: + if enable_hash_filtering: + if commit_hash not in hash_filters: logging.info( - "Skipping {} given it does not match commit hash {}".format( - commit_hash, args.git_hash - ) + f"Skipping {commit_hash} given it does not match any commit hash in {hash_filters}" ) continue commit_summary = cdict["commit_summary"] diff --git a/redis_benchmarks_specification/__common__/builder_schema.py b/redis_benchmarks_specification/__common__/builder_schema.py index 0f2fcd0..8c45edc 100644 --- a/redis_benchmarks_specification/__common__/builder_schema.py +++ b/redis_benchmarks_specification/__common__/builder_schema.py @@ -174,6 +174,10 @@ def get_branch_version_from_test_details(testDetails): git_branch = git_branch.decode() if git_branch.startswith("/refs/heads/"): git_branch = git_branch.replace("/refs/heads/", "") + if git_branch.startswith("refs/heads/"): + git_branch = git_branch.replace("refs/heads/", "") + if git_branch.startswith("/"): + git_branch = git_branch[1:] if git_version is not None: if type(git_version) == bytes: git_version = git_version.decode() diff --git a/redis_benchmarks_specification/__compare__/args.py b/redis_benchmarks_specification/__compare__/args.py index 7407096..c35c6e5 100644 --- a/redis_benchmarks_specification/__compare__/args.py +++ b/redis_benchmarks_specification/__compare__/args.py @@ -92,11 +92,19 @@ def create_compare_arguments(parser): ) parser.add_argument("--baseline-branch", type=str, default=None, required=False) parser.add_argument("--baseline-tag", type=str, default=None, required=False) + parser.add_argument("--baseline-hash", type=str, default=None, required=False) parser.add_argument( "--baseline-target-version", type=str, default=None, required=False ) parser.add_argument("--comparison-branch", type=str, default=None, required=False) + parser.add_argument( + "--baseline-github-repo", type=str, default="redis", required=False + ) + parser.add_argument( + "--comparison-github-repo", type=str, default="redis", required=False + ) parser.add_argument("--comparison-tag", type=str, default=None, required=False) + parser.add_argument("--comparison-hash", type=str, default=None, required=False) parser.add_argument( "--comparison-target-version", type=str, default=None, required=False ) diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index 001c162..af72d69 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -250,6 +250,10 @@ def compare_command_logic(args, project_name, project_version): running_platform = args.running_platform baseline_target_version = args.baseline_target_version comparison_target_version = args.comparison_target_version + baseline_github_repo = args.baseline_github_repo + comparison_github_repo = args.comparison_github_repo + baseline_hash = args.baseline_hash + comparison_hash = args.comparison_hash if running_platform is not None: logging.info( @@ -310,6 +314,10 @@ def compare_command_logic(args, project_name, project_version): running_platform, baseline_target_version, comparison_target_version, + baseline_hash, + comparison_hash, + baseline_github_repo, + comparison_github_repo, ) prepare_regression_comment( auto_approve, @@ -535,6 +543,10 @@ def compute_regression_table( running_platform=None, baseline_target_version=None, comparison_target_version=None, + comparison_hash=None, + baseline_hash=None, + baseline_github_repo="redis", + comparison_github_repo="redis", ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -560,6 +572,8 @@ def compute_regression_table( comparison_tag, baseline_target_version, comparison_target_version, + comparison_hash, + baseline_hash, ) logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}") logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}") @@ -605,6 +619,8 @@ def compute_regression_table( total_stable, total_unstable, total_comparison_points, + regressions_list, + improvements_list, ) = from_rts_to_regression_table( baseline_deployment_name, comparison_deployment_name, @@ -629,6 +645,8 @@ def compute_regression_table( tf_triggering_env, verbose, running_platform, + baseline_github_repo, + comparison_github_repo, ) logging.info( "Printing differential analysis between {} and {}".format( @@ -661,6 +679,8 @@ def compute_regression_table( writer_regressions.dump(mystdout, False) table_output += mystdout.getvalue() table_output += "\n\n" + test_names_str = "|".join(regressions_list) + table_output += f"Regressions test regexp names: {test_names_str}\n\n" mystdout.close() sys.stdout = old_stdout @@ -682,6 +702,8 @@ def compute_regression_table( writer_regressions.dump(mystdout, False) table_output += mystdout.getvalue() table_output += "\n\n" + test_names_str = "|".join(improvements_list) + table_output += f"Improvements test regexp names: {test_names_str}\n\n" mystdout.close() sys.stdout = old_stdout @@ -724,6 +746,8 @@ def get_by_strings( comparison_tag, baseline_target_version=None, comparison_target_version=None, + baseline_hash=None, + comparison_hash=None, ): baseline_covered = False comparison_covered = False @@ -760,6 +784,16 @@ def get_by_strings( by_str_baseline = "target+version" baseline_str = baseline_target_version + if baseline_hash is not None: + if comparison_covered: + logging.error( + "--baseline-branch, --baseline-tag, --baseline-hash, and --baseline-target-version are mutually exclusive. Pick one..." + ) + exit(1) + baseline_covered = True + by_str_baseline = "hash" + baseline_str = baseline_hash + if comparison_tag is not None: # check if we had already covered comparison if comparison_covered: @@ -781,16 +815,27 @@ def get_by_strings( by_str_comparison = "target+version" comparison_str = comparison_target_version + if comparison_hash is not None: + # check if we had already covered comparison + if comparison_covered: + logging.error( + "--comparison-branch, --comparison-tag, --comparison-hash, and --comparison-target-table are mutually exclusive. Pick one..." + ) + exit(1) + comparison_covered = True + by_str_comparison = "hash" + comparison_str = comparison_hash + if baseline_covered is False: logging.error( "You need to provider either " - + "( --baseline-branch, --baseline-tag, or --baseline-target-version ) " + + "( --baseline-branch, --baseline-tag, --baseline-hash, or --baseline-target-version ) " ) exit(1) if comparison_covered is False: logging.error( "You need to provider either " - + "( --comparison-branch, --comparison-tag, or --comparison-target-version ) " + + "( --comparison-branch, --comparison-tag, --comparison-hash, or --comparison-target-version ) " ) exit(1) return baseline_str, by_str_baseline, comparison_str, by_str_comparison @@ -820,6 +865,8 @@ def from_rts_to_regression_table( tf_triggering_env, verbose, running_platform=None, + baseline_github_repo="redis", + comparison_github_repo="redis", ): print_all = print_regressions_only is False and print_improvements_only is False table_full = [] @@ -835,6 +882,8 @@ def from_rts_to_regression_table( total_comparison_points = 0 noise_waterline = 3 progress = tqdm(unit="benchmark time-series", total=len(test_names)) + regressions_list = [] + improvements_list = [] for test_name in test_names: compare_version = "v0.1.208" github_link = "https://github.com/redis/redis-benchmarks-specification/blob" @@ -848,6 +897,7 @@ def from_rts_to_regression_table( "metric={}".format(metric_name), "{}={}".format(test_filter, test_name), "deployment_name={}".format(baseline_deployment_name), + "github_repo={}".format(baseline_github_repo), "triggering_env={}".format(tf_triggering_env), ] if running_platform is not None: @@ -855,11 +905,15 @@ def from_rts_to_regression_table( filters_comparison = [ "{}={}".format(by_str_comparison, comparison_str), "metric={}".format(metric_name), - "hash==", "{}={}".format(test_filter, test_name), "deployment_name={}".format(comparison_deployment_name), + "github_repo={}".format(comparison_github_repo), "triggering_env={}".format(tf_triggering_env), ] + if "hash" not in by_str_baseline: + filters_baseline.append("hash==") + if "hash" not in by_str_comparison: + filters_comparison.append("hash==") if running_platform is not None: filters_comparison.append("running_platform={}".format(running_platform)) baseline_timeseries = rts.ts().queryindex(filters_baseline) @@ -1037,9 +1091,11 @@ def from_rts_to_regression_table( test_link, ) if detected_regression: + regressions_list.append(test_name) table_regressions.append(line) if detected_improvement: + improvements_list.append(test_name) table_improvements.append(line) if unstable: @@ -1072,6 +1128,8 @@ def from_rts_to_regression_table( total_stable, total_unstable, total_comparison_points, + regressions_list, + improvements_list, ) From 322f35c082a8872ee7f7f36e34f7f8b2f23a0cbc Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 20 Aug 2024 14:25:27 +0100 Subject: [PATCH 04/12] skipping cli builder test on ci --- utils/tests/test_builder.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/tests/test_builder.py b/utils/tests/test_builder.py index 4d70035..d6bdd65 100644 --- a/utils/tests/test_builder.py +++ b/utils/tests/test_builder.py @@ -154,9 +154,7 @@ def test_get_branch_version_from_test_details(): def test_cli_build(): try: - # if should_run_builder(): - if True: - + if should_run_builder(): db_port = int(os.getenv("DATASINK_PORT", "6379")) conn = redis.StrictRedis(port=db_port) conn.ping() From 0b1738e4352493d1b056d3e5ab1e5cabb33e0c0d Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 27 Aug 2024 10:23:43 +0100 Subject: [PATCH 05/12] Added --baseline-target-branch and --comparison-target-branch to the compare tool --- .../__compare__/args.py | 6 ++ .../__compare__/compare.py | 96 +++++++++++++------ ...set-100-elements-sismember-is-a-member.yml | 32 +++++++ ...et-100-elements-sismember-not-a-member.yml | 32 +++++++ ...et-10M-elements-sismember-50pct-chance.yml | 33 +++++++ ...set-1M-elements-sismember-50pct-chance.yml | 33 +++++++ 6 files changed, 205 insertions(+), 27 deletions(-) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml diff --git a/redis_benchmarks_specification/__compare__/args.py b/redis_benchmarks_specification/__compare__/args.py index c35c6e5..3d24498 100644 --- a/redis_benchmarks_specification/__compare__/args.py +++ b/redis_benchmarks_specification/__compare__/args.py @@ -96,6 +96,9 @@ def create_compare_arguments(parser): parser.add_argument( "--baseline-target-version", type=str, default=None, required=False ) + parser.add_argument( + "--baseline-target-branch", type=str, default=None, required=False + ) parser.add_argument("--comparison-branch", type=str, default=None, required=False) parser.add_argument( "--baseline-github-repo", type=str, default="redis", required=False @@ -108,6 +111,9 @@ def create_compare_arguments(parser): parser.add_argument( "--comparison-target-version", type=str, default=None, required=False ) + parser.add_argument( + "--comparison-target-branch", type=str, default=None, required=False + ) parser.add_argument("--print-regressions-only", type=bool, default=False) parser.add_argument("--print-improvements-only", type=bool, default=False) parser.add_argument("--skip-unstable", type=bool, default=False) diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index af72d69..13b2ff0 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -203,6 +203,8 @@ def compare_command_logic(args, project_name, project_version): ) ) baseline_branch = default_baseline_branch + if baseline_branch == "": + baseline_branch = None comparison_branch = args.comparison_branch simplify_table = args.simple_table print_regressions_only = args.print_regressions_only @@ -250,6 +252,8 @@ def compare_command_logic(args, project_name, project_version): running_platform = args.running_platform baseline_target_version = args.baseline_target_version comparison_target_version = args.comparison_target_version + baseline_target_branch = args.baseline_target_branch + comparison_target_branch = args.comparison_target_branch baseline_github_repo = args.baseline_github_repo comparison_github_repo = args.comparison_github_repo baseline_hash = args.baseline_hash @@ -318,6 +322,8 @@ def compare_command_logic(args, project_name, project_version): comparison_hash, baseline_github_repo, comparison_github_repo, + baseline_target_branch, + comparison_target_branch, ) prepare_regression_comment( auto_approve, @@ -547,6 +553,8 @@ def compute_regression_table( baseline_hash=None, baseline_github_repo="redis", comparison_github_repo="redis", + baseline_target_branch=None, + comparison_target_branch=None, ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -574,6 +582,8 @@ def compute_regression_table( comparison_target_version, comparison_hash, baseline_hash, + baseline_target_branch, + comparison_target_branch, ) logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}") logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}") @@ -739,6 +749,11 @@ def compute_regression_table( ) +def get_by_error(name, by_str_arr): + by_string = ",".join(by_str_arr) + return f"--{name}-branch, --{name}-tag, --{name}-target-branch, --{name}-hash, and --{name}-target-version are mutually exclusive. You selected a total of {len(by_str_arr)}: {by_string}. Pick one..." + + def get_by_strings( baseline_branch, comparison_branch, @@ -748,6 +763,8 @@ def get_by_strings( comparison_target_version=None, baseline_hash=None, comparison_hash=None, + baseline_target_branch=None, + comparison_target_branch=None, ): baseline_covered = False comparison_covered = False @@ -755,50 +772,64 @@ def get_by_strings( by_str_comparison = "" baseline_str = "" comparison_str = "" + baseline_by_arr = [] + comparison_by_arr = [] + + ################# BASELINE BY .... + if baseline_branch is not None: - baseline_covered = True by_str_baseline = "branch" + baseline_covered = True baseline_str = baseline_branch - if comparison_branch is not None: - comparison_covered = True - by_str_comparison = "branch" - comparison_str = comparison_branch + baseline_by_arr.append(by_str_baseline) if baseline_tag is not None: - if comparison_covered: - logging.error( - "--baseline-branch and --baseline-tag are mutually exclusive. Pick one..." - ) + by_str_baseline = "version" + if baseline_covered: + baseline_by_arr.append(by_str_baseline) + logging.error(get_by_error("baseline", baseline_by_arr)) exit(1) baseline_covered = True - by_str_baseline = "version" baseline_str = baseline_tag if baseline_target_version is not None: - if comparison_covered: - logging.error( - "--baseline-branch, --baseline-tag and --baseline-target-version are mutually exclusive. Pick one..." - ) + by_str_baseline = "target+version" + if baseline_covered: + baseline_by_arr.append(by_str_baseline) + logging.error(get_by_error("baseline", baseline_by_arr)) exit(1) baseline_covered = True - by_str_baseline = "target+version" baseline_str = baseline_target_version if baseline_hash is not None: - if comparison_covered: - logging.error( - "--baseline-branch, --baseline-tag, --baseline-hash, and --baseline-target-version are mutually exclusive. Pick one..." - ) + by_str_baseline = "hash" + if baseline_covered: + baseline_by_arr.append(by_str_baseline) + logging.error(get_by_error("baseline", baseline_by_arr)) exit(1) baseline_covered = True - by_str_baseline = "hash" baseline_str = baseline_hash + if baseline_target_branch is not None: + by_str_baseline = "target+branch" + if baseline_covered: + baseline_by_arr.append(by_str_baseline) + logging.error(get_by_error("baseline", baseline_by_arr)) + exit(1) + baseline_covered = True + baseline_str = baseline_target_branch + + ################# COMPARISON BY .... + + if comparison_branch is not None: + by_str_comparison = "branch" + comparison_covered = True + comparison_str = comparison_branch if comparison_tag is not None: # check if we had already covered comparison if comparison_covered: logging.error( - "--comparison-branch and --comparison-tag are mutually exclusive. Pick one..." + "--comparison-branch and --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..." ) exit(1) comparison_covered = True @@ -808,18 +839,29 @@ def get_by_strings( # check if we had already covered comparison if comparison_covered: logging.error( - "--comparison-branch, --comparison-tag, and --comparison-target-table are mutually exclusive. Pick one..." + "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..." ) exit(1) comparison_covered = True by_str_comparison = "target+version" comparison_str = comparison_target_version + if comparison_target_branch is not None: + # check if we had already covered comparison + if comparison_covered: + logging.error( + "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..." + ) + exit(1) + comparison_covered = True + by_str_comparison = "target+branch" + comparison_str = comparison_target_branch + if comparison_hash is not None: # check if we had already covered comparison if comparison_covered: logging.error( - "--comparison-branch, --comparison-tag, --comparison-hash, and --comparison-target-table are mutually exclusive. Pick one..." + "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..." ) exit(1) comparison_covered = True @@ -829,13 +871,13 @@ def get_by_strings( if baseline_covered is False: logging.error( "You need to provider either " - + "( --baseline-branch, --baseline-tag, --baseline-hash, or --baseline-target-version ) " + + "( --baseline-branch, --baseline-tag, --baseline-hash, --baseline-target-branch or --baseline-target-version ) " ) exit(1) if comparison_covered is False: logging.error( "You need to provider either " - + "( --comparison-branch, --comparison-tag, --comparison-hash, or --comparison-target-version ) " + + "( --comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch or --comparison-target-version ) " ) exit(1) return baseline_str, by_str_baseline, comparison_str, by_str_comparison @@ -1252,9 +1294,9 @@ def get_v_pct_change_and_largest_var( if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n): comparison_values.append(tuple[1]) comparison_df = pd.DataFrame(comparison_values) - comparison_median = float(comparison_df.median()) + comparison_median = float(comparison_df.median().iloc[0]) comparison_v = comparison_median - comparison_std = float(comparison_df.std()) + comparison_std = float(comparison_df.std().iloc[0]) if verbose: logging.info( "comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format( diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml new file mode 100644 index 0000000..6f2eba0 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-set-100-elements-sismember-is-a-member +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SISMEMBER in which the value is a member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"' +tested-groups: +- set +tested-commands: +- sismember +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SISMEMBER set:100 lysbgqqfqw" --hide-histogram --test-time 180 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml new file mode 100644 index 0000000..ae1e511 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-set-100-elements-sismember-not-a-member +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SISMEMBER in which the value is not a member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"' +tested-groups: +- set +tested-commands: +- sismember +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SISMEMBER set:100 not-a-member" --hide-histogram --test-time 180 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml new file mode 100644 index 0000000..646f448 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml @@ -0,0 +1,33 @@ +version: 0.4 +name: memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key with 10M elements. We query it using SISMEMBER in which the value has 50% change of being member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SADD set:10M:elements __key__" --command-key-pattern=P --key-maximum 10000000 --key-prefix "" -n 10000000 --hide-histogram -t 1 -c 1 +tested-groups: +- set +tested-commands: +- sadd +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SISMEMBER set:10M:elements __key__" --key-maximum 20000000 --key-prefix "" --hide-histogram --test-time 180 + resources: + requests: + cpus: '4' + memory: 2g +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml new file mode 100644 index 0000000..ad7c789 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml @@ -0,0 +1,33 @@ +version: 0.4 +name: memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key with 1M elements. We query it using SISMEMBER in which the value has 50% change of being member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SADD set:1M:elements __key__" --command-key-pattern=P --key-maximum 1000000 --key-prefix "" -n 1000000 --hide-histogram -t 1 -c 1 +tested-groups: +- set +tested-commands: +- sadd +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SISMEMBER set:1M:elements __key__" --key-maximum 2000000 --key-prefix "" --hide-histogram --test-time 180 + resources: + requests: + cpus: '4' + memory: 2g +priority: 1 From c3293a3acac4bf8f9c83027197481e454d127ba1 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 27 Aug 2024 15:33:07 +0100 Subject: [PATCH 06/12] Added GEOPOS and GEOSEARCH WITHCOORD new benchmarks --- ...r_benchmark-1key-geo-2-elements-geopos.yml | 32 ++++++++++++++++++ ...lements-geosearch-fromlonlat-withcoord.yml | 33 +++++++++++++++++++ ...et-10M-elements-sismember-50pct-chance.yml | 1 + ...set-1M-elements-sismember-50pct-chance.yml | 1 + 4 files changed, 67 insertions(+) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml new file mode 100644 index 0000000..c1785e2 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-geo-2-elements-geopos +description: 'Runs memtier_benchmark, for a keyspace length of 1 GEO key. The GEO key contains 2 elements and comes from the example of https://redis.io/docs/latest/commands/geopos, and we query it using GEOPOS command.' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"GEOADD" "Sicily" "13.361389" "38.115556" "Palermo" "15.087269" "37.502669" "Catania"' +tested-groups: +- geo +tested-commands: +- geopos +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: -c 50 -t 4 --command="GEOPOS Sicily Palermo Catania" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 138 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml new file mode 100644 index 0000000..04d37ed --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml @@ -0,0 +1,33 @@ +version: 0.4 +name: memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord +description: 'Runs memtier_benchmark, for a keyspace length of 1 GEO key. The GEO key contains 2 elements and comes from the example of https://redis.io/docs/latest/commands/geosearch, and we query it using GEOSEARCH command.' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"GEOADD" "Sicily" "13.361389" "38.115556" "Palermo" "15.087269" "37.502669" "Catania"' + - '"GEOADD" "Sicily" "12.758489" "38.788135" "edge1" "17.241510" "38.788135" "edge2"' +tested-groups: +- geo +tested-commands: +- geosearch +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: -c 50 -t 4 --command="GEOSEARCH Sicily FROMLONLAT 15 37 BYBOX 400 400 km ASC WITHCOORD WITHDIST" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 138 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml index 646f448..76ec81e 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml @@ -17,6 +17,7 @@ tested-groups: - set tested-commands: - sadd +- sismember redis-topologies: - oss-standalone build-variants: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml index ad7c789..03cb27d 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml @@ -17,6 +17,7 @@ tested-groups: - set tested-commands: - sadd +- sismember redis-topologies: - oss-standalone build-variants: From 8f333d074011cf525995d7f1c8bb98f477552401 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 27 Aug 2024 15:52:39 +0100 Subject: [PATCH 07/12] Included the connection setup benchmark using HELLO --- .../memtier_benchmark-connection-hello.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml new file mode 100644 index 0000000..9db858f --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml @@ -0,0 +1,22 @@ +version: 0.4 +name: memtier_benchmark-connection-hello +description: 'Runs memtier_benchmark, for no keyspace, benchmarking the connection setup scenario using HELLO command.' +tested-groups: +- connection +tested-commands: +- hello +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: -c 50 -t 4 --command="HELLO 2 SETNAME __key__" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 1 From cb07f14ad917e56dcc3eec1c04e1d20eeeafbdc5 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 27 Aug 2024 15:53:13 +0100 Subject: [PATCH 08/12] Bumping version from 0.1.218 to 0.1.219 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2041577..696d974 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.218" +version = "0.1.219" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" From 78b7d149d848b197f23b68fee250b9d77144ae07 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Fri, 30 Aug 2024 12:09:40 +0100 Subject: [PATCH 09/12] Added APPEND/INCRBY/INCRBYFLOAT/SETRANGE benchmarks --- pyproject.toml | 2 +- .../__builder__/builder.py | 8 ++--- .../__compare__/args.py | 2 ++ .../__compare__/compare.py | 14 ++++++++ ..._benchmark-1Mkeys-string-append-1-100B.yml | 34 +++++++++++++++++++ ...memtier_benchmark-1Mkeys-string-incrby.yml | 28 +++++++++++++++ ...er_benchmark-1Mkeys-string-incrbyfloat.yml | 28 +++++++++++++++ ..._benchmark-1Mkeys-string-setrange-100B.yml | 34 +++++++++++++++++++ .../memtier_benchmark-connection-hello.yml | 8 +++++ 9 files changed, 153 insertions(+), 5 deletions(-) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml diff --git a/pyproject.toml b/pyproject.toml index 696d974..9dab74a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.219" +version = "0.1.222" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index 70c0132..4cfcec1 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -644,11 +644,11 @@ def generate_benchmark_stream_request( build_stream_fields["git_version"] = git_version if git_timestamp_ms is not None: build_stream_fields["git_timestamp_ms"] = git_timestamp_ms + + prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}" for artifact in build_artifacts: - bin_key = "zipped:artifacts:{}:{}.zip".format(id, artifact) - bin_artifact = open( - "{}src/{}".format(redis_temporary_dir, artifact), "rb" - ).read() + bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip" + bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read() bin_artifact_len = len(bytes(bin_artifact)) assert bin_artifact_len > 0 conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS) diff --git a/redis_benchmarks_specification/__compare__/args.py b/redis_benchmarks_specification/__compare__/args.py index 3d24498..dc7b8f4 100644 --- a/redis_benchmarks_specification/__compare__/args.py +++ b/redis_benchmarks_specification/__compare__/args.py @@ -44,6 +44,8 @@ def create_compare_arguments(parser): ) parser.add_argument("--github_repo", type=str, default="redis") parser.add_argument("--github_org", type=str, default="redis") + parser.add_argument("--baseline_github_org", type=str, default="") + parser.add_argument("--comparison_github_org", type=str, default="") parser.add_argument("--triggering_env", type=str, default="ci") parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN) parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="") diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index 13b2ff0..699006e 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -256,6 +256,8 @@ def compare_command_logic(args, project_name, project_version): comparison_target_branch = args.comparison_target_branch baseline_github_repo = args.baseline_github_repo comparison_github_repo = args.comparison_github_repo + baseline_github_org = args.baseline_github_org + comparison_github_org = args.comparison_github_org baseline_hash = args.baseline_hash comparison_hash = args.comparison_hash @@ -324,6 +326,8 @@ def compare_command_logic(args, project_name, project_version): comparison_github_repo, baseline_target_branch, comparison_target_branch, + baseline_github_org, + comparison_github_org, ) prepare_regression_comment( auto_approve, @@ -555,6 +559,8 @@ def compute_regression_table( comparison_github_repo="redis", baseline_target_branch=None, comparison_target_branch=None, + baseline_github_org="redis", + comparison_github_org="redis", ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -657,6 +663,8 @@ def compute_regression_table( running_platform, baseline_github_repo, comparison_github_repo, + baseline_github_org, + comparison_github_org, ) logging.info( "Printing differential analysis between {} and {}".format( @@ -909,6 +917,8 @@ def from_rts_to_regression_table( running_platform=None, baseline_github_repo="redis", comparison_github_repo="redis", + baseline_github_org="redis", + comparison_github_org="redis", ): print_all = print_regressions_only is False and print_improvements_only is False table_full = [] @@ -942,6 +952,8 @@ def from_rts_to_regression_table( "github_repo={}".format(baseline_github_repo), "triggering_env={}".format(tf_triggering_env), ] + if baseline_github_org != "": + filters_baseline.append(f"github_org={baseline_github_org}") if running_platform is not None: filters_baseline.append("running_platform={}".format(running_platform)) filters_comparison = [ @@ -952,6 +964,8 @@ def from_rts_to_regression_table( "github_repo={}".format(comparison_github_repo), "triggering_env={}".format(tf_triggering_env), ] + if comparison_github_org != "": + filters_baseline.append(f"github_org={comparison_github_org}") if "hash" not in by_str_baseline: filters_baseline.append("hash==") if "hash" not in by_str_comparison: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B.yml new file mode 100644 index 0000000..72c46d2 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-append-1-100B +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 1 Bytes for each key initially. Then we append to the same keys with values ranging from 1 to 100B. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "1" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- append +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --data-size-range=1-100 --command "APPEND __key__ __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- string +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml new file mode 100644 index 0000000..89498a7 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-incrby +description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incrby of 1. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- incrby +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --command "INCRBY __key__ 1" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml new file mode 100644 index 0000000..0229121 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-incrbyfloat +description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incrby of 0.01. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- incrbyfloat +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --command "INCRBYFLOAT __key__ 0.01" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml new file mode 100644 index 0000000..4200f2f --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-setrange-100B +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 100 Bytes for each key. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "100" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- setrange +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --data-size-range=1-95 --command "SETRANGE __key__ 5 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- string +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml index 9db858f..249ad60 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml @@ -1,6 +1,14 @@ version: 0.4 name: memtier_benchmark-connection-hello description: 'Runs memtier_benchmark, for no keyspace, benchmarking the connection setup scenario using HELLO command.' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g tested-groups: - connection tested-commands: From 3ad6683ae4910ee7d87b9f5425fc34a0996035c3 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Sat, 31 Aug 2024 00:30:22 +0100 Subject: [PATCH 10/12] Included APPEND,INCRBY,INCRBYFLOAT,SETRANGE pipeline 10 benchmarks --- pyproject.toml | 2 +- .../__compare__/compare.py | 4 +-- ...Mkeys-string-append-1-100B-pipeline-10.yml | 34 +++++++++++++++++++ ...hmark-1Mkeys-string-incrby-pipeline-10.yml | 28 +++++++++++++++ ...-1Mkeys-string-incrbyfloat-pipeline-10.yml | 28 +++++++++++++++ ...Mkeys-string-setrange-100B-pipeline-10.yml | 34 +++++++++++++++++++ 6 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B-pipeline-10.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby-pipeline-10.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat-pipeline-10.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B-pipeline-10.yml diff --git a/pyproject.toml b/pyproject.toml index 9dab74a..ba1210e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.222" +version = "0.1.223" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index 699006e..9789fea 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -937,7 +937,7 @@ def from_rts_to_regression_table( regressions_list = [] improvements_list = [] for test_name in test_names: - compare_version = "v0.1.208" + compare_version = "main" github_link = "https://github.com/redis/redis-benchmarks-specification/blob" test_path = f"redis_benchmarks_specification/test-suites/{test_name}.yml" test_link = f"[{test_name}]({github_link}/{compare_version}/{test_path})" @@ -965,7 +965,7 @@ def from_rts_to_regression_table( "triggering_env={}".format(tf_triggering_env), ] if comparison_github_org != "": - filters_baseline.append(f"github_org={comparison_github_org}") + filters_comparison.append(f"github_org={comparison_github_org}") if "hash" not in by_str_baseline: filters_baseline.append("hash==") if "hash" not in by_str_comparison: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B-pipeline-10.yml new file mode 100644 index 0000000..3821903 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B-pipeline-10.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-append-1-100B-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 1 Bytes for each key initially. Then we append to the same keys with values ranging from 1 to 100B. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "1" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- append +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 --data-size-range=1-100 --command "APPEND __key__ __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- string +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby-pipeline-10.yml new file mode 100644 index 0000000..7b2c905 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby-pipeline-10.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-incrby-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incrby of 1. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- incrby +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 --command "INCRBY __key__ 1" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat-pipeline-10.yml new file mode 100644 index 0000000..2eea19c --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat-pipeline-10.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-incrbyfloat-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incrby of 0.01. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- incrbyfloat +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 --command "INCRBYFLOAT __key__ 0.01" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B-pipeline-10.yml new file mode 100644 index 0000000..fc97097 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B-pipeline-10.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-setrange-100B-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 100 Bytes for each key. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "100" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- setrange +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 --data-size-range=1-95 --command "SETRANGE __key__ 5 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- string +priority: 1 From 5c0573a23370a0c0ffe7bc8c60e08b69f073f956 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Sat, 31 Aug 2024 01:56:50 +0100 Subject: [PATCH 11/12] Added SETEX benchmark --- pyproject.toml | 2 +- ...k-1Mkeys-string-setex-100B-pipeline-10.yml | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setex-100B-pipeline-10.yml diff --git a/pyproject.toml b/pyproject.toml index ba1210e..d08ec88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.223" +version = "0.1.224" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setex-100B-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setex-100B-pipeline-10.yml new file mode 100644 index 0000000..9c97b9f --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setex-100B-pipeline-10.yml @@ -0,0 +1,33 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-setex-100B-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 100 Bytes for each key. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "100" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- setex +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 --command "SETEX __key__ 3600 __value__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g +tested-groups: +- string +priority: 1 From d520cfa0a94a1f3ffbe1171370c19f246cab4ed6 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 3 Sep 2024 17:04:44 +0100 Subject: [PATCH 12/12] Added ZUNION, ZUNIONSTORE, and extra ZADD benchmarks. Included SADD benchmarks with intset underlying encoding --- pyproject.toml | 2 +- .../__compare__/args.py | 6 +++ .../__compare__/compare.py | 39 ++++++++++++++----- .../self_contained_coordinator.py | 4 +- ...t-intset-with-100-elements-pipeline-10.yml | 30 ++++++++++++++ ...keys-load-set-intset-with-100-elements.yml | 30 ++++++++++++++ ...istpack-with-100-elements-double-score.yml | 30 ++++++++++++++ ...t-300-elements-skiplist-encoded-zunion.yml | 33 ++++++++++++++++ ...-elements-skiplist-encoded-zunionstore.yml | 33 ++++++++++++++++ utils/create-intset.py | 9 +++++ utils/create-zset-300-elements.py | 21 ++++++++++ 11 files changed, 225 insertions(+), 12 deletions(-) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-zset-listpack-with-100-elements-double-score.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunion.yml create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunionstore.yml create mode 100644 utils/create-intset.py create mode 100644 utils/create-zset-300-elements.py diff --git a/pyproject.toml b/pyproject.toml index d08ec88..eb85969 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.224" +version = "0.1.226" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__compare__/args.py b/redis_benchmarks_specification/__compare__/args.py index dc7b8f4..df087fd 100644 --- a/redis_benchmarks_specification/__compare__/args.py +++ b/redis_benchmarks_specification/__compare__/args.py @@ -123,6 +123,12 @@ def create_compare_arguments(parser): parser.add_argument("--simple-table", type=bool, default=False) parser.add_argument("--use_metric_context_path", type=bool, default=False) parser.add_argument("--testname_regex", type=str, default=".*", required=False) + parser.add_argument( + "--regression_str", type=str, default="REGRESSION", required=False + ) + parser.add_argument( + "--improvement_str", type=str, default="IMPROVEMENT", required=False + ) parser.add_argument( "--regressions-percent-lower-limit", type=float, diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index 9789fea..835543d 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -285,7 +285,7 @@ def compare_command_logic(args, project_name, project_version): detected_regressions, table_output, total_improvements, - total_regressions, + regressions_list, total_stable, total_unstable, total_comparison_points, @@ -328,7 +328,10 @@ def compare_command_logic(args, project_name, project_version): comparison_target_branch, baseline_github_org, comparison_github_org, + args.regression_str, + args.improvement_str, ) + total_regressions = len(regressions_list) prepare_regression_comment( auto_approve, baseline_branch, @@ -355,6 +358,7 @@ def compare_command_logic(args, project_name, project_version): total_unstable, verbose, args.regressions_percent_lower_limit, + regressions_list, ) return ( detected_regressions, @@ -393,6 +397,7 @@ def prepare_regression_comment( total_unstable, verbose, regressions_percent_lower_limit, + regressions_list=[], ): if total_comparison_points > 0: comment_body = "### Automated performance analysis summary\n\n" @@ -423,6 +428,14 @@ def prepare_regression_comment( comparison_summary += "- Detected a total of {} regressions bellow the regression water line {}.\n".format( total_regressions, regressions_percent_lower_limit ) + if len(regressions_list) > 0: + regression_values = [l[1] for l in regressions_list] + regression_df = pd.DataFrame(regression_values) + median_regression = round(float(regression_df.median().iloc[0]), 2) + max_regression = round(float(regression_df.max().iloc[0]), 2) + min_regression = round(float(regression_df.min().iloc[0]), 2) + + comparison_summary += f" - Median/Common-Case regression was {median_regression}%% and ranged from [{min_regression},{max_regression}] %%.\n" comment_body += comparison_summary comment_body += "\n" @@ -561,6 +574,8 @@ def compute_regression_table( comparison_target_branch=None, baseline_github_org="redis", comparison_github_org="redis", + regression_str="REGRESSION", + improvement_str="IMPROVEMENT", ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -665,6 +680,8 @@ def compute_regression_table( comparison_github_repo, baseline_github_org, comparison_github_org, + regression_str, + improvement_str, ) logging.info( "Printing differential analysis between {} and {}".format( @@ -697,7 +714,7 @@ def compute_regression_table( writer_regressions.dump(mystdout, False) table_output += mystdout.getvalue() table_output += "\n\n" - test_names_str = "|".join(regressions_list) + test_names_str = "|".join([l[0] for l in regressions_list]) table_output += f"Regressions test regexp names: {test_names_str}\n\n" mystdout.close() sys.stdout = old_stdout @@ -750,7 +767,7 @@ def compute_regression_table( detected_regressions, table_output, total_improvements, - total_regressions, + regressions_list, total_stable, total_unstable, total_comparison_points, @@ -919,6 +936,8 @@ def from_rts_to_regression_table( comparison_github_repo="redis", baseline_github_org="redis", comparison_github_org="redis", + regression_str="REGRESSION", + improvement_str="IMPROVEMENT", ): print_all = print_regressions_only is False and print_improvements_only is False table_full = [] @@ -1103,27 +1122,27 @@ def from_rts_to_regression_table( if baseline_v != "N/A" or comparison_v != "N/A": detected_regression = False detected_improvement = False - if percentage_change < 0.0 and not unstable: + if percentage_change < 0.0: if -waterline >= percentage_change: detected_regression = True total_regressions = total_regressions + 1 - note = note + " REGRESSION" + note = note + f" {regression_str}" detected_regressions.append(test_name) elif percentage_change < -noise_waterline: if simplify_table is False: - note = note + " potential REGRESSION" + note = note + f" potential {regression_str}" else: if simplify_table is False: note = note + " No Change" - if percentage_change > 0.0 and not unstable: + if percentage_change > 0.0: if percentage_change > waterline: detected_improvement = True total_improvements = total_improvements + 1 - note = note + " IMPROVEMENT" + note = note + f" {improvement_str}" elif percentage_change > noise_waterline: if simplify_table is False: - note = note + " potential IMPROVEMENT" + note = note + f" potential {improvement_str}" else: if simplify_table is False: note = note + " No Change" @@ -1147,7 +1166,7 @@ def from_rts_to_regression_table( test_link, ) if detected_regression: - regressions_list.append(test_name) + regressions_list.append([test_name, percentage_change]) table_regressions.append(line) if detected_improvement: diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 50a985b..c6f4bab 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -1375,7 +1375,7 @@ def process_self_contained_coordinator_stream( detected_regressions, table_output, total_improvements, - total_regressions, + regressions_list, total_stable, total_unstable, total_comparison_points, @@ -1409,6 +1409,7 @@ def process_self_contained_coordinator_stream( use_metric_context_path, running_platform, ) + total_regressions = len(regressions_list) auto_approve = True grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks" @@ -1438,6 +1439,7 @@ def process_self_contained_coordinator_stream( total_unstable, verbose, regressions_percent_lower_limit, + regressions_list, ) logging.info( f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}" diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10.yml new file mode 100644 index 0000000..6f2aa74 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading SETs with 100 integer elements. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- sadd +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --pipeline 10 --test-time 180 -c 50 -t 4 --command "SADD __key__ 436858 768277 10664 860016 865292 841848 313147 896678 386308 977048 203069 283373 593503 457419 169542 391186 130304 916639 453967 273773 589383 657683 182813 641487 580095 195884 372142 774005 768470 95729 556839 771113 432139 613472 528415 174665 155343 215707 753892 81624 199200 877732 358153 337683 24664 481128 356011 28176 535519 724263 780697 435661 3623 6998 985912 696150 913066 171765 808279 608289 187563 300496 104333 684239 434669 111037 928553 714514 606168 190478 772363 393219 243921 549032 624950 743846 957491 864902 345918 846332 42207 243593 125622 915607 523476 226892 773003 840853 792775 582050 702242 194856 270254 617201 454742 543414 960526 136178 43558 172112" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram + resources: + requests: + cpus: '4' + memory: 4g + +tested-groups: +- set +priority: 12 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements.yml new file mode 100644 index 0000000..e0ef9f4 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-set-intset-with-100-elements +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading SETs with 100 integer elements. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- sadd +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 180 -c 50 -t 4 --command "SADD __key__ 436858 768277 10664 860016 865292 841848 313147 896678 386308 977048 203069 283373 593503 457419 169542 391186 130304 916639 453967 273773 589383 657683 182813 641487 580095 195884 372142 774005 768470 95729 556839 771113 432139 613472 528415 174665 155343 215707 753892 81624 199200 877732 358153 337683 24664 481128 356011 28176 535519 724263 780697 435661 3623 6998 985912 696150 913066 171765 808279 608289 187563 300496 104333 684239 434669 111037 928553 714514 606168 190478 772363 393219 243921 549032 624950 743846 957491 864902 345918 846332 42207 243593 125622 915607 523476 226892 773003 840853 792775 582050 702242 194856 270254 617201 454742 543414 960526 136178 43558 172112" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram + resources: + requests: + cpus: '4' + memory: 4g + +tested-groups: +- set +priority: 12 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-zset-listpack-with-100-elements-double-score.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-zset-listpack-with-100-elements-double-score.yml new file mode 100644 index 0000000..317329c --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-zset-listpack-with-100-elements-double-score.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-zset-listpack-with-100-elements-double-score +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading ZSETs with encoding:listpack with 100 elements. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- zadd +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 180 -c 50 -t 4 --command "ZADD __key__ 0.652540306855235 UaZtxmrKBkxhBqJOlNJlwVUfHVALGkjnUemvubDvbGSVAaaQkXLlsg 0.8731899671198792 RKlVyoHMTvZcoh 0.0785627468533846 VZWJQlqLBHicktajowIvkyuaddTC 0.7688563664469605 YaXTxnRGWhENdaOFuXNAJmx 0.0033318113277969186 JjfFkvonkksjIfHS 0.49606648747577575 Q 0.8234766164292862 MZgqGWzeYWDAlplbxvlzllKR 0.42687597903639085 AGSAuhePDVmONKmViHovKsiIGSXQZdqX 0.34912978268081996 ZHIKgOWejxTOcrVCRxztCNqtVFEdqCbowiaROZfbhMmzFlr 0.15838599188422475 UuyeNtxYcAmkfjtovBmajhGcYvzucpgFjLnqKIw 0.4376432899068222 cfgHnJGQAHaXHjLIuLWYkDACbxQbywRIQMbuE 0.5536939917085721 uyZtiyKNdKhFhgNlcdtnHgDFEdBoovVYkyLIpREln 0.403546330051068 kRUDWaPmeYXeJZhgqKpnroFxUSCQ 0.8808108541089766 PCGgHbGIRxpzFRfADhupCtBKhknUNyqK 0.23044636914343264 OaHUUrbZEvSOmYOtbUzxnJq 0.6404180580960533 qJabjKLYYivniNNtGsbnBzRLzYP 0.6907030016224224 KJrGsIeQESEPkBdTgUhrQuAUzEfySPhVBoNaWDjOawkRrjyktuOMuBcwp 0.24900135884008867 bwQzFiMXWxSKtYJiFCSaDWxoAEoNGvFsMoAugjlIcWTRWhXyixemQnagvqL 0.7928656841898908 NKtwuzbLhLCCPupqgkRsRdGijHmSrnqwdfSj 0.9331302297178864 TcZibcjyBLAbljRGHjFgwVeVjbeZwi 0.640319798434186 nsmIvzevkqEJkJTiybCjqiXRjJVMqSqsSaHKxTzJmygg 0.5239886233297175 DZGleQLgYgAjWUcCOhzqsi 0.6136266546940706 ZVlOgBjrpZhQuQzJYVLYqnxytasTT 0.6591139297465682 AFgEEoLcwlGmoWeYLZZJnlilPagiaWdqhItaEMLovOfbaPAgtCeeyjCuEV 0.4468461455464968 pAjBRwToyaDwhSExhMreHmbgzPuJFPlpVrBbjhkyogmUAQamL 0.5614231865616031 KyArAdlPTTgxCatXdDQIPgZWHCBZicfkZjCKCbuAuvA 0.022487789155224203 qOuBflYpFEUWKIfvrIuZYEwJYtSIwSlvegDBDYCUTKnBoRWpwDuBAHgYbTwURCW 0.8510804209364501 QcjxLOamZbsRzGPlmSvuvBpYmDaooLoY 0.43460695876638156 WcTDiiDHQwNoubgkihXThvM 0.5610492156901229 XPGwnyfBmCnhkPlJxhZhPrjvXoGKWEWbqrFvZauVZGQ 0.7513407089150304 MTXpbeTMlXoxUsCDccRYHeKKfYunLBZuHbLgJmNStiiALTm 0.09175823221394674 vkAxXiwoCWTBUFUVtpBfpAlILPGaMvUrFaRuBqEgqvWnISaZaPTxiklQtzGS 0.8586634748104288 wyThKKWAeRg 0.36952587840155204 cRkudmpGSSMUnPOMhPSzjQ 0.538356756374977 TsRYebuuDgtamrxVbrxOaCBUGWEaGheHKWgzWsgPxVBfLwAev 0.5218317459277027 N 0.658938125456635 mAI 0.8474868095237909 JrYvYRtaFZldysTlMUVfZfVpHrSzayZGWysedghOV 0.20143282352735015 WjaRWpmPlWwPuepbXywfDmqIEPWnBh 0.23967093347793234 ZCLtDpzxZHqWUMzAJOfavhqZrGrPiGZQSBYDPqfyqqYAbGIHuyskvLk 0.8108325344648399 WhxmpALjwYdCOMwTeHRbFRSFfktMmQUVItotZiICjWfwebl 0.1322111892750637 dfVVGlvVTMufbfSsy 0.2285660170875491 DBqEkifRzIyvzTydaSqNrqalAXBZAkccYOuIBFEn 0.9396586731821924 NyIOQsgOAQevXYKYhGZXjVzTeqPiwjdJhtq 0.32374056012496966 TQYLPdswKkUntJEjvGWJTWxmVJGdBcbIclOrYzVqqtfFznolqcHYTVj 0.9080986140709963 IdUdmeLZNHLsCGaVkEdfKMemYRdzhQUqvhuQdXnYchGWXyYRAYZQWkb 0.3129458198716534 LJFPbCuN 0.39349745859330765 YTPdFVszdCVVgGGHKqoQxfjU 0.14704886640549086 fOFwwDNEsxjfpkUrwd 0.9168814654086035 rJSASiFxeYIINRbbgTBqPbATrcKOP 0.3100306836090321 VKKOyHveatcnmQR 0.2370937718635434 OCaIAjJkxSLYaWpMAVjmQoOXoZJMcAO 0.4941647120371836 neeyLswlmLnZXSCDtFx 0.269336828778751 UuIrk 0.12151325035284255 FmCgjvkoPmAcscWZixkzsFgsQBGAmDHqFSTaKcUdfxlfYWu 0.0035001439463812067 aThlZZWlDITljruiWa 0.5009568203132024 TBszlGVnMdDvMOLiCysTjjANuFqeq 0.4783242512285928 dafCundgmRFJLzTEexYHeprmCXFjxrJYLZAcsLXhTrgC 0.531664059031722 reqfWtmvctIfWtqVEuaTEGfarDa 0.47961187976147 CBvTozpAXrLpS 0.10226598211977789 eGQCebwBiUOPhMfrVRHiThsDeIFzPDEKHczj 0.721486119508813 nHhoxBCSgFFhJWBhTSVVXBpdFoLKYYKZmWEgvCfa 0.3044462915617381 AMu 0.8380361220680647 tQiutM 0.7791893412340167 jcKQrBTFeuiuHCprihKx 0.9140067046543505 bHuocqbTLhTHlHKIfDdemFDPwrcYFeJrXXafsATvSmFvKGYKRJYnqFmwKoRuaptI 0.6755477551341951 sJuZYHZFrVYyTvMdbMZJDuCqUmpRIsEgizBVplIPCkSAtGc 0.843803368180551 WgcCuSQuSzcvSPOzzXViuXfnOtBSxjm 0.4683803962889672 CqNaseFeIzhsgZdZDgySGfNawLz 0.300477038321727 pLiQVPjQuEROtmKceRjEcaiZxMvMSIeailtKgwUeVsgqQUdTUbWg 0.6782593142253811 bgbVdSCYQstqQQloWsJLCYYrmdvtmKeNXDfWGHvzdyTFuuiMwSxuSZa 0.09916181882671649 jHWIgQCltkPBgzLWWTfRJlLFrxesFUmaJzfeeKDsFglvPAwNIDCDdlvicM 0.031870198089671176 YfsobnenpUlKHCXNgKYVeWHNRFEPOsAU 0.9753495730511261 OHVmNYCiDZFlkmsqhwgc 0.6796131843206142 CPVsbsqYVeWPVRsTZKTvlcDwHnUzHpZyEchJqxnbrWM 0.41149806297291536 MGFGt 0.0635854843880973 xVozQKiljlffdYDherQcdYdEmEX 0.17349539011413317 SORWOElEDHqZuOvYslqzY 0.6957717601786134 XNVNKDBXRoFHDAu 0.514568844593022 GTagGWDGPcixUFOSdZTBaKpiJSPiKhmyricEAMLBwjQyEXmJIZn 0.5054082476784039 YOxGoUKeShnXhcdvTJQFGTukjSiRZFidGRfkttgXJBeKKnJebZNubpk 0.5791117949403571 PFfNxLyEMLCXtgEQVpFVGxadSZeOXdaSCaQmDBrMdJLSlIXAnerUpaF 0.6869490812905924 EDFLipKWqONzJ 0.5858117779265328 ufGNBddDuAsmqgFMQYqtMrOTUbOiexjEYqLKdPLscwavpbHWCbGvMSKfkp 0.3328807181469634 kIxgjCASHjQbqrvhxiPozqCtRVebWwuHCptoxBJ 0.5593178033061493 BHmhWwOEWhCelxVLRsUSiAsSicrsXyAYhbrOWaDtKVVDyVpgTtkbO 0.9568394270185203 XrsvMeAof 0.2895493213801318 eVIlCARlsKEygkqngxlVoJIhthE 0.36404485659899855 tAahXQIJDAbNWOgsSyAUkgwVcIzNvkawniDRxludhXrgdbReUJWKaDjPkLNCi 0.6751249599564046 zUNDGaygGLVztZyQgSMyevPstQKjDVTMafZnepYqPvANg 0.4002349143471098 blBkPgUMFnFxBlocRBolScezbONPxsKrGsdsMsiZWrhuRsINONXImqQlQY 0.4373588125087955 LpfFjgOkMnuixmgfGamVKyPICLQQzNKmdJaJRnflC 0.8323339473420782 TsvDvTRrfMUEskXqXTEHTKzYcuzaoBTAyKVkcnvvqRTgXItNQwJ 0.7464672802658118 GpXkYQokvVduxNQfcaSYTSiZsOMxrAlNMtPPdWCLKWWqbeunMDLgkIRRu 0.15234267491477727 bTapHmxvqZNdGNP 0.42610519579163275 DllLIwysTAcQqwVPjAGkGEedTflBevgZmdgwMbKpNdKJoWGteGWJwyhrI 0.05073435890699274 uoKiwzrJXTOqaDpeWknKEXdTkQziOGXjZZikNbaEPwOMFvlKoERaUq 0.8053618509879708 UYuKZw 0.21335197746306034 OqRQLePoVHjfSoZV 0.9955843393406656 s" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram + resources: + requests: + cpus: '4' + memory: 4g + +tested-groups: +- sorted-set +priority: 12 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunion.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunion.yml new file mode 100644 index 0000000..4391002 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunion.yml @@ -0,0 +1,33 @@ +version: 0.4 +name: memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunion +description: 'Runs memtier_benchmark, for a keyspace length of 2 ZSET keys. We query their union using ZUNION that returns 600 elements. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 2 + resources: + requests: + memory: 1g + init_commands: + - '"ZADD" "zset:skiplist:1:300" "0.652540306855235" "ele:1:UaZtxmrKBkxhBqJOlNJlwVUfHVALGkjnUemvubDvbGSVAaaQkXLlsg" "0.8731899671198792" "ele:2:RKlVyoHMTvZcoh" "0.0785627468533846" "ele:3:VZWJQlqLBHicktajowIvkyuaddTC" "0.7688563664469605" "ele:4:YaXTxnRGWhENdaOFuXNAJmx" "0.0033318113277969186" "ele:5:JjfFkvonkksjIfHS" "0.49606648747577575" "ele:6:Q" "0.8234766164292862" "ele:7:MZgqGWzeYWDAlplbxvlzllKR" "0.42687597903639085" "ele:8:AGSAuhePDVmONKmViHovKsiIGSXQZdqX" "0.34912978268081996" "ele:9:ZHIKgOWejxTOcrVCRxztCNqtVFEdqCbowiaROZfbhMmzFlr" "0.15838599188422475" "ele:10:UuyeNtxYcAmkfjtovBmajhGcYvzucpgFjLnqKIw" "0.4376432899068222" "ele:11:cfgHnJGQAHaXHjLIuLWYkDACbxQbywRIQMbuE" "0.5536939917085721" "ele:12:uyZtiyKNdKhFhgNlcdtnHgDFEdBoovVYkyLIpREln" "0.403546330051068" "ele:13:kRUDWaPmeYXeJZhgqKpnroFxUSCQ" "0.8808108541089766" "ele:14:PCGgHbGIRxpzFRfADhupCtBKhknUNyqK" "0.23044636914343264" "ele:15:OaHUUrbZEvSOmYOtbUzxnJq" "0.6404180580960533" "ele:16:qJabjKLYYivniNNtGsbnBzRLzYP" "0.6907030016224224" "ele:17:KJrGsIeQESEPkBdTgUhrQuAUzEfySPhVBoNaWDjOawkRrjyktuOMuBcwp" "0.24900135884008867" "ele:18:bwQzFiMXWxSKtYJiFCSaDWxoAEoNGvFsMoAugjlIcWTRWhXyixemQnagvqL" "0.7928656841898908" "ele:19:NKtwuzbLhLCCPupqgkRsRdGijHmSrnqwdfSj" "0.9331302297178864" "ele:20:TcZibcjyBLAbljRGHjFgwVeVjbeZwi" "0.640319798434186" "ele:21:nsmIvzevkqEJkJTiybCjqiXRjJVMqSqsSaHKxTzJmygg" "0.5239886233297175" "ele:22:DZGleQLgYgAjWUcCOhzqsi" "0.6136266546940706" "ele:23:ZVlOgBjrpZhQuQzJYVLYqnxytasTT" "0.6591139297465682" "ele:24:AFgEEoLcwlGmoWeYLZZJnlilPagiaWdqhItaEMLovOfbaPAgtCeeyjCuEV" "0.4468461455464968" "ele:25:pAjBRwToyaDwhSExhMreHmbgzPuJFPlpVrBbjhkyogmUAQamL" "0.5614231865616031" "ele:26:KyArAdlPTTgxCatXdDQIPgZWHCBZicfkZjCKCbuAuvA" "0.022487789155224203" "ele:27:qOuBflYpFEUWKIfvrIuZYEwJYtSIwSlvegDBDYCUTKnBoRWpwDuBAHgYbTwURCW" "0.8510804209364501" "ele:28:QcjxLOamZbsRzGPlmSvuvBpYmDaooLoY" "0.43460695876638156" "ele:29:WcTDiiDHQwNoubgkihXThvM" "0.5610492156901229" "ele:30:XPGwnyfBmCnhkPlJxhZhPrjvXoGKWEWbqrFvZauVZGQ" "0.7513407089150304" "ele:31:MTXpbeTMlXoxUsCDccRYHeKKfYunLBZuHbLgJmNStiiALTm" "0.09175823221394674" "ele:32:vkAxXiwoCWTBUFUVtpBfpAlILPGaMvUrFaRuBqEgqvWnISaZaPTxiklQtzGS" "0.8586634748104288" "ele:33:wyThKKWAeRg" "0.36952587840155204" "ele:34:cRkudmpGSSMUnPOMhPSzjQ" "0.538356756374977" "ele:35:TsRYebuuDgtamrxVbrxOaCBUGWEaGheHKWgzWsgPxVBfLwAev" "0.5218317459277027" "ele:36:N" "0.658938125456635" "ele:37:mAI" "0.8474868095237909" "ele:38:JrYvYRtaFZldysTlMUVfZfVpHrSzayZGWysedghOV" "0.20143282352735015" "ele:39:WjaRWpmPlWwPuepbXywfDmqIEPWnBh" "0.23967093347793234" "ele:40:ZCLtDpzxZHqWUMzAJOfavhqZrGrPiGZQSBYDPqfyqqYAbGIHuyskvLk" "0.8108325344648399" "ele:41:WhxmpALjwYdCOMwTeHRbFRSFfktMmQUVItotZiICjWfwebl" "0.1322111892750637" "ele:42:dfVVGlvVTMufbfSsy" "0.2285660170875491" "ele:43:DBqEkifRzIyvzTydaSqNrqalAXBZAkccYOuIBFEn" "0.9396586731821924" "ele:44:NyIOQsgOAQevXYKYhGZXjVzTeqPiwjdJhtq" "0.32374056012496966" "ele:45:TQYLPdswKkUntJEjvGWJTWxmVJGdBcbIclOrYzVqqtfFznolqcHYTVj" "0.9080986140709963" "ele:46:IdUdmeLZNHLsCGaVkEdfKMemYRdzhQUqvhuQdXnYchGWXyYRAYZQWkb" "0.3129458198716534" "ele:47:LJFPbCuN" "0.39349745859330765" "ele:48:YTPdFVszdCVVgGGHKqoQxfjU" "0.14704886640549086" "ele:49:fOFwwDNEsxjfpkUrwd" "0.9168814654086035" "ele:50:rJSASiFxeYIINRbbgTBqPbATrcKOP" "0.3100306836090321" "ele:51:VKKOyHveatcnmQR" "0.2370937718635434" "ele:52:OCaIAjJkxSLYaWpMAVjmQoOXoZJMcAO" "0.4941647120371836" "ele:53:neeyLswlmLnZXSCDtFx" "0.269336828778751" "ele:54:UuIrk" "0.12151325035284255" "ele:55:FmCgjvkoPmAcscWZixkzsFgsQBGAmDHqFSTaKcUdfxlfYWu" "0.0035001439463812067" "ele:56:aThlZZWlDITljruiWa" "0.5009568203132024" "ele:57:TBszlGVnMdDvMOLiCysTjjANuFqeq" "0.4783242512285928" "ele:58:dafCundgmRFJLzTEexYHeprmCXFjxrJYLZAcsLXhTrgC" "0.531664059031722" "ele:59:reqfWtmvctIfWtqVEuaTEGfarDa" "0.47961187976147" "ele:60:CBvTozpAXrLpS" "0.10226598211977789" "ele:61:eGQCebwBiUOPhMfrVRHiThsDeIFzPDEKHczj" "0.721486119508813" "ele:62:nHhoxBCSgFFhJWBhTSVVXBpdFoLKYYKZmWEgvCfa" "0.3044462915617381" "ele:63:AMu" "0.8380361220680647" "ele:64:tQiutM" "0.7791893412340167" "ele:65:jcKQrBTFeuiuHCprihKx" "0.9140067046543505" "ele:66:bHuocqbTLhTHlHKIfDdemFDPwrcYFeJrXXafsATvSmFvKGYKRJYnqFmwKoRuaptI" "0.6755477551341951" "ele:67:sJuZYHZFrVYyTvMdbMZJDuCqUmpRIsEgizBVplIPCkSAtGc" "0.843803368180551" "ele:68:WgcCuSQuSzcvSPOzzXViuXfnOtBSxjm" "0.4683803962889672" "ele:69:CqNaseFeIzhsgZdZDgySGfNawLz" "0.300477038321727" "ele:70:pLiQVPjQuEROtmKceRjEcaiZxMvMSIeailtKgwUeVsgqQUdTUbWg" "0.6782593142253811" "ele:71:bgbVdSCYQstqQQloWsJLCYYrmdvtmKeNXDfWGHvzdyTFuuiMwSxuSZa" "0.09916181882671649" "ele:72:jHWIgQCltkPBgzLWWTfRJlLFrxesFUmaJzfeeKDsFglvPAwNIDCDdlvicM" "0.031870198089671176" "ele:73:YfsobnenpUlKHCXNgKYVeWHNRFEPOsAU" "0.9753495730511261" "ele:74:OHVmNYCiDZFlkmsqhwgc" "0.6796131843206142" "ele:75:CPVsbsqYVeWPVRsTZKTvlcDwHnUzHpZyEchJqxnbrWM" "0.41149806297291536" "ele:76:MGFGt" "0.0635854843880973" "ele:77:xVozQKiljlffdYDherQcdYdEmEX" "0.17349539011413317" "ele:78:SORWOElEDHqZuOvYslqzY" "0.6957717601786134" "ele:79:XNVNKDBXRoFHDAu" "0.514568844593022" "ele:80:GTagGWDGPcixUFOSdZTBaKpiJSPiKhmyricEAMLBwjQyEXmJIZn" "0.5054082476784039" "ele:81:YOxGoUKeShnXhcdvTJQFGTukjSiRZFidGRfkttgXJBeKKnJebZNubpk" "0.5791117949403571" "ele:82:PFfNxLyEMLCXtgEQVpFVGxadSZeOXdaSCaQmDBrMdJLSlIXAnerUpaF" "0.6869490812905924" "ele:83:EDFLipKWqONzJ" "0.5858117779265328" "ele:84:ufGNBddDuAsmqgFMQYqtMrOTUbOiexjEYqLKdPLscwavpbHWCbGvMSKfkp" "0.3328807181469634" "ele:85:kIxgjCASHjQbqrvhxiPozqCtRVebWwuHCptoxBJ" "0.5593178033061493" "ele:86:BHmhWwOEWhCelxVLRsUSiAsSicrsXyAYhbrOWaDtKVVDyVpgTtkbO" "0.9568394270185203" "ele:87:XrsvMeAof" "0.2895493213801318" "ele:88:eVIlCARlsKEygkqngxlVoJIhthE" "0.36404485659899855" "ele:89:tAahXQIJDAbNWOgsSyAUkgwVcIzNvkawniDRxludhXrgdbReUJWKaDjPkLNCi" "0.6751249599564046" "ele:90:zUNDGaygGLVztZyQgSMyevPstQKjDVTMafZnepYqPvANg" "0.4002349143471098" "ele:91:blBkPgUMFnFxBlocRBolScezbONPxsKrGsdsMsiZWrhuRsINONXImqQlQY" "0.4373588125087955" "ele:92:LpfFjgOkMnuixmgfGamVKyPICLQQzNKmdJaJRnflC" "0.8323339473420782" "ele:93:TsvDvTRrfMUEskXqXTEHTKzYcuzaoBTAyKVkcnvvqRTgXItNQwJ" "0.7464672802658118" "ele:94:GpXkYQokvVduxNQfcaSYTSiZsOMxrAlNMtPPdWCLKWWqbeunMDLgkIRRu" "0.15234267491477727" "ele:95:bTapHmxvqZNdGNP" "0.42610519579163275" "ele:96:DllLIwysTAcQqwVPjAGkGEedTflBevgZmdgwMbKpNdKJoWGteGWJwyhrI" "0.05073435890699274" "ele:97:uoKiwzrJXTOqaDpeWknKEXdTkQziOGXjZZikNbaEPwOMFvlKoERaUq" "0.8053618509879708" "ele:98:UYuKZw" "0.21335197746306034" "ele:99:OqRQLePoVHjfSoZV" "0.9955843393406656" "ele:100:s" "0.8638417234689056" "ele:101:uninyp" "0.6170733682662144" "ele:102:lAEvUqfinjrCuTdqJeZpMOqTGvxPeGRyPlQgwunbDdbTEwCsIiUsteGjmwH" "0.21067098312505894" "ele:103:cLsWBIpwTWpKBrmSXb" "0.028805240455834213" "ele:104:ySsyBzpsJRyLIAkqneYVrUXPSZVHjpEYkuPt" "0.011280501394645426" "ele:105:LoyMrzfbpcBTBvjgSaalQslVYEsvOjWxPFeLL" "0.8115524648082592" "ele:106:PemidyGhQD" "0.035339107659025926" "ele:107:udmOOXuqRqJxINvXvFplPC" "0.4088305270182697" "ele:108:KpDIy" "0.7970215729800464" "ele:109:ADZJJiLuNsGIaWjDtovm" "0.6539941469274039" "ele:110:wkRPjlYtkmmRJGsaKTTlNqggqMJEqPziZOGtDlMoEQBFcJZgJSTJdwEAv" "0.6393759361590676" "ele:111:SqLVSBeiOrFcbKmUUqeFvXcyXmfPuAFnRgaiFHPUUhlHHNkzbGjeIPVJS" "0.2775184026673655" "ele:112:NCSCTxgpSSGUSVzcWWZVcGlEwoVuKkjwzMYPOTBjvHHgTxuwPjLOpJCTpQnK" "0.8279369714332658" "ele:113:VfSDdgYSpSeXVhbfntnUfUchciRBrJBZDrvxMPhOiIp" "0.8297173264793561" "ele:114:TXarLyioSIyrUKkOeMXvaJvJaknKVQeaNErnRoufOnArsovay" "0.921999914094598" "ele:115:luAfTJtIHOjwkvrAmjILtyLShtdxPljZptJJQVhhAhCzzBe" "0.1408651634335023" "ele:116:SEpIqRVTaiuoxSCVlaVHjClWDwMEqaBUPMjoEAxzzQgylhwdbYkJu" "0.9948615128480278" "ele:117:ruBCNLrVXV" "0.28579196360737913" "ele:118:rVSIXyykbQnnnQEPddYhrTskBJVTwatBoNFCIyajGejYyDzK" "0.48423755726111095" "ele:119:xgSdRSFMaNILA" "0.7626249612557783" "ele:120:EwYCAzoAXXaXkVnxrtBULVzEoRUc" "0.8752501969266744" "ele:121:LfegRMqEenhHURSmftuFgmOXRtOKcVsTpWqqQxuybwdAWBJduQZxCNyWCz" "0.18280846923814642" "ele:122:GRqtlaLHKmwlLyzsVuSRZX" "0.37855655211354067" "ele:123:YyuHDEqKXwhJkmKKLIRMCbiwrrldSooerdtGFSAAQfWLg" "0.5528763818820261" "ele:124:ynUYqHxGTImWvrHkzVWNKxYvllmfKI" "0.027904528470665824" "ele:125:ICQAYRVcxiqABLuLUSZrsDtCQCTFbMe" "0.47753076845502873" "ele:126:ycQETDJKxyOXEnXzAlytRgOusgLNkd" "0.8606084882797624" "ele:127:tvmUFgtvmjdxLkTEFnFHLGxhLDSPAUfHRhKLSWYbqa" "0.2695904518559701" "ele:128:BaHcXnCofYiGMaxujBsmBX" "0.825149665561988" "ele:129:JBKxSCR" "0.5873452590489512" "ele:130:YGoDxkbekHCOlecubsBeKjmWezcLEbksCtLLRLzxtzvjWrsmbzNLYKehEP" "0.4853159409847696" "ele:131:IZcFPOMzPHcFIHxowCHSTFrkz" "0.5277987567294383" "ele:132:ljcFEIPIpqvYkqtuTMtTXjbBLQOPRHHrqup" "0.9746883646151202" "ele:133:fYcqTpqdzPZANfWuvHIarLYgYsOUoqlQnKPw" "0.42010203003852076" "ele:134:IZwUCcFtBWLXQmMtlMPwa" "0.6802068264362765" "ele:135:mOEPkdtczWkOlMuGGAFAQSJSAzSguNtlkLNVLRdriYltynDBrE" "0.5605617357048859" "ele:136:TaqJicNvVjnCZNKPJuuiStVfruAlkGgGVaTnMbvCbSoZitGWn" "0.4780678427736166" "ele:137:qtBqNJpcFllvQXdoISfeQrTfcKyuMBqWRKWnmzPpTweQyOZBTQQdpog" "0.9561513047181356" "ele:138:UTOEvbGjBMjYZiBs" "0.8298720638792384" "ele:139:gKXkpmgiTrUqh" "0.6737117765393265" "ele:140:EkktDpJsFgJjhKWTaxldBFqqTNBzdownDwxUZIxYveC" "0.2239791912985728" "ele:141:dyvMXjLusRKyYuvDFHXfLcIodjkniz" "0.3232124237169193" "ele:142:JuESJkOktNOtmFITnZdMXZqPmexwZpETgaTlFVVzpXct" "0.2473393788924324" "ele:143:cpSyDhnVPmGNLmeBsFJufaWWgGOwIjQaAXNzrhGbJDrpbeqFDQamnLu" "0.6849106786577351" "ele:144:VooyGbMEAigOuQtVaVEckAxrSQWNtypiEaQEBLGTtbRkPZvsAyXv" "0.3646945793371218" "ele:145:wzvoPCzE" "0.4263757643063858" "ele:146:EqOnUtiDML" "0.11877588855760746" "ele:147:pZteEnbVnuAOtHIrRGNqclHBYATQxnBuZGBfmnNVfagNEIB" "0.8313214144471833" "ele:148:vbXuqfTwwpCPGUJQtKQKUroVrfNxuuKdHcgtLOBbGvRbbwLvaWjlN" "0.3552235823133526" "ele:149:wTUOXktMFXeuxGgVsbzVWudxXvVFGsB" "0.8670386139794473" "ele:150:uJTtytjczUmGkvsUJQcqcpovnyjcvYUUbLtJCSClvzhaOSJDjS" "0.36577274012963557" "ele:151:lyfmWPlIeqc" "0.24997912597471894" "ele:152:TOwZZHcrChOptJHLflNWYtMacvyAfMOlgdAdJlBwXJfRRzzQzXkcOWFhGdOnLQX" "0.8930265134635483" "ele:153:CqkyWatRYKqEIoUffGbyhbOYEWXmMSCOFiWtVTHSoGBLKGixPGOAHJSwaA" "0.8844448577667497" "ele:154:PpiojYsguWELV" "0.20663410393014814" "ele:155:ebykSXjGhZUYxPFNVUCNSHGNKXESmbthHLdBMIqxoevIKBNVajvToF" "0.6375009411769629" "ele:156:BKyktVmLOkuhGFnj" "0.7758172657193476" "ele:157:TTyfrulmMejuyeUsOqVpxnZlIUKukfBQzfFiWbJtF" "0.7611282500726224" "ele:158:iHCjJFEPDIKAvxzcmHaIka" "0.6032754373225729" "ele:159:HvqgRWNliaavH" "0.03311820686884104" "ele:160:Mc" "0.34191832438430536" "ele:161:ueTQVZrHBiAnDl" "0.4234716642081342" "ele:162:OnmAvoDKw" "0.5301950080950026" "ele:163:QKCLvxWwXHxJcFCEelBdFtEpefx" "0.022733115513385638" "ele:164:vDBwpjkPsgMLSMqORsEaDUPrKSERewhtkSKeiXSg" "0.7166885759719259" "ele:165:XhWRCtQNRxRocWTCaLeAEPwpaZnkTpuXjLdKbOOfYZKkXaSQHhpSXqKXCPmxgvo" "0.2712986802084326" "ele:166:nYfETiDXlAbQRwHV" "0.5341470637872556" "ele:167:pvPkJe" "0.9127872805617512" "ele:168:FDtkhJUZjwqLpzRKOXrlHswdBRwWosyOEWbIWcexvVQMZfPzBB" "0.047439916319365505" "ele:169:BqzapfPLtCxXpDpwXRUIqhuArCXhVbLDVe" "0.7003965818703025" "ele:170:DRWtKsWQBXPylejYIjweSlEUYHjQ" "0.9295842042213984" "ele:171:rpCOjLwmoPSdRSBgAsvnslHoBRjcjVgs" "0.15337097639157538" "ele:172:NBcujL" "0.35038314902687495" "ele:173:BPFNhDL" "0.2972028976456653" "ele:174:jDLdVxGIuQu" "0.05070784214978197" "ele:175:tGvWuCBnevIuRAMJkJM" "0.8737583830123556" "ele:176:usk" "0.2596502464090681" "ele:177:mLdja" "0.6512549702440724" "ele:178:ADVf" "0.171917523837822" "ele:179:GJznUmhoHwfVUKgyPmTcHBSOeXgzbjYjzGGCVFMRapXYdnLaIrqP" "0.9979397420983357" "ele:180:sxmXzXmdFnmLemE" "0.7014210018416481" "ele:181:wkrprcygzJTIyenjHRmbSlKjPkCFZNCpxEfmfHPtJOvhGnmuTJN" "0.47129995449384954" "ele:182:kLMiOVKqDAySJCleSQNmCpxHiWKsiiHAB" "0.9380196572341066" "ele:183:mAPuAsnTTWt" "0.7985363184110637" "ele:184:nsKyWhvIJRENgInWpMcYtlXUdYlrxAmwPsMJNfceYIisVAWkUHXLzkvtCsWE" "0.30436502305230007" "ele:185:MtIHsabFodqvQfXFMohljmJmjySkBQytRvsqiwaZsvqXiFaSBhCASpj" "0.18301728746159307" "ele:186:XvcnJWRlzUTIReQLYEvHanoPk" "0.6830517248352558" "ele:187:mWPnYVSFpfHXZQQAbfPmEjYhlZBmNbiEtrELg" "0.0573871247075346" "ele:188:fcpBmLgqkyICmxewGKyu" "0.7366175249486746" "ele:189:lqenYUoRTmdXgVZSxipJtAANzywKKADhcLjBITUfqrXNSp" "0.6626013234824965" "ele:190:xLRDyJxCopIvZIRnCKccnsqdcmbaEMqPvhzWLFiqZdcoWnTqVtZjVymzWIAXa" "0.9960600588618732" "ele:191:PRteciWuildZLvmEeTCJmgMllDKoOJhJwixuZLBGlbsQvadWVLfhnlV" "0.727874550086809" "ele:192:DMUrMSntsHqYQYxcyFphyVPUK" "0.6604159554316062" "ele:193:pEGlqTKafZFuSDMDOKpGZxdIi" "0.09721821831280353" "ele:194:tzLNAcTqXdGzNFbh" "0.9791379052279395" "ele:195:fB" "0.2838642126919253" "ele:196:gRtNokZLMzuqJRkScwAjS" "0.6981639965614237" "ele:197:fXBKJwUT" "0.9696961287425543" "ele:198:jWFKhNjahnRsbroFeGsfFxwnqqGnahaOFiWjHZorNvhLTxttVJRJQQOFY" "0.4258376716676413" "ele:199:IdmFRBLBHLVjGL" "0.858169214478561" "ele:200:zNJFLPdwPyNXNQKiV" "0.3927031503235572" "ele:201:dRj" "0.237872815865668" "ele:202:pjmoJAOJJUtJvqbsZUaiVJejdqIsiDzdeIfbtDenOfIecKOUiGkQ" "0.11542254177692224" "ele:203:MafcbGHWHjuSFeJnbsmwZSey" "0.5318198291186923" "ele:204:JemvwOIgWnBOsPGsStj" "0.5340725029771403" "ele:205:RTZNBRvrFlroSePZmaZhFiwiLhIVW" "0.17377941268306563" "ele:206:XdkL" "0.30708537669522173" "ele:207:OHUUxtNrNFWGGmCwQXGRCrmTkCIpAXGhVkOrYQ" "0.3642470025832414" "ele:208:qaAxqlRyOUGZKMGUVYKUMOeKTxAJGhWvQNzobjTpgHJmqA" "0.2580334519092651" "ele:209:uyUOGLOzNJScaHhOnWPlsEXonSluZJiTMejoBbmhCBiNPnykKcFLRCPYLQNh" "0.2624285220413499" "ele:210:MuwzwnrcuJwdIKLcWMSbnw" "0.34635829232927273" "ele:211:xyJm" "0.10681423588864192" "ele:212:yovowsSuNWRajWXYNf" "0.8816729069830225" "ele:213:DsOEyomgaAPysPtXOHUsAHRjoaYFAGjkYpfAuzWUyIisUuxhSYgueB" "0.11729073608149843" "ele:214:lYWIYclvxtRgJWlvaqwRxdu" "0.2873790683625198" "ele:215:yvgaPYEzVRGwEYEXRGaAcslGqMMCbCbCCfQxuduv" "0.03381283119846146" "ele:216:oEcjbUMKXHVmheaVDplZJhlzwvohIGSBtimMPkYCwHlVNCEUudhptbDT" "0.6756254954730867" "ele:217:lgkJrJMgdFDLqKbxLuZKuNcIbnzinlnEPO" "0.41449552533526235" "ele:218:zlXNUWHqaCTqefHAyhhsbpjIBwPbRgyKkbNmrxPuRmIspZFJcCavcTIxfvoWxA" "0.493707932686583" "ele:219:BpeNnAZtparUSxZapRlQrmheeSYAwUrmoV" "0.5902863264146183" "ele:220:sKaQfmuFTtIoTWSpOM" "0.5253670876810722" "ele:221:lthVVlxBYtYfhKWbKlmespgrYSqooRsxfEhsJfknVIXRlqxTsxoZiFTDpsxXijW" "0.3979963963224349" "ele:222:dypPPwhZYSSZdLLbaTdqNJvxTbMzLOqbaTsqZoplptkmeVfCoyaUzA" "0.3264162174828448" "ele:223:rWRQxKaqsIHKEIndbSWaRsbtxdLKGmbjIySumpCBTVAGl" "0.5865194234543977" "ele:224:TVWmksLFBdwalaCrcQZRsxBP" "0.28559508093149" "ele:225:lweqdVnCultPdfA" "0.4390884403874875" "ele:226:uwOenAghOaAIzmwlOXrRUTAUnWekan" "0.08146792623238486" "ele:227:NxjScuRQyNcFnWqkRTHLolqrxZRekUBUPtr" "0.56411402327654" "ele:228:TVolwVnFHikQlXWOlzKqZfbSvacowfOABsdeVJLkDPXYmGl" "0.05018525036598798" "ele:229:puOxHskikfnTicBCMzJBLaQnrayTlKspWQu" "0.9726447630383682" "ele:230:SoZUJqIJBKsKgJSEShIwONErhhRgBUqJoqecwqgJrgRkRcidMfLaPBzrpBY" "0.2609811554039797" "ele:231:B" "0.4312484747031583" "ele:232:DoZogQEBRkqEfEzGrnAY" "0.13663455127121937" "ele:233:rYjRcYSynHaSRrVKKJIYI" "0.6922873613563912" "ele:234:bWqKHzRRjENdGkYSOARBsSOwQNLvQbUaKnGxwqaupsxg" "0.26794259830799383" "ele:235:ChaEtSGzPuSjhDOKWYKLJmAGGjPRafTbILOWDTPjECztaBMu" "0.9354162431701218" "ele:236:wfJvpZanAmkVtByeNbReOTsGHuxwpFNmBUNHHKFZG" "0.06293157585414022" "ele:237:WXTGbAxNRqkrEjPrrutKPvGufVmhfdxS" "0.14584091869740912" "ele:238:BaAdncxLcTxXSQrjaDePSS" "0.5768086947048872" "ele:239:NshApWUBkEmPuKNkJcEzdNHGwOzPNbtsjJqwFiRhDGfLJKdpwWIqhomLbV" "0.1465696901238831" "ele:240:jUeWhVOTNZsjeoNuEEiLveNn" "0.8008567772804235" "ele:241:bItAchwIsfNZwxpXMsVzByShOaeidVgPUwHcYlKWbRcdmqVvBDtsr" "0.015881745702420402" "ele:242:zdZZudYQqwmzIdtwriWwnnSBRUjOMxgMyBDFlgifIjPCtRAuFqsIjEYD" "0.708215367473081" "ele:243:YHqmOTBgnTISUeIJJYUYvScqHPhistRICcmpGaVbC" "0.39871686306191767" "ele:244:FhW" "0.9063885465673784" "ele:245:TXtHLEKrnsHVLAUKugMxXYqStLjfOuplXltOeBQJzajnPG" "0.06971324400435519" "ele:246:iBVEpOsIQoRLJNXenfYpXLMMMrLSpjHrFTXbdPvMqMaCrPXSjWwUq" "0.7801801140370873" "ele:247:WQJxHppuKdjrMHEkadNLIBNzbbl" "0.791869958747193" "ele:248:sWCJjeZLjQRozg" "0.11309761860918266" "ele:249:plATLnqtFtSgqqQKdVmdClWzznco" "0.44101158627489334" "ele:250:VojKDXEzXkTK" "0.8836040369092594" "ele:251:RBpRJjERYngmXMJfhQnnCyTEZWJoqVnjeTcqzFzzMlXmvATdihbjl" "0.6308249820355207" "ele:252:UZTymvHpshovPSgGjIDJOxJdjwghvyTBZDHkVQO" "0.5006090429638539" "ele:253:uFpUOJOzJZphVtPaDLJMsyyYwjwFjMRGpDgZe" "0.20079043114588013" "ele:254:zKkpUbLcsdyoLl" "0.8872601871594847" "ele:255:icXC" "0.37517625327876525" "ele:256:nwZmgjIJHB" "0.4472302965803041" "ele:257:CEZcDflAQTNZdzObNbrwLtEqjNgkWbcLcXuZQXJAEqyoGymOYqppuJJjyOPEoPFQ" "0.780663262913547" "ele:258:NbjitZzbvvAnffFGnTBhYqkZRbQlPVvc" "0.6114790546701429" "ele:259:EZYOaVLeDkKLGytYZrxLGRzr" "0.18378642318141092" "ele:260:pWYbNbgRlWPddxsjKHmOW" "0.9576760507667561" "ele:261:sZzqVDT" "0.46086383365401096" "ele:262:mGJfvnLGBwbINxDCEXIRtHdmXyqPlOQhvyvmQTGaHAwZfqLsLjJppkdDdMwwsBv" "0.5751175337887499" "ele:263:NmAoaFQUsjupBVUJSiQbUzexEMKCLVHVrRpJrPzlndc" "0.7387122963781291" "ele:264:YWtBJMQbuojBcfwCHTfh" "0.7526407515005629" "ele:265:ILLeKkqt" "0.8982229649976201" "ele:266:udmQBnKTaKXjlaHyfAvffWdNXIKAMPMbIfVNZMCbMUwvlhNCVRJpMsoU" "0.13915355988685996" "ele:267:TG" "0.11838397899318787" "ele:268:LjoaredUaqedrhRQLJViXILOffWJlDgNaCGhROnZIig" "0.2720595275779307" "ele:269:OLvPf" "0.05213699817329576" "ele:270:TGXeaegcvaWaexsIWJODTEudunbCJWjOnfmKszg" "0.9760555384384879" "ele:271:fCQccupsw" "0.1826479936554769" "ele:272:bTCJyfrkZoFKqxNdCsTLxNAfmXPRCSEdNDrMRlvKCRFrjjPuJxlNFkKnaNQ" "0.1816006773234322" "ele:273:SSicxIfZQttLnNPbzDMvxzfSFhnPHAkfFPeclMecFQvnjLmCQpEwjJiq" "0.644883358870358" "ele:274:rpLDqOGlO" "0.5993794681833459" "ele:275:EbFjBeehfQxqNpOQMUUtGvrJKkZVWxWFJxFCKV" "0.6528913316681816" "ele:276:ewiSyYmEjbtTf" "0.5686335211517553" "ele:277:jeVTPCizAMtXIkuTchJUxkYLeTYQwOneymekogMzIvRxzTDzIMLSvujA" "0.6181915174781923" "ele:278:LFxy" "0.14900559309428074" "ele:279:hfghiprQqSviaaVAiIFKekzwQjWhlkZZlVprXxbaqgHYmnFzEOuTtFwBStUfB" "0.8338524744347505" "ele:280:hqwGMGZVWelRXBEphXvTmwXMeVvJc" "0.771379497890405" "ele:281:HVrnrXGygZsIYancOKsmUaLEkPoUpxQRPG" "0.5994072231684636" "ele:282:lIqDYWuyipeWAOAdBQDuTvsKys" "0.534524220798845" "ele:283:kKTDdgRCCxfiPBundKdDyieG" "0.7242139542264654" "ele:284:ni" "0.748944785839734" "ele:285:PdcemqxiBQyHNkhLpTObryItAXvDpRjjmVHDikyedhTeYAhNwQKMHeVcfApk" "0.13370039714229587" "ele:286:FYwSoEUHBhDBsrJMdzpluc" "0.16028766133142947" "ele:287:aDKzQKcuBfiyyuqHEFHLRRCLRamOUARk" "0.026062274014891695" "ele:288:huXBuVvepspZbb" "0.22018183573052252" "ele:289:AnWyUVUBaTyVmRaUcLZWmStRhKeHAgVckxyFojKTBF" "0.07982614064135407" "ele:290:tgggRahsumg" "0.3697557070272254" "ele:291:rksriVmscvrYXzIYayeHJdvPTaGQCdqXnev" "0.09422878535627355" "ele:292:EABGSXaJ" "0.8325274594683421" "ele:293:CgCfRcqeCzNAHINpiUDuMeBeRLRdcd" "0.9693896699692307" "ele:294:sSatYuMLPzLp" "0.5311474696918715" "ele:295:YAaIFIfsiogCsuvVUgrst" "0.39224110837672066" "ele:296:mEHNOjSErtEsZOtlBJPzCRcXmftzwRPNTBVKYtqVoGZssx" "0.8176954488229201" "ele:297:XUBXZiCSdAUJpFEGVzozodrzsDkFYgezysediNYbMkBGfEsqCqY" "0.6137094587670581" "ele:298:zRtQCUMmJkbIxcWFeJhlrnwvhDOcsCTuQmIUIhiDugArfHxbMMwyyUJtxqhirG" "0.7719959033122741" "ele:299:ebFOLQRWlTZoaDHGTjjtKivqSh" "0.37933435410799776" "ele:300:gZjmUQgmBiqNfiPsOCrjlfeTlEEbOfBVrAwqvO"' + - '"ZADD" "zset:skiplist:2:300" "0.09262750297586564" "ele:1:or" "0.9383651132213807" "ele:2:awUEuWVMajnpzOiOTArpuiSQekbfhVWn" "0.5214936071707239" "ele:3:dLvVjtXgfYwUEOxXFQmag" "0.9079214934215405" "ele:4:wMOfziikZlxOokdQjWovNdHdjFFsDAdJaTsZoQmsnEZarl" "0.9067368627115772" "ele:5:HsmxitflvwMNNhEtFbLO" "0.5918503322692709" "ele:6:ksWvrVzSlLItxbntLMhsJiwEZtZoTefTLdTPucwVBLNEHDkVSBNaNCawnVHq" "0.3360474296101915" "ele:7:VmqzHXRmiQzEsGlfdl" "0.7021374853906152" "ele:8:bNt" "0.40311940256418133" "ele:9:HXRaRKNjkdKs" "0.14791692909385812" "ele:10:PWxaeVvawgdqBTonQoBXTylfhnrLAHXJiHtNQdqQOdTr" "0.5499073150836501" "ele:11:WwqfjEXLLihgdcDuddksFuHVAwFvZgfuVfOeNPHFADpJMNySPsVyNt" "0.6739165529304422" "ele:12:SCtBitWxpz" "0.20235214431676818" "ele:13:cPlWndhlUf" "0.981893336127039" "ele:14:doYXBBDseRsuLWCTFIHAuPUxEducFdUYRQDZfNEvVfuDZXKBdOZnpaOiYW" "0.6255137319908027" "ele:15:vHMz" "0.35225381591612226" "ele:16:FFmOJmOgTObVwKIoXdbQFAkIAThqcmuxgAAOzEFlG" "0.6556189581423885" "ele:17:QMjmHFEILYQBtcbQxsazwEkHG" "0.1086786080705826" "ele:18:aQJgvxcBkRtfGQAMgWYhsnhRuHvWeXOzx" "0.4443262581173478" "ele:19:yzixcbOpFNbGrZTlyAlOFrSPvpkkWqrGodYsZWeipzXpHeZ" "0.03407764418162318" "ele:20:vStWKgLIlsakjbFJd" "0.6036090393945417" "ele:21:jEekc" "0.25987832126438726" "ele:22:KpsJAyUDNdNQzsfSkMDfHZ" "0.22654291425478534" "ele:23:tqI" "0.24666995271407322" "ele:24:LGIGPhYgkxfwbKmXWqvPLbYeHhkFUCLFsnOdnLHYtGyjzZkiIdY" "0.4267692300380157" "ele:25:eXcwFSZJCrCCFJynzpgdQDyMWoXDYwCcfTmNbWkcfbk" "0.6293648111766171" "ele:26:EAMIUZvUsAkDhOgAZOylhhQZNyhPurbXSCdaoTCEiZbKxTzXmEZuoQXLz" "0.9918223572334939" "ele:27:IJviLAjBfTjpnygkNme" "0.5400327173421172" "ele:28:WjVMTKGMQhBtwExljSYstaaoWHCwKTBAjwzUbbOAx" "0.4391638034400295" "ele:29:GaTkBmQtkABFNYXgRXBFQYrngrXCuUjLGzLdoOMSANzbAXGN" "0.5248632352049256" "ele:30:wruIHqSfNHqhQdoWyUrkwToqZjh" "0.4245142889842648" "ele:31:QNrjPwqaFhwvzIvRnDrShNDqVvGHHIDnQddMSDyEdxDMIjZvuQSTtcTERRpqDf" "0.6630427890269389" "ele:32:XsvmtdTG" "0.7021725631889002" "ele:33:zXNKoihjyQxKbVTFCsoexA" "0.5991014496086998" "ele:34:xMAdMecdzpMwLOKLsmVM" "0.4167006541149544" "ele:35:XNPlwYjUbEO" "0.21748473627901788" "ele:36:AfNMHPOCbeAksrJDhTMgToaloyhwouVaw" "0.40372213029096016" "ele:37:ELpdvJAfmUdBcPCnZSCXObXTOWLgiEXnEMlOTGXXPCnswBjbHdxjeBHg" "0.6140401066542219" "ele:38:OHhcyIKEemBnEXrQVcRdZhcKkoGOdHWXNCMzSKusoLHENoHXujdS" "0.7550574793380168" "ele:39:MSYpiWjfKhvpfkBSPVCYgjDKdrN" "0.8269371540962509" "ele:40:cfZEjxhlfRgVZfefeVAPlBHpeJxQQwykwaZjj" "0.7674080844005543" "ele:41:CCLbStaDdBZaTUnJPidjwSbEZeeoNigsjnUQDZnjVnWa" "0.036595137191906546" "ele:42:xiULvOJbNphDvlpwSzdoexRlCXtEnlvAUrkppFBlddyJXwaElEcJMPRoqObnm" "0.5698074730307818" "ele:43:SEvLWl" "0.3369129481509787" "ele:44:VcOjsKDJyinLQIRrVnKCWPjgjbDuNzjiyweNxESMKbrQbugYHsIiPoknumhPR" "0.6422106412803937" "ele:45:IAmzgSECLDxqgd" "0.6894083278054212" "ele:46:PcglhQAZxSfSMxJUMwnDvEzlyrZpPxsNyGBujHQvzddmusSaTHtpAkpdzMsNtg" "0.6450057709395803" "ele:47:TJPQSHCulWh" "0.3070909482618994" "ele:48:blWTtuxKnoRamyEtyRbQjJbYYCRaseaIXXtvBihOIadnSKMyKaFGwWBy" "0.8598262989702926" "ele:49:IVJoggskzWEkHKovKwoOHOmaKbgIUYdINWmChdmr" "0.10354415035880926" "ele:50:kgcLRlwdZFgFhhIENJIspcjGvllDOVLsvZtUrVTMyWDfRpfVz" "0.28928908386800456" "ele:51:AvQiIRwghGePoNguwjfUmvnNpOrsmeHKvjoIcBlQNnNRRlIrXD" "0.5442717124338943" "ele:52:RCTfhiXHCPBwOxtWxdSywwfLquWKLAgMvnZGJSUvPqYndYK" "0.7687123004251277" "ele:53:ZuqUsKCutHPBSYKZBuZJQZhRVyTcYMnUzXFFimIJyzfiJNWn" "0.5345711507419348" "ele:54:sXejHVSkPzasBIBzbBwJNOrmKibrflyQWueOY" "0.9658634461828092" "ele:55:YQPZRMGbtDEpPBEofYZMKKrVHKJkpnXEsmqh" "0.23019227710204926" "ele:56:CufakjBGhcZTOWLtUgznxEDhuvNhFqpBzIrGJtdDouAjRFhLWvQSlFu" "0.6068781157494676" "ele:57:onqdsVasiTWusQQCdCswLqp" "0.8301688570711953" "ele:58:GCsxXdadpevs" "0.5694294722761978" "ele:59:poyUfSRMFmwfErZPJUHlgHlubdBXtQhSJGEpiQRHXfNPHFoVFoWJmrGzNhOP" "0.4493023274389891" "ele:60:k" "0.37642031836089995" "ele:61:YQPaAXgTZiFgtsookRhfWWLMVumXoBodaVNJcHPwb" "0.2228297094226499" "ele:62:ejYqfmTTyrYYvLxPDAse" "0.14198066739390958" "ele:63:fsXUMHeAEejToaWjEhTcrmWMbFFdTRSXUOtGWpEbomwlbiOCxjBkDuargUNurtSL" "0.5233946066825867" "ele:64:TezvDO" "0.4878495944197222" "ele:65:HzsWRNuQBNjMJMoxIfjjlfczTcgfWDJfWlKysFAM" "0.017359009875882236" "ele:66:qvHuOqbAosEcdJqW" "0.4003374590717428" "ele:67:MouLyXUsVzEjmjlLCKqEyGBfCsnenlcvPoFWsERYzTvMNfkLKSgKW" "0.23253292632629075" "ele:68:VjFxenLXwgrkQEAbqklWpggsOMMYeLzxTECqbwwGVwhUGZimqAVURMsxzxm" "0.646901662020837" "ele:69:xColhsZjMEXgAKAKOWKtkoJTUGxRp" "0.8839172533047469" "ele:70:lfmulumeTiacynAyztlI" "0.31940112981806046" "ele:71:OfhTwuZNKxrQLEPWPWLaqNseTfoVbEgBZWqSRCf" "0.9457823004980779" "ele:72:yXkYmdBegJjXwgFpWcZnQnEnonqrUKwLxrpydrxuALbVqLbdQEMeuhQNvYJYCmNL" "0.42349649546528356" "ele:73:nqITDNFxSZCgTrGAlsGR" "0.1491128401167493" "ele:74:wSlpcsEWUAxEfdWdGarbiyZBnHqqXMLbabkR" "0.7164223539226408" "ele:75:aIhaMQxatzemPJXL" "0.45353981831873524" "ele:76:PKyZiUhHjWxRavredJTVpxPfWiSIyw" "0.5055761893027574" "ele:77:fCwJdIVTYUcRpxyVtvgrpOShBVKvuDfnezaEicBSaEkufAYjkbPQZLdmpqA" "0.579969466017893" "ele:78:ttgVNrKMzdhyWKM" "0.37257323396198194" "ele:79:tqhZoSrQzrhOrsVgBimIhUxxIpHCpVgAxXeUEQuQAmwwT" "0.12062537296309861" "ele:80:riTBhsaejufrrtAtSqOowA" "0.42813583015090506" "ele:81:ahRppAcoxMSVLt" "0.6486347611701347" "ele:82:cA" "0.034587858704615115" "ele:83:qqrNxMSTddFARf" "0.7593627828586745" "ele:84:FplVKLwuuOafeOeFSItktKgiy" "0.4994696285128135" "ele:85:tIYVAAegmAqMASVk" "0.11891388133869885" "ele:86:YEmCqaoJjXbJltAOoiFnVQNDuVacCaLlOFaeetedotVui" "0.3894622762499911" "ele:87:JdrKgsAPlek" "0.5336569471727015" "ele:88:hcCtXGaLVHWCgMUlYzUKhduXYLemTyQhEYVFg" "0.920251905287605" "ele:89:jnvoEEFVGKKoOjzUtvfoXhNeTetuiJtaDGEnITEBIFuhPaX" "0.968897738066963" "ele:90:qlwTOlqYcziWhbvBMBTVIgCVFnqpVaPNCTpmSPt" "0.9465629005814513" "ele:91:GOVRIARgGyaAIHPwkeEZzMAnqSyTxFfoqWgM" "0.6059363868454227" "ele:92:SfHzXZvuqNU" "0.7358711669072902" "ele:93:gHeGpNWExpYDynWGcoehLcJEHRAhDZfpHzZAABtY" "0.002391740771283568" "ele:94:uwSSATCqTETlbTuEsmJXbgXAEUltRFNuLGTpMbcNQCwWYt" "0.7158149076298387" "ele:95:HwAliXptdbfTvaAVTOOULUEJyTutsHqnXCWaRTTefOWSWHphznqRhpbJZWs" "0.6038934139572911" "ele:96:VFWrpCzknVyyvPRlvcMNurUszGhKYAJFotTWlsGXFBepexNzhWU" "0.6222780454322215" "ele:97:ehNF" "0.9901469156377757" "ele:98:KnywilEdvSsJWLFHbSZCdGdzwCwqRzZgADKsixcQ" "0.3538196101694284" "ele:99:oBPoRiKlTEbPlNjGYQDBLvUkfozTGdFuP" "0.979768652983662" "ele:100:sFiKSSRMgkIQRuwTcWHjKShPvZdfYHKha" "0.0038377506742498513" "ele:101:Res" "0.2662937612113552" "ele:102:TyAW" "0.2792638772861509" "ele:103:HwgGPitqrKxuhVWgFcnBJPIODYPXtajLeLQxumaekHVdlYfqbmkFnkgiwGAa" "0.1781892660507567" "ele:104:LPuxUVFjLm" "0.04705661819915241" "ele:105:TSouAycARbZOqJrgTwNyjjXrbeXKdVtUMFNswZU" "0.7902202191071308" "ele:106:HPWXuahhazgiqzysESJYkoArGYVcQXJgzOCEUPdOWFFIVJXunbCGGtqqntblixFU" "0.4481460935563768" "ele:107:BJDIpNXqOQWSamkceKuigPXGOLEKPOBSpUHxeoEsqzSvIidEdvcqgYiSVG" "0.3432861842856956" "ele:108:taUgdwQZKdCCodjpsjrhHpANRzpisggAPIUbbNYHEtuvE" "0.42832042624818045" "ele:109:AAOSEmNLVtKbrdVrroPeNRxnxbpnJoTlq" "0.30812383447047176" "ele:110:mXRaSwDGEuPrfoIULekfXFDKYsIzmilkIGYTCeJKwbZhQJrfgQtxFhBARoWJZ" "0.7522373944538988" "ele:111:GgBlqTMGaGyqOFRisSUHUTfCiuUL" "0.21562431158684814" "ele:112:CvwkTqOoXtDtu" "0.8428189114630859" "ele:113:EcWUGnCkfXINMaIgGKkgScPZGIvWuPStFXSAMeOxjXeRptoefDUeIuyJPzRBzRHi" "0.6823372379983094" "ele:114:xzIoKVgZNBRQDQfiUF" "0.13935362874792767" "ele:115:KrpPjoVMOyBmuQqpqT" "0.4332332406044198" "ele:116:UOWNObnoXoaJKzh" "0.08138617832932371" "ele:117:rVhWPfDBDxvwJE" "0.30356446861651587" "ele:118:ZSLbxvYgeGGGiLDww" "0.32067439081280513" "ele:119:bQcKjlcGGQNPqeXDxJDnthZflDwBZEnSosGoXcpgkUACLAMGxw" "0.40934598601386774" "ele:120:cLkEBEgUnFyVcuJGoGcPNYJwilsQjBAw" "0.6433179848407005" "ele:121:nnnQlvic" "0.33121455232187447" "ele:122:NzwulvsTLXrMthvgborfBhEsWPizqgwnTzw" "0.8803242323217817" "ele:123:k" "0.5024021684133856" "ele:124:QxzGwrBCzyjrxScubOiOrGJcTzePOFsPhEHFEzOxkkZCwuUirhETCcjIdoWyA" "0.025065994225742627" "ele:125:jqExxEpycRHNbe" "0.043490094554752834" "ele:126:nareUMxBfzRlJVTvbylndDpEgHoZsZbNDBVInjychmAraHPfaekwZzAwglFw" "0.9816097986428053" "ele:127:IQzEwdcKwNBYqI" "0.6572929124981133" "ele:128:stSHqZwAsTzOsZddsAFwHbfEEsfdII" "0.9613433392746732" "ele:129:dEUztMXSn" "0.6147262075260899" "ele:130:gFwBVpGOtjMuDGVuYRfsFqxLozUbKfIbSnouPpkXziBPLipkyBEToMPgNO" "0.4051062573367158" "ele:131:iYlahFoWqUpvpi" "0.34363782060223536" "ele:132:QzxFJqb" "0.8359405554401546" "ele:133:SwhGBfDsmAFlWxKEuCfBNxdff" "0.22939148700606238" "ele:134:cllbmiogUFJoscJuKiaWvMONpxrWLfuRGMCpvGNFpyMxPJSLvZP" "0.8744608124944555" "ele:135:sYvQFgPAzBKmDspUtlvJdlQVklBHKOFUIHrRoSXgBAVQ" "0.5959086484602159" "ele:136:LPnvKvJYBFLAeCCOQECNeyQsTrPMxvVpqKLjGxKgUBEAhE" "0.9540265436675929" "ele:137:gRldWgnHoDZdu" "0.7669071517114534" "ele:138:DXyqifsGEzBQZNReTCmUGmtUGRjSnYMRjxFpClo" "0.4318348989018863" "ele:139:KtyUQh" "0.39059761540419147" "ele:140:hauqIjYbSoFtMcOOKixBAeGacoockSdTvWdHcqTKUXobKk" "0.24983864913879872" "ele:141:uqZhAekgMEGuxtmrZqmRmJVWoUhhPlyhlghNGmWMFRuxFmFptejHsXs" "0.3210435235914324" "ele:142:CTIJViEGArQbStmcRofVExLPAzMWhDyWQJXDque" "0.18980206607687544" "ele:143:MEcsFtRZtouIWHjYfCXwstdYUpDRmiTQjUkmQDnWi" "0.15361329127026002" "ele:144:dLhflQgp" "0.7110669295479786" "ele:145:meAopAZzKlFlTBIeOETzpEGUjWasknpzyPDfbfugugPOMy" "0.471301584307086" "ele:146:c" "0.453076241949232" "ele:147:hQjzXuyJgtetPOWOxMhUvNJXQbxshZmnSasQsaCEbZglftDYEaNwwoUm" "0.32693695754886076" "ele:148:XbqSlCTEBnEqBzLKGsQ" "0.35856995662107494" "ele:149:EhjPJWQnDvkGAejPAthUUuD" "0.31262357180080613" "ele:150:mHxeZoWhEz" "0.005845222671651107" "ele:151:IyvMEPnFfHSenczOjGODbXJwtMXkuaZFVLntXwtL" "0.9235968098260897" "ele:152:lrpVmOGFgBpPRFjMGZIkJJYPKUKZqRFrUpLqyKTEJBEoleEgCSJZaRxHWXZe" "0.13738091090283422" "ele:153:ntihnyZWbefEPtNiSeQqlVEbqaMSSoCvwZuySkgaTpEVBuaYibjXKCpPPdogPY" "0.8819919106825659" "ele:154:ihNNulMflfjrXcnmwbdTKmwRznOAipcJoPIelWDxaJYtmcixuGWxBsksfIxUWHAg" "0.8932991308660218" "ele:155:segCTllUZbjnrkOODLouJuJXiuAzOvoDcPCggmyMTVzzgudrvwVDX" "0.476578059810815" "ele:156:hDedhNThJbQzAhswOyXSdRPMolSQswuUCeTDlWaAjMtZcBOdoyeBUBxwHAWdlKU" "0.15072385720743042" "ele:157:rDjSR" "0.2161277707711633" "ele:158:ZvNhxpHnEUXOdTdxfRDh" "0.9488145816867642" "ele:159:Q" "0.6787840316052304" "ele:160:sQGndmhzUNtmrDtBzsErELxzfgipMunWyBNABPLhAtCAqbVOXtpPlCeDd" "0.7009800982563948" "ele:161:UwGgQFuNoYhVasdZvJjKEOMcVZJWd" "0.3795646206727531" "ele:162:GdsUALhOYcMICnYiDiYkZVvwhH" "0.017890110952071048" "ele:163:zBqByAKlPKPPOC" "0.5533082346819248" "ele:164:bAdZPWtNjtYoGHdqR" "0.23109286509501725" "ele:165:iUJtSYmwqDRGlMsJREewzEHlvuWaMZPJTCvvgFryO" "0.4713040256603368" "ele:166:wjxQPETQlCQXrRLRTDQGmdTzLlKoUeyMFIRsaNQFhyaJbyXJmlCOSCsHvp" "0.049974994424209385" "ele:167:DGPbIzVMMLUWWjVsevouwaypQhbJyMxjwXsjOkIxTtrtKimvOVC" "0.25554197482909946" "ele:168:yFeyxFl" "0.06546972032558007" "ele:169:OJFPPOAfhqYDChfIsKgmwiaTfDXTUbtEWR" "0.16747563632090212" "ele:170:VfVZvYtxJxcvgOFxPjc" "0.8154729106809464" "ele:171:DmJQkcRAyjgHBMCAFkTucwWQvAICMvtSPXxCZYjBTsgaYxjBEcZeEViUboQA" "0.7721499842095603" "ele:172:KeAwetETxSKBInxxSgcjApglCEEjeapcyAxjKWxWmVUxgedBRvOAuRkXjUZdKqw" "0.5689791211786035" "ele:173:lvGjXnJSwVVaxoNr" "0.8855889473005578" "ele:174:CNroyxiGNciqCZmloBvziBxnjfGcrzWdskAMghlNPyIJ" "0.41912320182786467" "ele:175:ejxnnFYYFrbQVVVTxeLjYeVXWrDZopvmgeBDSHQOPevqXslinL" "0.8035165431612967" "ele:176:VdJifmoOOSJzPOfmpOvrCLFkmtNEmEZqF" "0.6869604080666722" "ele:177:btsGNcuRBzmY" "0.3848414466927291" "ele:178:hnmrVTAOGxtrolmLuvMcDzgBDYEnsOOZvUnOtkxLYPFv" "0.28812018831830133" "ele:179:PXvDrJNIcQFbzQuiYokqPt" "0.11921245346059317" "ele:180:aCufiJDTEkX" "0.024182370939839104" "ele:181:HgnVa" "0.04754239386647818" "ele:182:wosItJxaLEdvLzTjjydyvGqJLUSqT" "0.07505696070559253" "ele:183:rpCPWChrqFvgVEVVKRoWVwOJhVQPTAcEYFz" "0.7810121297050087" "ele:184:VpxhuPTEzZtxYRTVjwNczKXaBkoxhJgbKlLoyVawHuGXunvkJk" "0.7319508963422978" "ele:185:MOwjbZkEMvyOrYFexoyxKFsbexmXqqozGMidaWfES" "0.8182970116273036" "ele:186:RmWeqjsbMYAKOXIpflkbOOiMxRYNrREUFWnBsPywsUTjpRszeVD" "0.13763305291470818" "ele:187:jOsLqpJ" "0.7680871215735682" "ele:188:eFmdPITqTFFhxQFDMxrArrRQNz" "0.9520468889430076" "ele:189:ivzhdLabpxFhrXVumNABm" "0.3334255618135925" "ele:190:jHaWvedsjhAxDTNlXtGMqEggBjmkTARXefpU" "0.02094320460632315" "ele:191:jkMJkvC" "0.782568781581072" "ele:192:CBTuXWkjDEGmCnMoTpvLYZMWbgTHYPzZjmtmXFLRiqeOmTOjZHNjoQo" "0.4130369581381168" "ele:193:IejPDoXOfwXhIBVbWApoqnjm" "0.05032810808761934" "ele:194:GpXjiMttIGORqGmnmKEbUhbvJDtcSEgpwgBqAshNgwuXcoXVzWmN" "0.03562147096755652" "ele:195:TdlepDegXuyVNCYDWpcgWmmTCWTdpdEkxDyduMSOuPdmEdUyMaPepfMgMJQYqfxJ" "0.16058179698958497" "ele:196:dvezkSgvqUSe" "0.19543066541353393" "ele:197:PdrxoChVVQKLNBzhjwNDCbfpZ" "0.543247204423523" "ele:198:cjeiZuqQvFu" "0.33635134143621626" "ele:199:lhZJcssulOevtNgyWxxoNpLvfezhbqgWFJTIDmyxIBzC" "0.5115911058102967" "ele:200:ulngahGBnWlOYgQMBGpxDaupccQfMtbOEPON" "0.3379695196719398" "ele:201:bGtRDNQXwIrwETejqWbnluhGIkIPZpdYmmPTXeowpyneLwHa" "0.2728034878865264" "ele:202:SyQzBNVlAeZKPeRPriqO" "0.5368846611030937" "ele:203:IygYZUGEDlgWTfOebFnhJxOaUAKdRMrimlQOHsvYWd" "0.4539679929781939" "ele:204:HEhoVrszrpmkBDlXjPNFaOrfr" "0.3006834739749523" "ele:205:aROjCcIvSVppcmZvuNFDktThdjDGdHWoAowxMhnBSzmlhwEXzXnGFxINjSfybr" "0.0448639195084084" "ele:206:Tl" "0.5049616444247743" "ele:207:kuhdxIfvoPQppaUlJhUBCtlJzgsxFqelgwmsCbbLv" "0.28294877456995937" "ele:208:zDvaPScfjOwDEfAPsjYAdI" "0.6174513829147165" "ele:209:vqsNtI" "0.18252715540611175" "ele:210:QETJHAoEwrcgWgqMvtpSkxOJIPWKGfDksckQjLy" "0.11630546595702906" "ele:211:SfWoguwffTGTlWKSwrDbLKF" "0.3606182682742908" "ele:212:MbKFUFyFmLLTCfEFgdC" "0.4926337206305448" "ele:213:mdDZaxfdkurWkYykEOMYkPx" "0.6000623954063804" "ele:214:ynHVzFwNLgrrVXYDuGtrufyCVaOzzRfraXTXmxegsCcPeFTSaIVQZqwXCb" "0.2775621516125364" "ele:215:GkhknjnPJlvxwURrjpueAxLAEvSY" "0.7922460208724172" "ele:216:nnDjHaHZMisFPagOGsyRuWIGmkNlxkmrgkotedJGWAZPMwqAVfhHyR" "0.5889353014735185" "ele:217:LjXgi" "0.5497216884074748" "ele:218:jdIUkcxVllwKxk" "0.15158869373843586" "ele:219:lfaHJBQSZxAXoQvQFlPigiVHkZgTuMVws" "0.1471601248923734" "ele:220:ErtZpvDixJoWzYaZeDGLNlyvHj" "0.34543282445022816" "ele:221:T" "0.19990737182076168" "ele:222:YxJjmoYLFozIIwrMYOlTxxQMVnnoRNfMJZNsoFkRgtRJxjWLqanXK" "0.9235704327462902" "ele:223:jknpDgOcYsFJkAdHes" "0.028979981957491407" "ele:224:PecWcubMjFo" "0.4819827930201377" "ele:225:BzYNIyOhFAXYdTkiFMVahLnUBpDiNVtfzHr" "0.7450560599120607" "ele:226:npTXtRZLkvUdRxzjAJhBMxrmMBvhbEgxkhpoLDwRADuYpsRddDmNuMaPjfPbsnxd" "0.2026432885675793" "ele:227:XQzBnvrfYQFiADkbGOfRNEqMnfSsuknCCmcYObVsVYqHX" "0.33615596635066225" "ele:228:gi" "0.2998550975047304" "ele:229:TZRTennwunsTOdkYpeygbUpdbMWAfSXuHW" "0.6648237145509738" "ele:230:WVURIAfgDtEhYPyFqlwEGmSryqDTKOcmZFCN" "0.9372645320573785" "ele:231:udBlpLQIU" "0.3002401369498503" "ele:232:QZtIphwpqQZgcMUmeqQkrZTzatshryjTJIPQYFWkGRwjlqwjigMtlKMMemuKTbMg" "0.25105472672858375" "ele:233:xUkFKUrwJNybmdNQRGVehzHfOuGElLrJuuAYxGqbIoHoHmVWfsPglk" "0.9789489212458264" "ele:234:qsatMznJjZjwVeWoDwAsB" "0.7078139907480708" "ele:235:GaaGmlTHoEgtKbHiKGIFM" "0.1046069368957796" "ele:236:KEflKyuBnrCIoilhIaARgQ" "0.6933650368323319" "ele:237:enRNBKg" "0.8675947064153726" "ele:238:MKHpkdutXQrLuD" "0.9649700026640867" "ele:239:TsTIfIuABLEeAQIwTfJaeHpVxkWiapFXnQTQMJnvLLZyXlRwItuSNswMW" "0.5263629254265921" "ele:240:ocCdGAupDHWtVayoOIdguBoxLIyDRqsvckyTWcXqmVfOROMPKrRp" "0.7524063653536449" "ele:241:hW" "0.19599955008534597" "ele:242:akVJEnuYoJgcIWAnSADGMZuwUfVbIq" "0.15609750914096354" "ele:243:hEUhanHWOxoScINZOwKZrbYdUOBKHtoOFcZsAIzoMEvCsPJegRaQlXobtXnaautk" "0.7890752169170604" "ele:244:Rl" "0.7120506976992904" "ele:245:hoceJlyQdVWNISvutYhokdHFydfUWljaVjTfrNuUDZnBJdm" "0.7417159545801839" "ele:246:zgQTYWnUeOwnYPvtYaN" "0.6658632542669134" "ele:247:UVi" "0.8541463086909872" "ele:248:ohJEVIwKMTixERomyzuLCjFinZgZmqkMZFRLqijAEiHjHUcQgZ" "0.9446951127019414" "ele:249:CcIcveuZxIuoeAyJMaYiwDMLGzdqYOuPExXhZsuVfcWUzr" "0.15818622003981309" "ele:250:JwxuBJCEMZbMJAbsIAwPwUMWkNOmRMvVwlVnMBWbRkSUaTNGtDCiBfXdqWI" "0.549760006394579" "ele:251:JAhok" "0.04924382329883947" "ele:252:JIgeRfRMaTziVyjIdodjtorGgHTeoiYHoSewGEvcvjsTKiFLP" "0.3823858466872766" "ele:253:bjZbtanq" "0.40286373652099416" "ele:254:WlWolwQXSRSZATDgsfdGLuRrOFPaG" "0.3864202824447852" "ele:255:iQNzZfsdAUMpqXRpOyuqJJQfPzhJLcfnstKeqpNyEkUBUsx" "0.30087671810763006" "ele:256:uulbqpiGBlgInmlFKRxUAOlXCLZcRCGyQyEHaXFOEvQai" "0.7410145944905596" "ele:257:GyPpWZoEMASkcgOY" "0.7996004149665447" "ele:258:zZNxkTNMqRlGsYahKYsgsPDCDcIsgVDNUrQijkcJxgGHrI" "0.5402362797766433" "ele:259:iqxwwQCxMUAMJEHVpXEwwN" "0.6362128978606277" "ele:260:MUAqsNaJGfPRaOwkBFQVwLBufjgdIyhusYTPwrmWiyRseJCPODBlzb" "0.4996926196597391" "ele:261:oFUZGTyOAQdfnVCGeoCQqJUibzagyMTmdeKMFjsObfORKkudkIzevOqz" "0.42924714893927196" "ele:262:dNzGqFyCVjeHaWeWqoYQfxBbzdUIJCibOSeoXoyceRCPNNJnoyMQeFnQWlk" "0.5322722081965099" "ele:263:cPLVSPyNeYmivofbOXAZERBvCvJBXvthUlMVOAuPzgjuQNmKDBLSBDqsDesYsv" "0.8777291326872585" "ele:264:KJllfZUWKBMJrwxXBjktUrOHwrtTWOuAVpkssvPjgowDHUNheWZHOQJEYuFszrbl" "0.7072711055085867" "ele:265:EkzJFgpBpqiwSpoXFwcjZljQJDkJeQkbdfcdtGYEhGShlPFXwKnBx" "0.20436439828556952" "ele:266:FqpERXPFhMjCNGCeeyGMpFTCicoKSruuqBdZeeGJKPtKktFoamFo" "0.3472173471264659" "ele:267:YhmHpuDCLfD" "0.18021513778229503" "ele:268:uOiGyqEpsyATQiRwtVvJFRrzSzClsNyDHiaywBXAihdvBdpnWtoffjdOoKyh" "0.5522539745259306" "ele:269:iBiK" "0.8385123045630564" "ele:270:zUvpgWWoOAzbaSrFBvznXoKtHkcAmPwOqivMlixNxq" "0.33714769519639143" "ele:271:MsWzfaaReeRzwqWjGACzUicwZtlEFPrltDAXO" "0.4969992294244491" "ele:272:aANoKjfryoOpWqAhpytocmUHCsWxIAYrSwtiiTxw" "0.44636036938360946" "ele:273:clnfwugmPurOKJqMjxn" "0.27285660039441184" "ele:274:atrzvlsfzPNUvAgptZbMSHbGHhvLQyYUH" "0.500326800049104" "ele:275:RuYDbMeeKwElAXxvXLjExHtcF" "0.5466675758624856" "ele:276:dCEGKvzaWNWcvHVeMAgoocWMwdJyootwqFPMHGKKUMPxCH" "0.2018109903813089" "ele:277:eDuQGFBnossxyIbspBTooBfr" "0.014311547995801921" "ele:278:AYJqzOLPgChWlpdtcaZwHIxLRoMiXelshNtOnDMIvtPJioFxddicddrxj" "0.6159669102868971" "ele:279:ujQJI" "0.9039994047199217" "ele:280:fHiRITtiScqaUlJZhyifQIAaAVHfXWmgHMlyQwOlrNxvpXpRoClYWqndtH" "0.41822330412883957" "ele:281:VRbEUYMjyzfbkRvKYGAxlOjvfrVDWdsfMoNaGNJDUYiNhPnaoYJ" "0.20182675744516632" "ele:282:QLyityIrTBIXSZOdHMheKUACiJNrVJelyo" "0.9536655109799361" "ele:283:UlVmxieouLFT" "0.8985298311311088" "ele:284:QAYdFWfyObPVqDkJoJWakCV" "0.8723763850119008" "ele:285:UqMDNekVComtOxijGYdhNkmRimugdNt" "0.5379751008291339" "ele:286:vzWRFCEvBJrOrxlPIiAxbqDByXGE" "0.37873243809375445" "ele:287:zBrbywqJeVbezx" "0.4779307824595257" "ele:288:DOMbWHcXdPYBWUjyLnqHBgVindbTWXxTzaRcVawIbUqHYFDecXMLc" "0.9342932630637866" "ele:289:UiAnOVrnvvtuyTBABOkaRFNIkHfogJCNrNTKXrqcC" "0.034545001035145906" "ele:290:Fz" "0.40461815986202543" "ele:291:klzKqsrhNAxHPnQUjeQXfSQMIIZwWtAFYYBKuqWhNqUnnZpgRrblAPVtoDB" "0.7300189023460038" "ele:292:YFXCvQGOMKOewMiEilWwRQp" "0.1378877900987362" "ele:293:HMlNQLAKOslfMwwUOqEwwuowDEKZ" "0.9649651643860695" "ele:294:DmNeQumEcMMOMhsBZMrKRNhnDwYXKMm" "0.25366932043182866" "ele:295:LrBATkEONciKHMYPEBRTdGLIqYJolXTgMepcMQlQxbdaXkxMYAIlZhqoayDQn" "0.33409580445339604" "ele:296:tyYcCWTtaQAFidhATOrNnrZAynrppZdOVborlhuWuR" "0.6857753861540998" "ele:297:Q" "0.2846544060829831" "ele:298:Zq" "0.3958936266598334" "ele:299:KhIAswbBjpRQnbKRBRLdWdCDt" "0.6572602342327124" "ele:300:oTBAmFvCABLYQTZWxYnPAUqUmpeOuGMHIwSQiAIxUu"' +tested-commands: +- zunion +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="ZUNION 2 zset:skiplist:1:300 zset:skiplist:2:300" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- sorted-set +priority: 131 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunionstore.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunionstore.yml new file mode 100644 index 0000000..9fbce8f --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunionstore.yml @@ -0,0 +1,33 @@ +version: 0.4 +name: memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunionstore +description: 'Runs memtier_benchmark, for a keyspace length of 2 ZSET keys. We query their union using ZUNIONSTORE that returns 600 elements. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 2 + resources: + requests: + memory: 1g + init_commands: + - '"ZADD" "zset:skiplist:1:300" "0.652540306855235" "ele:1:UaZtxmrKBkxhBqJOlNJlwVUfHVALGkjnUemvubDvbGSVAaaQkXLlsg" "0.8731899671198792" "ele:2:RKlVyoHMTvZcoh" "0.0785627468533846" "ele:3:VZWJQlqLBHicktajowIvkyuaddTC" "0.7688563664469605" "ele:4:YaXTxnRGWhENdaOFuXNAJmx" "0.0033318113277969186" "ele:5:JjfFkvonkksjIfHS" "0.49606648747577575" "ele:6:Q" "0.8234766164292862" "ele:7:MZgqGWzeYWDAlplbxvlzllKR" "0.42687597903639085" "ele:8:AGSAuhePDVmONKmViHovKsiIGSXQZdqX" "0.34912978268081996" "ele:9:ZHIKgOWejxTOcrVCRxztCNqtVFEdqCbowiaROZfbhMmzFlr" "0.15838599188422475" "ele:10:UuyeNtxYcAmkfjtovBmajhGcYvzucpgFjLnqKIw" "0.4376432899068222" "ele:11:cfgHnJGQAHaXHjLIuLWYkDACbxQbywRIQMbuE" "0.5536939917085721" "ele:12:uyZtiyKNdKhFhgNlcdtnHgDFEdBoovVYkyLIpREln" "0.403546330051068" "ele:13:kRUDWaPmeYXeJZhgqKpnroFxUSCQ" "0.8808108541089766" "ele:14:PCGgHbGIRxpzFRfADhupCtBKhknUNyqK" "0.23044636914343264" "ele:15:OaHUUrbZEvSOmYOtbUzxnJq" "0.6404180580960533" "ele:16:qJabjKLYYivniNNtGsbnBzRLzYP" "0.6907030016224224" "ele:17:KJrGsIeQESEPkBdTgUhrQuAUzEfySPhVBoNaWDjOawkRrjyktuOMuBcwp" "0.24900135884008867" "ele:18:bwQzFiMXWxSKtYJiFCSaDWxoAEoNGvFsMoAugjlIcWTRWhXyixemQnagvqL" "0.7928656841898908" "ele:19:NKtwuzbLhLCCPupqgkRsRdGijHmSrnqwdfSj" "0.9331302297178864" "ele:20:TcZibcjyBLAbljRGHjFgwVeVjbeZwi" "0.640319798434186" "ele:21:nsmIvzevkqEJkJTiybCjqiXRjJVMqSqsSaHKxTzJmygg" "0.5239886233297175" "ele:22:DZGleQLgYgAjWUcCOhzqsi" "0.6136266546940706" "ele:23:ZVlOgBjrpZhQuQzJYVLYqnxytasTT" "0.6591139297465682" "ele:24:AFgEEoLcwlGmoWeYLZZJnlilPagiaWdqhItaEMLovOfbaPAgtCeeyjCuEV" "0.4468461455464968" "ele:25:pAjBRwToyaDwhSExhMreHmbgzPuJFPlpVrBbjhkyogmUAQamL" "0.5614231865616031" "ele:26:KyArAdlPTTgxCatXdDQIPgZWHCBZicfkZjCKCbuAuvA" "0.022487789155224203" "ele:27:qOuBflYpFEUWKIfvrIuZYEwJYtSIwSlvegDBDYCUTKnBoRWpwDuBAHgYbTwURCW" "0.8510804209364501" "ele:28:QcjxLOamZbsRzGPlmSvuvBpYmDaooLoY" "0.43460695876638156" "ele:29:WcTDiiDHQwNoubgkihXThvM" "0.5610492156901229" "ele:30:XPGwnyfBmCnhkPlJxhZhPrjvXoGKWEWbqrFvZauVZGQ" "0.7513407089150304" "ele:31:MTXpbeTMlXoxUsCDccRYHeKKfYunLBZuHbLgJmNStiiALTm" "0.09175823221394674" "ele:32:vkAxXiwoCWTBUFUVtpBfpAlILPGaMvUrFaRuBqEgqvWnISaZaPTxiklQtzGS" "0.8586634748104288" "ele:33:wyThKKWAeRg" "0.36952587840155204" "ele:34:cRkudmpGSSMUnPOMhPSzjQ" "0.538356756374977" "ele:35:TsRYebuuDgtamrxVbrxOaCBUGWEaGheHKWgzWsgPxVBfLwAev" "0.5218317459277027" "ele:36:N" "0.658938125456635" "ele:37:mAI" "0.8474868095237909" "ele:38:JrYvYRtaFZldysTlMUVfZfVpHrSzayZGWysedghOV" "0.20143282352735015" "ele:39:WjaRWpmPlWwPuepbXywfDmqIEPWnBh" "0.23967093347793234" "ele:40:ZCLtDpzxZHqWUMzAJOfavhqZrGrPiGZQSBYDPqfyqqYAbGIHuyskvLk" "0.8108325344648399" "ele:41:WhxmpALjwYdCOMwTeHRbFRSFfktMmQUVItotZiICjWfwebl" "0.1322111892750637" "ele:42:dfVVGlvVTMufbfSsy" "0.2285660170875491" "ele:43:DBqEkifRzIyvzTydaSqNrqalAXBZAkccYOuIBFEn" "0.9396586731821924" "ele:44:NyIOQsgOAQevXYKYhGZXjVzTeqPiwjdJhtq" "0.32374056012496966" "ele:45:TQYLPdswKkUntJEjvGWJTWxmVJGdBcbIclOrYzVqqtfFznolqcHYTVj" "0.9080986140709963" "ele:46:IdUdmeLZNHLsCGaVkEdfKMemYRdzhQUqvhuQdXnYchGWXyYRAYZQWkb" "0.3129458198716534" "ele:47:LJFPbCuN" "0.39349745859330765" "ele:48:YTPdFVszdCVVgGGHKqoQxfjU" "0.14704886640549086" "ele:49:fOFwwDNEsxjfpkUrwd" "0.9168814654086035" "ele:50:rJSASiFxeYIINRbbgTBqPbATrcKOP" "0.3100306836090321" "ele:51:VKKOyHveatcnmQR" "0.2370937718635434" "ele:52:OCaIAjJkxSLYaWpMAVjmQoOXoZJMcAO" "0.4941647120371836" "ele:53:neeyLswlmLnZXSCDtFx" "0.269336828778751" "ele:54:UuIrk" "0.12151325035284255" "ele:55:FmCgjvkoPmAcscWZixkzsFgsQBGAmDHqFSTaKcUdfxlfYWu" "0.0035001439463812067" "ele:56:aThlZZWlDITljruiWa" "0.5009568203132024" "ele:57:TBszlGVnMdDvMOLiCysTjjANuFqeq" "0.4783242512285928" "ele:58:dafCundgmRFJLzTEexYHeprmCXFjxrJYLZAcsLXhTrgC" "0.531664059031722" "ele:59:reqfWtmvctIfWtqVEuaTEGfarDa" "0.47961187976147" "ele:60:CBvTozpAXrLpS" "0.10226598211977789" "ele:61:eGQCebwBiUOPhMfrVRHiThsDeIFzPDEKHczj" "0.721486119508813" "ele:62:nHhoxBCSgFFhJWBhTSVVXBpdFoLKYYKZmWEgvCfa" "0.3044462915617381" "ele:63:AMu" "0.8380361220680647" "ele:64:tQiutM" "0.7791893412340167" "ele:65:jcKQrBTFeuiuHCprihKx" "0.9140067046543505" "ele:66:bHuocqbTLhTHlHKIfDdemFDPwrcYFeJrXXafsATvSmFvKGYKRJYnqFmwKoRuaptI" "0.6755477551341951" "ele:67:sJuZYHZFrVYyTvMdbMZJDuCqUmpRIsEgizBVplIPCkSAtGc" "0.843803368180551" "ele:68:WgcCuSQuSzcvSPOzzXViuXfnOtBSxjm" "0.4683803962889672" "ele:69:CqNaseFeIzhsgZdZDgySGfNawLz" "0.300477038321727" "ele:70:pLiQVPjQuEROtmKceRjEcaiZxMvMSIeailtKgwUeVsgqQUdTUbWg" "0.6782593142253811" "ele:71:bgbVdSCYQstqQQloWsJLCYYrmdvtmKeNXDfWGHvzdyTFuuiMwSxuSZa" "0.09916181882671649" "ele:72:jHWIgQCltkPBgzLWWTfRJlLFrxesFUmaJzfeeKDsFglvPAwNIDCDdlvicM" "0.031870198089671176" "ele:73:YfsobnenpUlKHCXNgKYVeWHNRFEPOsAU" "0.9753495730511261" "ele:74:OHVmNYCiDZFlkmsqhwgc" "0.6796131843206142" "ele:75:CPVsbsqYVeWPVRsTZKTvlcDwHnUzHpZyEchJqxnbrWM" "0.41149806297291536" "ele:76:MGFGt" "0.0635854843880973" "ele:77:xVozQKiljlffdYDherQcdYdEmEX" "0.17349539011413317" "ele:78:SORWOElEDHqZuOvYslqzY" "0.6957717601786134" "ele:79:XNVNKDBXRoFHDAu" "0.514568844593022" "ele:80:GTagGWDGPcixUFOSdZTBaKpiJSPiKhmyricEAMLBwjQyEXmJIZn" "0.5054082476784039" "ele:81:YOxGoUKeShnXhcdvTJQFGTukjSiRZFidGRfkttgXJBeKKnJebZNubpk" "0.5791117949403571" "ele:82:PFfNxLyEMLCXtgEQVpFVGxadSZeOXdaSCaQmDBrMdJLSlIXAnerUpaF" "0.6869490812905924" "ele:83:EDFLipKWqONzJ" "0.5858117779265328" "ele:84:ufGNBddDuAsmqgFMQYqtMrOTUbOiexjEYqLKdPLscwavpbHWCbGvMSKfkp" "0.3328807181469634" "ele:85:kIxgjCASHjQbqrvhxiPozqCtRVebWwuHCptoxBJ" "0.5593178033061493" "ele:86:BHmhWwOEWhCelxVLRsUSiAsSicrsXyAYhbrOWaDtKVVDyVpgTtkbO" "0.9568394270185203" "ele:87:XrsvMeAof" "0.2895493213801318" "ele:88:eVIlCARlsKEygkqngxlVoJIhthE" "0.36404485659899855" "ele:89:tAahXQIJDAbNWOgsSyAUkgwVcIzNvkawniDRxludhXrgdbReUJWKaDjPkLNCi" "0.6751249599564046" "ele:90:zUNDGaygGLVztZyQgSMyevPstQKjDVTMafZnepYqPvANg" "0.4002349143471098" "ele:91:blBkPgUMFnFxBlocRBolScezbONPxsKrGsdsMsiZWrhuRsINONXImqQlQY" "0.4373588125087955" "ele:92:LpfFjgOkMnuixmgfGamVKyPICLQQzNKmdJaJRnflC" "0.8323339473420782" "ele:93:TsvDvTRrfMUEskXqXTEHTKzYcuzaoBTAyKVkcnvvqRTgXItNQwJ" "0.7464672802658118" "ele:94:GpXkYQokvVduxNQfcaSYTSiZsOMxrAlNMtPPdWCLKWWqbeunMDLgkIRRu" "0.15234267491477727" "ele:95:bTapHmxvqZNdGNP" "0.42610519579163275" "ele:96:DllLIwysTAcQqwVPjAGkGEedTflBevgZmdgwMbKpNdKJoWGteGWJwyhrI" "0.05073435890699274" "ele:97:uoKiwzrJXTOqaDpeWknKEXdTkQziOGXjZZikNbaEPwOMFvlKoERaUq" "0.8053618509879708" "ele:98:UYuKZw" "0.21335197746306034" "ele:99:OqRQLePoVHjfSoZV" "0.9955843393406656" "ele:100:s" "0.8638417234689056" "ele:101:uninyp" "0.6170733682662144" "ele:102:lAEvUqfinjrCuTdqJeZpMOqTGvxPeGRyPlQgwunbDdbTEwCsIiUsteGjmwH" "0.21067098312505894" "ele:103:cLsWBIpwTWpKBrmSXb" "0.028805240455834213" "ele:104:ySsyBzpsJRyLIAkqneYVrUXPSZVHjpEYkuPt" "0.011280501394645426" "ele:105:LoyMrzfbpcBTBvjgSaalQslVYEsvOjWxPFeLL" "0.8115524648082592" "ele:106:PemidyGhQD" "0.035339107659025926" "ele:107:udmOOXuqRqJxINvXvFplPC" "0.4088305270182697" "ele:108:KpDIy" "0.7970215729800464" "ele:109:ADZJJiLuNsGIaWjDtovm" "0.6539941469274039" "ele:110:wkRPjlYtkmmRJGsaKTTlNqggqMJEqPziZOGtDlMoEQBFcJZgJSTJdwEAv" "0.6393759361590676" "ele:111:SqLVSBeiOrFcbKmUUqeFvXcyXmfPuAFnRgaiFHPUUhlHHNkzbGjeIPVJS" "0.2775184026673655" "ele:112:NCSCTxgpSSGUSVzcWWZVcGlEwoVuKkjwzMYPOTBjvHHgTxuwPjLOpJCTpQnK" "0.8279369714332658" "ele:113:VfSDdgYSpSeXVhbfntnUfUchciRBrJBZDrvxMPhOiIp" "0.8297173264793561" "ele:114:TXarLyioSIyrUKkOeMXvaJvJaknKVQeaNErnRoufOnArsovay" "0.921999914094598" "ele:115:luAfTJtIHOjwkvrAmjILtyLShtdxPljZptJJQVhhAhCzzBe" "0.1408651634335023" "ele:116:SEpIqRVTaiuoxSCVlaVHjClWDwMEqaBUPMjoEAxzzQgylhwdbYkJu" "0.9948615128480278" "ele:117:ruBCNLrVXV" "0.28579196360737913" "ele:118:rVSIXyykbQnnnQEPddYhrTskBJVTwatBoNFCIyajGejYyDzK" "0.48423755726111095" "ele:119:xgSdRSFMaNILA" "0.7626249612557783" "ele:120:EwYCAzoAXXaXkVnxrtBULVzEoRUc" "0.8752501969266744" "ele:121:LfegRMqEenhHURSmftuFgmOXRtOKcVsTpWqqQxuybwdAWBJduQZxCNyWCz" "0.18280846923814642" "ele:122:GRqtlaLHKmwlLyzsVuSRZX" "0.37855655211354067" "ele:123:YyuHDEqKXwhJkmKKLIRMCbiwrrldSooerdtGFSAAQfWLg" "0.5528763818820261" "ele:124:ynUYqHxGTImWvrHkzVWNKxYvllmfKI" "0.027904528470665824" "ele:125:ICQAYRVcxiqABLuLUSZrsDtCQCTFbMe" "0.47753076845502873" "ele:126:ycQETDJKxyOXEnXzAlytRgOusgLNkd" "0.8606084882797624" "ele:127:tvmUFgtvmjdxLkTEFnFHLGxhLDSPAUfHRhKLSWYbqa" "0.2695904518559701" "ele:128:BaHcXnCofYiGMaxujBsmBX" "0.825149665561988" "ele:129:JBKxSCR" "0.5873452590489512" "ele:130:YGoDxkbekHCOlecubsBeKjmWezcLEbksCtLLRLzxtzvjWrsmbzNLYKehEP" "0.4853159409847696" "ele:131:IZcFPOMzPHcFIHxowCHSTFrkz" "0.5277987567294383" "ele:132:ljcFEIPIpqvYkqtuTMtTXjbBLQOPRHHrqup" "0.9746883646151202" "ele:133:fYcqTpqdzPZANfWuvHIarLYgYsOUoqlQnKPw" "0.42010203003852076" "ele:134:IZwUCcFtBWLXQmMtlMPwa" "0.6802068264362765" "ele:135:mOEPkdtczWkOlMuGGAFAQSJSAzSguNtlkLNVLRdriYltynDBrE" "0.5605617357048859" "ele:136:TaqJicNvVjnCZNKPJuuiStVfruAlkGgGVaTnMbvCbSoZitGWn" "0.4780678427736166" "ele:137:qtBqNJpcFllvQXdoISfeQrTfcKyuMBqWRKWnmzPpTweQyOZBTQQdpog" "0.9561513047181356" "ele:138:UTOEvbGjBMjYZiBs" "0.8298720638792384" "ele:139:gKXkpmgiTrUqh" "0.6737117765393265" "ele:140:EkktDpJsFgJjhKWTaxldBFqqTNBzdownDwxUZIxYveC" "0.2239791912985728" "ele:141:dyvMXjLusRKyYuvDFHXfLcIodjkniz" "0.3232124237169193" "ele:142:JuESJkOktNOtmFITnZdMXZqPmexwZpETgaTlFVVzpXct" "0.2473393788924324" "ele:143:cpSyDhnVPmGNLmeBsFJufaWWgGOwIjQaAXNzrhGbJDrpbeqFDQamnLu" "0.6849106786577351" "ele:144:VooyGbMEAigOuQtVaVEckAxrSQWNtypiEaQEBLGTtbRkPZvsAyXv" "0.3646945793371218" "ele:145:wzvoPCzE" "0.4263757643063858" "ele:146:EqOnUtiDML" "0.11877588855760746" "ele:147:pZteEnbVnuAOtHIrRGNqclHBYATQxnBuZGBfmnNVfagNEIB" "0.8313214144471833" "ele:148:vbXuqfTwwpCPGUJQtKQKUroVrfNxuuKdHcgtLOBbGvRbbwLvaWjlN" "0.3552235823133526" "ele:149:wTUOXktMFXeuxGgVsbzVWudxXvVFGsB" "0.8670386139794473" "ele:150:uJTtytjczUmGkvsUJQcqcpovnyjcvYUUbLtJCSClvzhaOSJDjS" "0.36577274012963557" "ele:151:lyfmWPlIeqc" "0.24997912597471894" "ele:152:TOwZZHcrChOptJHLflNWYtMacvyAfMOlgdAdJlBwXJfRRzzQzXkcOWFhGdOnLQX" "0.8930265134635483" "ele:153:CqkyWatRYKqEIoUffGbyhbOYEWXmMSCOFiWtVTHSoGBLKGixPGOAHJSwaA" "0.8844448577667497" "ele:154:PpiojYsguWELV" "0.20663410393014814" "ele:155:ebykSXjGhZUYxPFNVUCNSHGNKXESmbthHLdBMIqxoevIKBNVajvToF" "0.6375009411769629" "ele:156:BKyktVmLOkuhGFnj" "0.7758172657193476" "ele:157:TTyfrulmMejuyeUsOqVpxnZlIUKukfBQzfFiWbJtF" "0.7611282500726224" "ele:158:iHCjJFEPDIKAvxzcmHaIka" "0.6032754373225729" "ele:159:HvqgRWNliaavH" "0.03311820686884104" "ele:160:Mc" "0.34191832438430536" "ele:161:ueTQVZrHBiAnDl" "0.4234716642081342" "ele:162:OnmAvoDKw" "0.5301950080950026" "ele:163:QKCLvxWwXHxJcFCEelBdFtEpefx" "0.022733115513385638" "ele:164:vDBwpjkPsgMLSMqORsEaDUPrKSERewhtkSKeiXSg" "0.7166885759719259" "ele:165:XhWRCtQNRxRocWTCaLeAEPwpaZnkTpuXjLdKbOOfYZKkXaSQHhpSXqKXCPmxgvo" "0.2712986802084326" "ele:166:nYfETiDXlAbQRwHV" "0.5341470637872556" "ele:167:pvPkJe" "0.9127872805617512" "ele:168:FDtkhJUZjwqLpzRKOXrlHswdBRwWosyOEWbIWcexvVQMZfPzBB" "0.047439916319365505" "ele:169:BqzapfPLtCxXpDpwXRUIqhuArCXhVbLDVe" "0.7003965818703025" "ele:170:DRWtKsWQBXPylejYIjweSlEUYHjQ" "0.9295842042213984" "ele:171:rpCOjLwmoPSdRSBgAsvnslHoBRjcjVgs" "0.15337097639157538" "ele:172:NBcujL" "0.35038314902687495" "ele:173:BPFNhDL" "0.2972028976456653" "ele:174:jDLdVxGIuQu" "0.05070784214978197" "ele:175:tGvWuCBnevIuRAMJkJM" "0.8737583830123556" "ele:176:usk" "0.2596502464090681" "ele:177:mLdja" "0.6512549702440724" "ele:178:ADVf" "0.171917523837822" "ele:179:GJznUmhoHwfVUKgyPmTcHBSOeXgzbjYjzGGCVFMRapXYdnLaIrqP" "0.9979397420983357" "ele:180:sxmXzXmdFnmLemE" "0.7014210018416481" "ele:181:wkrprcygzJTIyenjHRmbSlKjPkCFZNCpxEfmfHPtJOvhGnmuTJN" "0.47129995449384954" "ele:182:kLMiOVKqDAySJCleSQNmCpxHiWKsiiHAB" "0.9380196572341066" "ele:183:mAPuAsnTTWt" "0.7985363184110637" "ele:184:nsKyWhvIJRENgInWpMcYtlXUdYlrxAmwPsMJNfceYIisVAWkUHXLzkvtCsWE" "0.30436502305230007" "ele:185:MtIHsabFodqvQfXFMohljmJmjySkBQytRvsqiwaZsvqXiFaSBhCASpj" "0.18301728746159307" "ele:186:XvcnJWRlzUTIReQLYEvHanoPk" "0.6830517248352558" "ele:187:mWPnYVSFpfHXZQQAbfPmEjYhlZBmNbiEtrELg" "0.0573871247075346" "ele:188:fcpBmLgqkyICmxewGKyu" "0.7366175249486746" "ele:189:lqenYUoRTmdXgVZSxipJtAANzywKKADhcLjBITUfqrXNSp" "0.6626013234824965" "ele:190:xLRDyJxCopIvZIRnCKccnsqdcmbaEMqPvhzWLFiqZdcoWnTqVtZjVymzWIAXa" "0.9960600588618732" "ele:191:PRteciWuildZLvmEeTCJmgMllDKoOJhJwixuZLBGlbsQvadWVLfhnlV" "0.727874550086809" "ele:192:DMUrMSntsHqYQYxcyFphyVPUK" "0.6604159554316062" "ele:193:pEGlqTKafZFuSDMDOKpGZxdIi" "0.09721821831280353" "ele:194:tzLNAcTqXdGzNFbh" "0.9791379052279395" "ele:195:fB" "0.2838642126919253" "ele:196:gRtNokZLMzuqJRkScwAjS" "0.6981639965614237" "ele:197:fXBKJwUT" "0.9696961287425543" "ele:198:jWFKhNjahnRsbroFeGsfFxwnqqGnahaOFiWjHZorNvhLTxttVJRJQQOFY" "0.4258376716676413" "ele:199:IdmFRBLBHLVjGL" "0.858169214478561" "ele:200:zNJFLPdwPyNXNQKiV" "0.3927031503235572" "ele:201:dRj" "0.237872815865668" "ele:202:pjmoJAOJJUtJvqbsZUaiVJejdqIsiDzdeIfbtDenOfIecKOUiGkQ" "0.11542254177692224" "ele:203:MafcbGHWHjuSFeJnbsmwZSey" "0.5318198291186923" "ele:204:JemvwOIgWnBOsPGsStj" "0.5340725029771403" "ele:205:RTZNBRvrFlroSePZmaZhFiwiLhIVW" "0.17377941268306563" "ele:206:XdkL" "0.30708537669522173" "ele:207:OHUUxtNrNFWGGmCwQXGRCrmTkCIpAXGhVkOrYQ" "0.3642470025832414" "ele:208:qaAxqlRyOUGZKMGUVYKUMOeKTxAJGhWvQNzobjTpgHJmqA" "0.2580334519092651" "ele:209:uyUOGLOzNJScaHhOnWPlsEXonSluZJiTMejoBbmhCBiNPnykKcFLRCPYLQNh" "0.2624285220413499" "ele:210:MuwzwnrcuJwdIKLcWMSbnw" "0.34635829232927273" "ele:211:xyJm" "0.10681423588864192" "ele:212:yovowsSuNWRajWXYNf" "0.8816729069830225" "ele:213:DsOEyomgaAPysPtXOHUsAHRjoaYFAGjkYpfAuzWUyIisUuxhSYgueB" "0.11729073608149843" "ele:214:lYWIYclvxtRgJWlvaqwRxdu" "0.2873790683625198" "ele:215:yvgaPYEzVRGwEYEXRGaAcslGqMMCbCbCCfQxuduv" "0.03381283119846146" "ele:216:oEcjbUMKXHVmheaVDplZJhlzwvohIGSBtimMPkYCwHlVNCEUudhptbDT" "0.6756254954730867" "ele:217:lgkJrJMgdFDLqKbxLuZKuNcIbnzinlnEPO" "0.41449552533526235" "ele:218:zlXNUWHqaCTqefHAyhhsbpjIBwPbRgyKkbNmrxPuRmIspZFJcCavcTIxfvoWxA" "0.493707932686583" "ele:219:BpeNnAZtparUSxZapRlQrmheeSYAwUrmoV" "0.5902863264146183" "ele:220:sKaQfmuFTtIoTWSpOM" "0.5253670876810722" "ele:221:lthVVlxBYtYfhKWbKlmespgrYSqooRsxfEhsJfknVIXRlqxTsxoZiFTDpsxXijW" "0.3979963963224349" "ele:222:dypPPwhZYSSZdLLbaTdqNJvxTbMzLOqbaTsqZoplptkmeVfCoyaUzA" "0.3264162174828448" "ele:223:rWRQxKaqsIHKEIndbSWaRsbtxdLKGmbjIySumpCBTVAGl" "0.5865194234543977" "ele:224:TVWmksLFBdwalaCrcQZRsxBP" "0.28559508093149" "ele:225:lweqdVnCultPdfA" "0.4390884403874875" "ele:226:uwOenAghOaAIzmwlOXrRUTAUnWekan" "0.08146792623238486" "ele:227:NxjScuRQyNcFnWqkRTHLolqrxZRekUBUPtr" "0.56411402327654" "ele:228:TVolwVnFHikQlXWOlzKqZfbSvacowfOABsdeVJLkDPXYmGl" "0.05018525036598798" "ele:229:puOxHskikfnTicBCMzJBLaQnrayTlKspWQu" "0.9726447630383682" "ele:230:SoZUJqIJBKsKgJSEShIwONErhhRgBUqJoqecwqgJrgRkRcidMfLaPBzrpBY" "0.2609811554039797" "ele:231:B" "0.4312484747031583" "ele:232:DoZogQEBRkqEfEzGrnAY" "0.13663455127121937" "ele:233:rYjRcYSynHaSRrVKKJIYI" "0.6922873613563912" "ele:234:bWqKHzRRjENdGkYSOARBsSOwQNLvQbUaKnGxwqaupsxg" "0.26794259830799383" "ele:235:ChaEtSGzPuSjhDOKWYKLJmAGGjPRafTbILOWDTPjECztaBMu" "0.9354162431701218" "ele:236:wfJvpZanAmkVtByeNbReOTsGHuxwpFNmBUNHHKFZG" "0.06293157585414022" "ele:237:WXTGbAxNRqkrEjPrrutKPvGufVmhfdxS" "0.14584091869740912" "ele:238:BaAdncxLcTxXSQrjaDePSS" "0.5768086947048872" "ele:239:NshApWUBkEmPuKNkJcEzdNHGwOzPNbtsjJqwFiRhDGfLJKdpwWIqhomLbV" "0.1465696901238831" "ele:240:jUeWhVOTNZsjeoNuEEiLveNn" "0.8008567772804235" "ele:241:bItAchwIsfNZwxpXMsVzByShOaeidVgPUwHcYlKWbRcdmqVvBDtsr" "0.015881745702420402" "ele:242:zdZZudYQqwmzIdtwriWwnnSBRUjOMxgMyBDFlgifIjPCtRAuFqsIjEYD" "0.708215367473081" "ele:243:YHqmOTBgnTISUeIJJYUYvScqHPhistRICcmpGaVbC" "0.39871686306191767" "ele:244:FhW" "0.9063885465673784" "ele:245:TXtHLEKrnsHVLAUKugMxXYqStLjfOuplXltOeBQJzajnPG" "0.06971324400435519" "ele:246:iBVEpOsIQoRLJNXenfYpXLMMMrLSpjHrFTXbdPvMqMaCrPXSjWwUq" "0.7801801140370873" "ele:247:WQJxHppuKdjrMHEkadNLIBNzbbl" "0.791869958747193" "ele:248:sWCJjeZLjQRozg" "0.11309761860918266" "ele:249:plATLnqtFtSgqqQKdVmdClWzznco" "0.44101158627489334" "ele:250:VojKDXEzXkTK" "0.8836040369092594" "ele:251:RBpRJjERYngmXMJfhQnnCyTEZWJoqVnjeTcqzFzzMlXmvATdihbjl" "0.6308249820355207" "ele:252:UZTymvHpshovPSgGjIDJOxJdjwghvyTBZDHkVQO" "0.5006090429638539" "ele:253:uFpUOJOzJZphVtPaDLJMsyyYwjwFjMRGpDgZe" "0.20079043114588013" "ele:254:zKkpUbLcsdyoLl" "0.8872601871594847" "ele:255:icXC" "0.37517625327876525" "ele:256:nwZmgjIJHB" "0.4472302965803041" "ele:257:CEZcDflAQTNZdzObNbrwLtEqjNgkWbcLcXuZQXJAEqyoGymOYqppuJJjyOPEoPFQ" "0.780663262913547" "ele:258:NbjitZzbvvAnffFGnTBhYqkZRbQlPVvc" "0.6114790546701429" "ele:259:EZYOaVLeDkKLGytYZrxLGRzr" "0.18378642318141092" "ele:260:pWYbNbgRlWPddxsjKHmOW" "0.9576760507667561" "ele:261:sZzqVDT" "0.46086383365401096" "ele:262:mGJfvnLGBwbINxDCEXIRtHdmXyqPlOQhvyvmQTGaHAwZfqLsLjJppkdDdMwwsBv" "0.5751175337887499" "ele:263:NmAoaFQUsjupBVUJSiQbUzexEMKCLVHVrRpJrPzlndc" "0.7387122963781291" "ele:264:YWtBJMQbuojBcfwCHTfh" "0.7526407515005629" "ele:265:ILLeKkqt" "0.8982229649976201" "ele:266:udmQBnKTaKXjlaHyfAvffWdNXIKAMPMbIfVNZMCbMUwvlhNCVRJpMsoU" "0.13915355988685996" "ele:267:TG" "0.11838397899318787" "ele:268:LjoaredUaqedrhRQLJViXILOffWJlDgNaCGhROnZIig" "0.2720595275779307" "ele:269:OLvPf" "0.05213699817329576" "ele:270:TGXeaegcvaWaexsIWJODTEudunbCJWjOnfmKszg" "0.9760555384384879" "ele:271:fCQccupsw" "0.1826479936554769" "ele:272:bTCJyfrkZoFKqxNdCsTLxNAfmXPRCSEdNDrMRlvKCRFrjjPuJxlNFkKnaNQ" "0.1816006773234322" "ele:273:SSicxIfZQttLnNPbzDMvxzfSFhnPHAkfFPeclMecFQvnjLmCQpEwjJiq" "0.644883358870358" "ele:274:rpLDqOGlO" "0.5993794681833459" "ele:275:EbFjBeehfQxqNpOQMUUtGvrJKkZVWxWFJxFCKV" "0.6528913316681816" "ele:276:ewiSyYmEjbtTf" "0.5686335211517553" "ele:277:jeVTPCizAMtXIkuTchJUxkYLeTYQwOneymekogMzIvRxzTDzIMLSvujA" "0.6181915174781923" "ele:278:LFxy" "0.14900559309428074" "ele:279:hfghiprQqSviaaVAiIFKekzwQjWhlkZZlVprXxbaqgHYmnFzEOuTtFwBStUfB" "0.8338524744347505" "ele:280:hqwGMGZVWelRXBEphXvTmwXMeVvJc" "0.771379497890405" "ele:281:HVrnrXGygZsIYancOKsmUaLEkPoUpxQRPG" "0.5994072231684636" "ele:282:lIqDYWuyipeWAOAdBQDuTvsKys" "0.534524220798845" "ele:283:kKTDdgRCCxfiPBundKdDyieG" "0.7242139542264654" "ele:284:ni" "0.748944785839734" "ele:285:PdcemqxiBQyHNkhLpTObryItAXvDpRjjmVHDikyedhTeYAhNwQKMHeVcfApk" "0.13370039714229587" "ele:286:FYwSoEUHBhDBsrJMdzpluc" "0.16028766133142947" "ele:287:aDKzQKcuBfiyyuqHEFHLRRCLRamOUARk" "0.026062274014891695" "ele:288:huXBuVvepspZbb" "0.22018183573052252" "ele:289:AnWyUVUBaTyVmRaUcLZWmStRhKeHAgVckxyFojKTBF" "0.07982614064135407" "ele:290:tgggRahsumg" "0.3697557070272254" "ele:291:rksriVmscvrYXzIYayeHJdvPTaGQCdqXnev" "0.09422878535627355" "ele:292:EABGSXaJ" "0.8325274594683421" "ele:293:CgCfRcqeCzNAHINpiUDuMeBeRLRdcd" "0.9693896699692307" "ele:294:sSatYuMLPzLp" "0.5311474696918715" "ele:295:YAaIFIfsiogCsuvVUgrst" "0.39224110837672066" "ele:296:mEHNOjSErtEsZOtlBJPzCRcXmftzwRPNTBVKYtqVoGZssx" "0.8176954488229201" "ele:297:XUBXZiCSdAUJpFEGVzozodrzsDkFYgezysediNYbMkBGfEsqCqY" "0.6137094587670581" "ele:298:zRtQCUMmJkbIxcWFeJhlrnwvhDOcsCTuQmIUIhiDugArfHxbMMwyyUJtxqhirG" "0.7719959033122741" "ele:299:ebFOLQRWlTZoaDHGTjjtKivqSh" "0.37933435410799776" "ele:300:gZjmUQgmBiqNfiPsOCrjlfeTlEEbOfBVrAwqvO"' + - '"ZADD" "zset:skiplist:2:300" "0.09262750297586564" "ele:1:or" "0.9383651132213807" "ele:2:awUEuWVMajnpzOiOTArpuiSQekbfhVWn" "0.5214936071707239" "ele:3:dLvVjtXgfYwUEOxXFQmag" "0.9079214934215405" "ele:4:wMOfziikZlxOokdQjWovNdHdjFFsDAdJaTsZoQmsnEZarl" "0.9067368627115772" "ele:5:HsmxitflvwMNNhEtFbLO" "0.5918503322692709" "ele:6:ksWvrVzSlLItxbntLMhsJiwEZtZoTefTLdTPucwVBLNEHDkVSBNaNCawnVHq" "0.3360474296101915" "ele:7:VmqzHXRmiQzEsGlfdl" "0.7021374853906152" "ele:8:bNt" "0.40311940256418133" "ele:9:HXRaRKNjkdKs" "0.14791692909385812" "ele:10:PWxaeVvawgdqBTonQoBXTylfhnrLAHXJiHtNQdqQOdTr" "0.5499073150836501" "ele:11:WwqfjEXLLihgdcDuddksFuHVAwFvZgfuVfOeNPHFADpJMNySPsVyNt" "0.6739165529304422" "ele:12:SCtBitWxpz" "0.20235214431676818" "ele:13:cPlWndhlUf" "0.981893336127039" "ele:14:doYXBBDseRsuLWCTFIHAuPUxEducFdUYRQDZfNEvVfuDZXKBdOZnpaOiYW" "0.6255137319908027" "ele:15:vHMz" "0.35225381591612226" "ele:16:FFmOJmOgTObVwKIoXdbQFAkIAThqcmuxgAAOzEFlG" "0.6556189581423885" "ele:17:QMjmHFEILYQBtcbQxsazwEkHG" "0.1086786080705826" "ele:18:aQJgvxcBkRtfGQAMgWYhsnhRuHvWeXOzx" "0.4443262581173478" "ele:19:yzixcbOpFNbGrZTlyAlOFrSPvpkkWqrGodYsZWeipzXpHeZ" "0.03407764418162318" "ele:20:vStWKgLIlsakjbFJd" "0.6036090393945417" "ele:21:jEekc" "0.25987832126438726" "ele:22:KpsJAyUDNdNQzsfSkMDfHZ" "0.22654291425478534" "ele:23:tqI" "0.24666995271407322" "ele:24:LGIGPhYgkxfwbKmXWqvPLbYeHhkFUCLFsnOdnLHYtGyjzZkiIdY" "0.4267692300380157" "ele:25:eXcwFSZJCrCCFJynzpgdQDyMWoXDYwCcfTmNbWkcfbk" "0.6293648111766171" "ele:26:EAMIUZvUsAkDhOgAZOylhhQZNyhPurbXSCdaoTCEiZbKxTzXmEZuoQXLz" "0.9918223572334939" "ele:27:IJviLAjBfTjpnygkNme" "0.5400327173421172" "ele:28:WjVMTKGMQhBtwExljSYstaaoWHCwKTBAjwzUbbOAx" "0.4391638034400295" "ele:29:GaTkBmQtkABFNYXgRXBFQYrngrXCuUjLGzLdoOMSANzbAXGN" "0.5248632352049256" "ele:30:wruIHqSfNHqhQdoWyUrkwToqZjh" "0.4245142889842648" "ele:31:QNrjPwqaFhwvzIvRnDrShNDqVvGHHIDnQddMSDyEdxDMIjZvuQSTtcTERRpqDf" "0.6630427890269389" "ele:32:XsvmtdTG" "0.7021725631889002" "ele:33:zXNKoihjyQxKbVTFCsoexA" "0.5991014496086998" "ele:34:xMAdMecdzpMwLOKLsmVM" "0.4167006541149544" "ele:35:XNPlwYjUbEO" "0.21748473627901788" "ele:36:AfNMHPOCbeAksrJDhTMgToaloyhwouVaw" "0.40372213029096016" "ele:37:ELpdvJAfmUdBcPCnZSCXObXTOWLgiEXnEMlOTGXXPCnswBjbHdxjeBHg" "0.6140401066542219" "ele:38:OHhcyIKEemBnEXrQVcRdZhcKkoGOdHWXNCMzSKusoLHENoHXujdS" "0.7550574793380168" "ele:39:MSYpiWjfKhvpfkBSPVCYgjDKdrN" "0.8269371540962509" "ele:40:cfZEjxhlfRgVZfefeVAPlBHpeJxQQwykwaZjj" "0.7674080844005543" "ele:41:CCLbStaDdBZaTUnJPidjwSbEZeeoNigsjnUQDZnjVnWa" "0.036595137191906546" "ele:42:xiULvOJbNphDvlpwSzdoexRlCXtEnlvAUrkppFBlddyJXwaElEcJMPRoqObnm" "0.5698074730307818" "ele:43:SEvLWl" "0.3369129481509787" "ele:44:VcOjsKDJyinLQIRrVnKCWPjgjbDuNzjiyweNxESMKbrQbugYHsIiPoknumhPR" "0.6422106412803937" "ele:45:IAmzgSECLDxqgd" "0.6894083278054212" "ele:46:PcglhQAZxSfSMxJUMwnDvEzlyrZpPxsNyGBujHQvzddmusSaTHtpAkpdzMsNtg" "0.6450057709395803" "ele:47:TJPQSHCulWh" "0.3070909482618994" "ele:48:blWTtuxKnoRamyEtyRbQjJbYYCRaseaIXXtvBihOIadnSKMyKaFGwWBy" "0.8598262989702926" "ele:49:IVJoggskzWEkHKovKwoOHOmaKbgIUYdINWmChdmr" "0.10354415035880926" "ele:50:kgcLRlwdZFgFhhIENJIspcjGvllDOVLsvZtUrVTMyWDfRpfVz" "0.28928908386800456" "ele:51:AvQiIRwghGePoNguwjfUmvnNpOrsmeHKvjoIcBlQNnNRRlIrXD" "0.5442717124338943" "ele:52:RCTfhiXHCPBwOxtWxdSywwfLquWKLAgMvnZGJSUvPqYndYK" "0.7687123004251277" "ele:53:ZuqUsKCutHPBSYKZBuZJQZhRVyTcYMnUzXFFimIJyzfiJNWn" "0.5345711507419348" "ele:54:sXejHVSkPzasBIBzbBwJNOrmKibrflyQWueOY" "0.9658634461828092" "ele:55:YQPZRMGbtDEpPBEofYZMKKrVHKJkpnXEsmqh" "0.23019227710204926" "ele:56:CufakjBGhcZTOWLtUgznxEDhuvNhFqpBzIrGJtdDouAjRFhLWvQSlFu" "0.6068781157494676" "ele:57:onqdsVasiTWusQQCdCswLqp" "0.8301688570711953" "ele:58:GCsxXdadpevs" "0.5694294722761978" "ele:59:poyUfSRMFmwfErZPJUHlgHlubdBXtQhSJGEpiQRHXfNPHFoVFoWJmrGzNhOP" "0.4493023274389891" "ele:60:k" "0.37642031836089995" "ele:61:YQPaAXgTZiFgtsookRhfWWLMVumXoBodaVNJcHPwb" "0.2228297094226499" "ele:62:ejYqfmTTyrYYvLxPDAse" "0.14198066739390958" "ele:63:fsXUMHeAEejToaWjEhTcrmWMbFFdTRSXUOtGWpEbomwlbiOCxjBkDuargUNurtSL" "0.5233946066825867" "ele:64:TezvDO" "0.4878495944197222" "ele:65:HzsWRNuQBNjMJMoxIfjjlfczTcgfWDJfWlKysFAM" "0.017359009875882236" "ele:66:qvHuOqbAosEcdJqW" "0.4003374590717428" "ele:67:MouLyXUsVzEjmjlLCKqEyGBfCsnenlcvPoFWsERYzTvMNfkLKSgKW" "0.23253292632629075" "ele:68:VjFxenLXwgrkQEAbqklWpggsOMMYeLzxTECqbwwGVwhUGZimqAVURMsxzxm" "0.646901662020837" "ele:69:xColhsZjMEXgAKAKOWKtkoJTUGxRp" "0.8839172533047469" "ele:70:lfmulumeTiacynAyztlI" "0.31940112981806046" "ele:71:OfhTwuZNKxrQLEPWPWLaqNseTfoVbEgBZWqSRCf" "0.9457823004980779" "ele:72:yXkYmdBegJjXwgFpWcZnQnEnonqrUKwLxrpydrxuALbVqLbdQEMeuhQNvYJYCmNL" "0.42349649546528356" "ele:73:nqITDNFxSZCgTrGAlsGR" "0.1491128401167493" "ele:74:wSlpcsEWUAxEfdWdGarbiyZBnHqqXMLbabkR" "0.7164223539226408" "ele:75:aIhaMQxatzemPJXL" "0.45353981831873524" "ele:76:PKyZiUhHjWxRavredJTVpxPfWiSIyw" "0.5055761893027574" "ele:77:fCwJdIVTYUcRpxyVtvgrpOShBVKvuDfnezaEicBSaEkufAYjkbPQZLdmpqA" "0.579969466017893" "ele:78:ttgVNrKMzdhyWKM" "0.37257323396198194" "ele:79:tqhZoSrQzrhOrsVgBimIhUxxIpHCpVgAxXeUEQuQAmwwT" "0.12062537296309861" "ele:80:riTBhsaejufrrtAtSqOowA" "0.42813583015090506" "ele:81:ahRppAcoxMSVLt" "0.6486347611701347" "ele:82:cA" "0.034587858704615115" "ele:83:qqrNxMSTddFARf" "0.7593627828586745" "ele:84:FplVKLwuuOafeOeFSItktKgiy" "0.4994696285128135" "ele:85:tIYVAAegmAqMASVk" "0.11891388133869885" "ele:86:YEmCqaoJjXbJltAOoiFnVQNDuVacCaLlOFaeetedotVui" "0.3894622762499911" "ele:87:JdrKgsAPlek" "0.5336569471727015" "ele:88:hcCtXGaLVHWCgMUlYzUKhduXYLemTyQhEYVFg" "0.920251905287605" "ele:89:jnvoEEFVGKKoOjzUtvfoXhNeTetuiJtaDGEnITEBIFuhPaX" "0.968897738066963" "ele:90:qlwTOlqYcziWhbvBMBTVIgCVFnqpVaPNCTpmSPt" "0.9465629005814513" "ele:91:GOVRIARgGyaAIHPwkeEZzMAnqSyTxFfoqWgM" "0.6059363868454227" "ele:92:SfHzXZvuqNU" "0.7358711669072902" "ele:93:gHeGpNWExpYDynWGcoehLcJEHRAhDZfpHzZAABtY" "0.002391740771283568" "ele:94:uwSSATCqTETlbTuEsmJXbgXAEUltRFNuLGTpMbcNQCwWYt" "0.7158149076298387" "ele:95:HwAliXptdbfTvaAVTOOULUEJyTutsHqnXCWaRTTefOWSWHphznqRhpbJZWs" "0.6038934139572911" "ele:96:VFWrpCzknVyyvPRlvcMNurUszGhKYAJFotTWlsGXFBepexNzhWU" "0.6222780454322215" "ele:97:ehNF" "0.9901469156377757" "ele:98:KnywilEdvSsJWLFHbSZCdGdzwCwqRzZgADKsixcQ" "0.3538196101694284" "ele:99:oBPoRiKlTEbPlNjGYQDBLvUkfozTGdFuP" "0.979768652983662" "ele:100:sFiKSSRMgkIQRuwTcWHjKShPvZdfYHKha" "0.0038377506742498513" "ele:101:Res" "0.2662937612113552" "ele:102:TyAW" "0.2792638772861509" "ele:103:HwgGPitqrKxuhVWgFcnBJPIODYPXtajLeLQxumaekHVdlYfqbmkFnkgiwGAa" "0.1781892660507567" "ele:104:LPuxUVFjLm" "0.04705661819915241" "ele:105:TSouAycARbZOqJrgTwNyjjXrbeXKdVtUMFNswZU" "0.7902202191071308" "ele:106:HPWXuahhazgiqzysESJYkoArGYVcQXJgzOCEUPdOWFFIVJXunbCGGtqqntblixFU" "0.4481460935563768" "ele:107:BJDIpNXqOQWSamkceKuigPXGOLEKPOBSpUHxeoEsqzSvIidEdvcqgYiSVG" "0.3432861842856956" "ele:108:taUgdwQZKdCCodjpsjrhHpANRzpisggAPIUbbNYHEtuvE" "0.42832042624818045" "ele:109:AAOSEmNLVtKbrdVrroPeNRxnxbpnJoTlq" "0.30812383447047176" "ele:110:mXRaSwDGEuPrfoIULekfXFDKYsIzmilkIGYTCeJKwbZhQJrfgQtxFhBARoWJZ" "0.7522373944538988" "ele:111:GgBlqTMGaGyqOFRisSUHUTfCiuUL" "0.21562431158684814" "ele:112:CvwkTqOoXtDtu" "0.8428189114630859" "ele:113:EcWUGnCkfXINMaIgGKkgScPZGIvWuPStFXSAMeOxjXeRptoefDUeIuyJPzRBzRHi" "0.6823372379983094" "ele:114:xzIoKVgZNBRQDQfiUF" "0.13935362874792767" "ele:115:KrpPjoVMOyBmuQqpqT" "0.4332332406044198" "ele:116:UOWNObnoXoaJKzh" "0.08138617832932371" "ele:117:rVhWPfDBDxvwJE" "0.30356446861651587" "ele:118:ZSLbxvYgeGGGiLDww" "0.32067439081280513" "ele:119:bQcKjlcGGQNPqeXDxJDnthZflDwBZEnSosGoXcpgkUACLAMGxw" "0.40934598601386774" "ele:120:cLkEBEgUnFyVcuJGoGcPNYJwilsQjBAw" "0.6433179848407005" "ele:121:nnnQlvic" "0.33121455232187447" "ele:122:NzwulvsTLXrMthvgborfBhEsWPizqgwnTzw" "0.8803242323217817" "ele:123:k" "0.5024021684133856" "ele:124:QxzGwrBCzyjrxScubOiOrGJcTzePOFsPhEHFEzOxkkZCwuUirhETCcjIdoWyA" "0.025065994225742627" "ele:125:jqExxEpycRHNbe" "0.043490094554752834" "ele:126:nareUMxBfzRlJVTvbylndDpEgHoZsZbNDBVInjychmAraHPfaekwZzAwglFw" "0.9816097986428053" "ele:127:IQzEwdcKwNBYqI" "0.6572929124981133" "ele:128:stSHqZwAsTzOsZddsAFwHbfEEsfdII" "0.9613433392746732" "ele:129:dEUztMXSn" "0.6147262075260899" "ele:130:gFwBVpGOtjMuDGVuYRfsFqxLozUbKfIbSnouPpkXziBPLipkyBEToMPgNO" "0.4051062573367158" "ele:131:iYlahFoWqUpvpi" "0.34363782060223536" "ele:132:QzxFJqb" "0.8359405554401546" "ele:133:SwhGBfDsmAFlWxKEuCfBNxdff" "0.22939148700606238" "ele:134:cllbmiogUFJoscJuKiaWvMONpxrWLfuRGMCpvGNFpyMxPJSLvZP" "0.8744608124944555" "ele:135:sYvQFgPAzBKmDspUtlvJdlQVklBHKOFUIHrRoSXgBAVQ" "0.5959086484602159" "ele:136:LPnvKvJYBFLAeCCOQECNeyQsTrPMxvVpqKLjGxKgUBEAhE" "0.9540265436675929" "ele:137:gRldWgnHoDZdu" "0.7669071517114534" "ele:138:DXyqifsGEzBQZNReTCmUGmtUGRjSnYMRjxFpClo" "0.4318348989018863" "ele:139:KtyUQh" "0.39059761540419147" "ele:140:hauqIjYbSoFtMcOOKixBAeGacoockSdTvWdHcqTKUXobKk" "0.24983864913879872" "ele:141:uqZhAekgMEGuxtmrZqmRmJVWoUhhPlyhlghNGmWMFRuxFmFptejHsXs" "0.3210435235914324" "ele:142:CTIJViEGArQbStmcRofVExLPAzMWhDyWQJXDque" "0.18980206607687544" "ele:143:MEcsFtRZtouIWHjYfCXwstdYUpDRmiTQjUkmQDnWi" "0.15361329127026002" "ele:144:dLhflQgp" "0.7110669295479786" "ele:145:meAopAZzKlFlTBIeOETzpEGUjWasknpzyPDfbfugugPOMy" "0.471301584307086" "ele:146:c" "0.453076241949232" "ele:147:hQjzXuyJgtetPOWOxMhUvNJXQbxshZmnSasQsaCEbZglftDYEaNwwoUm" "0.32693695754886076" "ele:148:XbqSlCTEBnEqBzLKGsQ" "0.35856995662107494" "ele:149:EhjPJWQnDvkGAejPAthUUuD" "0.31262357180080613" "ele:150:mHxeZoWhEz" "0.005845222671651107" "ele:151:IyvMEPnFfHSenczOjGODbXJwtMXkuaZFVLntXwtL" "0.9235968098260897" "ele:152:lrpVmOGFgBpPRFjMGZIkJJYPKUKZqRFrUpLqyKTEJBEoleEgCSJZaRxHWXZe" "0.13738091090283422" "ele:153:ntihnyZWbefEPtNiSeQqlVEbqaMSSoCvwZuySkgaTpEVBuaYibjXKCpPPdogPY" "0.8819919106825659" "ele:154:ihNNulMflfjrXcnmwbdTKmwRznOAipcJoPIelWDxaJYtmcixuGWxBsksfIxUWHAg" "0.8932991308660218" "ele:155:segCTllUZbjnrkOODLouJuJXiuAzOvoDcPCggmyMTVzzgudrvwVDX" "0.476578059810815" "ele:156:hDedhNThJbQzAhswOyXSdRPMolSQswuUCeTDlWaAjMtZcBOdoyeBUBxwHAWdlKU" "0.15072385720743042" "ele:157:rDjSR" "0.2161277707711633" "ele:158:ZvNhxpHnEUXOdTdxfRDh" "0.9488145816867642" "ele:159:Q" "0.6787840316052304" "ele:160:sQGndmhzUNtmrDtBzsErELxzfgipMunWyBNABPLhAtCAqbVOXtpPlCeDd" "0.7009800982563948" "ele:161:UwGgQFuNoYhVasdZvJjKEOMcVZJWd" "0.3795646206727531" "ele:162:GdsUALhOYcMICnYiDiYkZVvwhH" "0.017890110952071048" "ele:163:zBqByAKlPKPPOC" "0.5533082346819248" "ele:164:bAdZPWtNjtYoGHdqR" "0.23109286509501725" "ele:165:iUJtSYmwqDRGlMsJREewzEHlvuWaMZPJTCvvgFryO" "0.4713040256603368" "ele:166:wjxQPETQlCQXrRLRTDQGmdTzLlKoUeyMFIRsaNQFhyaJbyXJmlCOSCsHvp" "0.049974994424209385" "ele:167:DGPbIzVMMLUWWjVsevouwaypQhbJyMxjwXsjOkIxTtrtKimvOVC" "0.25554197482909946" "ele:168:yFeyxFl" "0.06546972032558007" "ele:169:OJFPPOAfhqYDChfIsKgmwiaTfDXTUbtEWR" "0.16747563632090212" "ele:170:VfVZvYtxJxcvgOFxPjc" "0.8154729106809464" "ele:171:DmJQkcRAyjgHBMCAFkTucwWQvAICMvtSPXxCZYjBTsgaYxjBEcZeEViUboQA" "0.7721499842095603" "ele:172:KeAwetETxSKBInxxSgcjApglCEEjeapcyAxjKWxWmVUxgedBRvOAuRkXjUZdKqw" "0.5689791211786035" "ele:173:lvGjXnJSwVVaxoNr" "0.8855889473005578" "ele:174:CNroyxiGNciqCZmloBvziBxnjfGcrzWdskAMghlNPyIJ" "0.41912320182786467" "ele:175:ejxnnFYYFrbQVVVTxeLjYeVXWrDZopvmgeBDSHQOPevqXslinL" "0.8035165431612967" "ele:176:VdJifmoOOSJzPOfmpOvrCLFkmtNEmEZqF" "0.6869604080666722" "ele:177:btsGNcuRBzmY" "0.3848414466927291" "ele:178:hnmrVTAOGxtrolmLuvMcDzgBDYEnsOOZvUnOtkxLYPFv" "0.28812018831830133" "ele:179:PXvDrJNIcQFbzQuiYokqPt" "0.11921245346059317" "ele:180:aCufiJDTEkX" "0.024182370939839104" "ele:181:HgnVa" "0.04754239386647818" "ele:182:wosItJxaLEdvLzTjjydyvGqJLUSqT" "0.07505696070559253" "ele:183:rpCPWChrqFvgVEVVKRoWVwOJhVQPTAcEYFz" "0.7810121297050087" "ele:184:VpxhuPTEzZtxYRTVjwNczKXaBkoxhJgbKlLoyVawHuGXunvkJk" "0.7319508963422978" "ele:185:MOwjbZkEMvyOrYFexoyxKFsbexmXqqozGMidaWfES" "0.8182970116273036" "ele:186:RmWeqjsbMYAKOXIpflkbOOiMxRYNrREUFWnBsPywsUTjpRszeVD" "0.13763305291470818" "ele:187:jOsLqpJ" "0.7680871215735682" "ele:188:eFmdPITqTFFhxQFDMxrArrRQNz" "0.9520468889430076" "ele:189:ivzhdLabpxFhrXVumNABm" "0.3334255618135925" "ele:190:jHaWvedsjhAxDTNlXtGMqEggBjmkTARXefpU" "0.02094320460632315" "ele:191:jkMJkvC" "0.782568781581072" "ele:192:CBTuXWkjDEGmCnMoTpvLYZMWbgTHYPzZjmtmXFLRiqeOmTOjZHNjoQo" "0.4130369581381168" "ele:193:IejPDoXOfwXhIBVbWApoqnjm" "0.05032810808761934" "ele:194:GpXjiMttIGORqGmnmKEbUhbvJDtcSEgpwgBqAshNgwuXcoXVzWmN" "0.03562147096755652" "ele:195:TdlepDegXuyVNCYDWpcgWmmTCWTdpdEkxDyduMSOuPdmEdUyMaPepfMgMJQYqfxJ" "0.16058179698958497" "ele:196:dvezkSgvqUSe" "0.19543066541353393" "ele:197:PdrxoChVVQKLNBzhjwNDCbfpZ" "0.543247204423523" "ele:198:cjeiZuqQvFu" "0.33635134143621626" "ele:199:lhZJcssulOevtNgyWxxoNpLvfezhbqgWFJTIDmyxIBzC" "0.5115911058102967" "ele:200:ulngahGBnWlOYgQMBGpxDaupccQfMtbOEPON" "0.3379695196719398" "ele:201:bGtRDNQXwIrwETejqWbnluhGIkIPZpdYmmPTXeowpyneLwHa" "0.2728034878865264" "ele:202:SyQzBNVlAeZKPeRPriqO" "0.5368846611030937" "ele:203:IygYZUGEDlgWTfOebFnhJxOaUAKdRMrimlQOHsvYWd" "0.4539679929781939" "ele:204:HEhoVrszrpmkBDlXjPNFaOrfr" "0.3006834739749523" "ele:205:aROjCcIvSVppcmZvuNFDktThdjDGdHWoAowxMhnBSzmlhwEXzXnGFxINjSfybr" "0.0448639195084084" "ele:206:Tl" "0.5049616444247743" "ele:207:kuhdxIfvoPQppaUlJhUBCtlJzgsxFqelgwmsCbbLv" "0.28294877456995937" "ele:208:zDvaPScfjOwDEfAPsjYAdI" "0.6174513829147165" "ele:209:vqsNtI" "0.18252715540611175" "ele:210:QETJHAoEwrcgWgqMvtpSkxOJIPWKGfDksckQjLy" "0.11630546595702906" "ele:211:SfWoguwffTGTlWKSwrDbLKF" "0.3606182682742908" "ele:212:MbKFUFyFmLLTCfEFgdC" "0.4926337206305448" "ele:213:mdDZaxfdkurWkYykEOMYkPx" "0.6000623954063804" "ele:214:ynHVzFwNLgrrVXYDuGtrufyCVaOzzRfraXTXmxegsCcPeFTSaIVQZqwXCb" "0.2775621516125364" "ele:215:GkhknjnPJlvxwURrjpueAxLAEvSY" "0.7922460208724172" "ele:216:nnDjHaHZMisFPagOGsyRuWIGmkNlxkmrgkotedJGWAZPMwqAVfhHyR" "0.5889353014735185" "ele:217:LjXgi" "0.5497216884074748" "ele:218:jdIUkcxVllwKxk" "0.15158869373843586" "ele:219:lfaHJBQSZxAXoQvQFlPigiVHkZgTuMVws" "0.1471601248923734" "ele:220:ErtZpvDixJoWzYaZeDGLNlyvHj" "0.34543282445022816" "ele:221:T" "0.19990737182076168" "ele:222:YxJjmoYLFozIIwrMYOlTxxQMVnnoRNfMJZNsoFkRgtRJxjWLqanXK" "0.9235704327462902" "ele:223:jknpDgOcYsFJkAdHes" "0.028979981957491407" "ele:224:PecWcubMjFo" "0.4819827930201377" "ele:225:BzYNIyOhFAXYdTkiFMVahLnUBpDiNVtfzHr" "0.7450560599120607" "ele:226:npTXtRZLkvUdRxzjAJhBMxrmMBvhbEgxkhpoLDwRADuYpsRddDmNuMaPjfPbsnxd" "0.2026432885675793" "ele:227:XQzBnvrfYQFiADkbGOfRNEqMnfSsuknCCmcYObVsVYqHX" "0.33615596635066225" "ele:228:gi" "0.2998550975047304" "ele:229:TZRTennwunsTOdkYpeygbUpdbMWAfSXuHW" "0.6648237145509738" "ele:230:WVURIAfgDtEhYPyFqlwEGmSryqDTKOcmZFCN" "0.9372645320573785" "ele:231:udBlpLQIU" "0.3002401369498503" "ele:232:QZtIphwpqQZgcMUmeqQkrZTzatshryjTJIPQYFWkGRwjlqwjigMtlKMMemuKTbMg" "0.25105472672858375" "ele:233:xUkFKUrwJNybmdNQRGVehzHfOuGElLrJuuAYxGqbIoHoHmVWfsPglk" "0.9789489212458264" "ele:234:qsatMznJjZjwVeWoDwAsB" "0.7078139907480708" "ele:235:GaaGmlTHoEgtKbHiKGIFM" "0.1046069368957796" "ele:236:KEflKyuBnrCIoilhIaARgQ" "0.6933650368323319" "ele:237:enRNBKg" "0.8675947064153726" "ele:238:MKHpkdutXQrLuD" "0.9649700026640867" "ele:239:TsTIfIuABLEeAQIwTfJaeHpVxkWiapFXnQTQMJnvLLZyXlRwItuSNswMW" "0.5263629254265921" "ele:240:ocCdGAupDHWtVayoOIdguBoxLIyDRqsvckyTWcXqmVfOROMPKrRp" "0.7524063653536449" "ele:241:hW" "0.19599955008534597" "ele:242:akVJEnuYoJgcIWAnSADGMZuwUfVbIq" "0.15609750914096354" "ele:243:hEUhanHWOxoScINZOwKZrbYdUOBKHtoOFcZsAIzoMEvCsPJegRaQlXobtXnaautk" "0.7890752169170604" "ele:244:Rl" "0.7120506976992904" "ele:245:hoceJlyQdVWNISvutYhokdHFydfUWljaVjTfrNuUDZnBJdm" "0.7417159545801839" "ele:246:zgQTYWnUeOwnYPvtYaN" "0.6658632542669134" "ele:247:UVi" "0.8541463086909872" "ele:248:ohJEVIwKMTixERomyzuLCjFinZgZmqkMZFRLqijAEiHjHUcQgZ" "0.9446951127019414" "ele:249:CcIcveuZxIuoeAyJMaYiwDMLGzdqYOuPExXhZsuVfcWUzr" "0.15818622003981309" "ele:250:JwxuBJCEMZbMJAbsIAwPwUMWkNOmRMvVwlVnMBWbRkSUaTNGtDCiBfXdqWI" "0.549760006394579" "ele:251:JAhok" "0.04924382329883947" "ele:252:JIgeRfRMaTziVyjIdodjtorGgHTeoiYHoSewGEvcvjsTKiFLP" "0.3823858466872766" "ele:253:bjZbtanq" "0.40286373652099416" "ele:254:WlWolwQXSRSZATDgsfdGLuRrOFPaG" "0.3864202824447852" "ele:255:iQNzZfsdAUMpqXRpOyuqJJQfPzhJLcfnstKeqpNyEkUBUsx" "0.30087671810763006" "ele:256:uulbqpiGBlgInmlFKRxUAOlXCLZcRCGyQyEHaXFOEvQai" "0.7410145944905596" "ele:257:GyPpWZoEMASkcgOY" "0.7996004149665447" "ele:258:zZNxkTNMqRlGsYahKYsgsPDCDcIsgVDNUrQijkcJxgGHrI" "0.5402362797766433" "ele:259:iqxwwQCxMUAMJEHVpXEwwN" "0.6362128978606277" "ele:260:MUAqsNaJGfPRaOwkBFQVwLBufjgdIyhusYTPwrmWiyRseJCPODBlzb" "0.4996926196597391" "ele:261:oFUZGTyOAQdfnVCGeoCQqJUibzagyMTmdeKMFjsObfORKkudkIzevOqz" "0.42924714893927196" "ele:262:dNzGqFyCVjeHaWeWqoYQfxBbzdUIJCibOSeoXoyceRCPNNJnoyMQeFnQWlk" "0.5322722081965099" "ele:263:cPLVSPyNeYmivofbOXAZERBvCvJBXvthUlMVOAuPzgjuQNmKDBLSBDqsDesYsv" "0.8777291326872585" "ele:264:KJllfZUWKBMJrwxXBjktUrOHwrtTWOuAVpkssvPjgowDHUNheWZHOQJEYuFszrbl" "0.7072711055085867" "ele:265:EkzJFgpBpqiwSpoXFwcjZljQJDkJeQkbdfcdtGYEhGShlPFXwKnBx" "0.20436439828556952" "ele:266:FqpERXPFhMjCNGCeeyGMpFTCicoKSruuqBdZeeGJKPtKktFoamFo" "0.3472173471264659" "ele:267:YhmHpuDCLfD" "0.18021513778229503" "ele:268:uOiGyqEpsyATQiRwtVvJFRrzSzClsNyDHiaywBXAihdvBdpnWtoffjdOoKyh" "0.5522539745259306" "ele:269:iBiK" "0.8385123045630564" "ele:270:zUvpgWWoOAzbaSrFBvznXoKtHkcAmPwOqivMlixNxq" "0.33714769519639143" "ele:271:MsWzfaaReeRzwqWjGACzUicwZtlEFPrltDAXO" "0.4969992294244491" "ele:272:aANoKjfryoOpWqAhpytocmUHCsWxIAYrSwtiiTxw" "0.44636036938360946" "ele:273:clnfwugmPurOKJqMjxn" "0.27285660039441184" "ele:274:atrzvlsfzPNUvAgptZbMSHbGHhvLQyYUH" "0.500326800049104" "ele:275:RuYDbMeeKwElAXxvXLjExHtcF" "0.5466675758624856" "ele:276:dCEGKvzaWNWcvHVeMAgoocWMwdJyootwqFPMHGKKUMPxCH" "0.2018109903813089" "ele:277:eDuQGFBnossxyIbspBTooBfr" "0.014311547995801921" "ele:278:AYJqzOLPgChWlpdtcaZwHIxLRoMiXelshNtOnDMIvtPJioFxddicddrxj" "0.6159669102868971" "ele:279:ujQJI" "0.9039994047199217" "ele:280:fHiRITtiScqaUlJZhyifQIAaAVHfXWmgHMlyQwOlrNxvpXpRoClYWqndtH" "0.41822330412883957" "ele:281:VRbEUYMjyzfbkRvKYGAxlOjvfrVDWdsfMoNaGNJDUYiNhPnaoYJ" "0.20182675744516632" "ele:282:QLyityIrTBIXSZOdHMheKUACiJNrVJelyo" "0.9536655109799361" "ele:283:UlVmxieouLFT" "0.8985298311311088" "ele:284:QAYdFWfyObPVqDkJoJWakCV" "0.8723763850119008" "ele:285:UqMDNekVComtOxijGYdhNkmRimugdNt" "0.5379751008291339" "ele:286:vzWRFCEvBJrOrxlPIiAxbqDByXGE" "0.37873243809375445" "ele:287:zBrbywqJeVbezx" "0.4779307824595257" "ele:288:DOMbWHcXdPYBWUjyLnqHBgVindbTWXxTzaRcVawIbUqHYFDecXMLc" "0.9342932630637866" "ele:289:UiAnOVrnvvtuyTBABOkaRFNIkHfogJCNrNTKXrqcC" "0.034545001035145906" "ele:290:Fz" "0.40461815986202543" "ele:291:klzKqsrhNAxHPnQUjeQXfSQMIIZwWtAFYYBKuqWhNqUnnZpgRrblAPVtoDB" "0.7300189023460038" "ele:292:YFXCvQGOMKOewMiEilWwRQp" "0.1378877900987362" "ele:293:HMlNQLAKOslfMwwUOqEwwuowDEKZ" "0.9649651643860695" "ele:294:DmNeQumEcMMOMhsBZMrKRNhnDwYXKMm" "0.25366932043182866" "ele:295:LrBATkEONciKHMYPEBRTdGLIqYJolXTgMepcMQlQxbdaXkxMYAIlZhqoayDQn" "0.33409580445339604" "ele:296:tyYcCWTtaQAFidhATOrNnrZAynrppZdOVborlhuWuR" "0.6857753861540998" "ele:297:Q" "0.2846544060829831" "ele:298:Zq" "0.3958936266598334" "ele:299:KhIAswbBjpRQnbKRBRLdWdCDt" "0.6572602342327124" "ele:300:oTBAmFvCABLYQTZWxYnPAUqUmpeOuGMHIwSQiAIxUu"' +tested-commands: +- zunionstore +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="ZUNIONSTORE zset:skiplist:3:600 2 zset:skiplist:1:300 zset:skiplist:2:300" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- sorted-set +priority: 131 diff --git a/utils/create-intset.py b/utils/create-intset.py new file mode 100644 index 0000000..ca66853 --- /dev/null +++ b/utils/create-intset.py @@ -0,0 +1,9 @@ +import random +random.seed(12345) + +command = "SADD intset:100" +for x in range (1,101): + randint = random.randint(1,1000000) + command = command + f" {randint}" + +print(command) \ No newline at end of file diff --git a/utils/create-zset-300-elements.py b/utils/create-zset-300-elements.py new file mode 100644 index 0000000..a012e44 --- /dev/null +++ b/utils/create-zset-300-elements.py @@ -0,0 +1,21 @@ + +import random +import string +random.seed(12345) + + +def generate_random_string(length): + """Generate a random string of specified length.""" + characters = string.ascii_letters + return ''.join(random.choice(characters) for _ in range(length)) + +for d in ["1","2"]: + command = f"ZADD zset:skiplist:{d}:300" + for x in range (1,300+1): + rand_ele = generate_random_string(random.randint(1, 64)) + rand_score = random.random() + command = command + f" {rand_score} ele:{x}:{rand_ele}" + + print(command) + +