Added ZUNION, ZUNIONSTORE, and extra ZADD benchmarks. Included SADD benchmarks with intset underlying encoding

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.222"
+version = "0.1.226"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

redis_benchmarks_specification/__compare__/args.py

@@ -123,6 +123,12 @@ def create_compare_arguments(parser):
     parser.add_argument("--simple-table", type=bool, default=False)
     parser.add_argument("--use_metric_context_path", type=bool, default=False)
     parser.add_argument("--testname_regex", type=str, default=".*", required=False)
+    parser.add_argument(
+        "--regression_str", type=str, default="REGRESSION", required=False
+    )
+    parser.add_argument(
+        "--improvement_str", type=str, default="IMPROVEMENT", required=False
+    )
     parser.add_argument(
         "--regressions-percent-lower-limit",
         type=float,

redis_benchmarks_specification/__compare__/compare.py

@@ -285,7 +285,7 @@ def compare_command_logic(args, project_name, project_version):
         detected_regressions,
         table_output,
         total_improvements,
-        total_regressions,
+        regressions_list,
         total_stable,
         total_unstable,
         total_comparison_points,
@@ -328,7 +328,10 @@ def compare_command_logic(args, project_name, project_version):
         comparison_target_branch,
         baseline_github_org,
         comparison_github_org,
+        args.regression_str,
+        args.improvement_str,
     )
+    total_regressions = len(regressions_list)
     prepare_regression_comment(
         auto_approve,
         baseline_branch,
@@ -355,6 +358,7 @@ def compare_command_logic(args, project_name, project_version):
         total_unstable,
         verbose,
         args.regressions_percent_lower_limit,
+        regressions_list,
     )
     return (
         detected_regressions,
@@ -393,6 +397,7 @@ def prepare_regression_comment(
     total_unstable,
     verbose,
     regressions_percent_lower_limit,
+    regressions_list=[],
 ):
     if total_comparison_points > 0:
         comment_body = "### Automated performance analysis summary\n\n"
@@ -423,6 +428,14 @@ def prepare_regression_comment(
             comparison_summary += "- Detected a total of {} regressions bellow the regression water line {}.\n".format(
                 total_regressions, regressions_percent_lower_limit
             )
+            if len(regressions_list) > 0:
+                regression_values = [l[1] for l in regressions_list]
+                regression_df = pd.DataFrame(regression_values)
+                median_regression = round(float(regression_df.median().iloc[0]), 2)
+                max_regression = round(float(regression_df.max().iloc[0]), 2)
+                min_regression = round(float(regression_df.min().iloc[0]), 2)
+
+                comparison_summary += f"   - Median/Common-Case regression was {median_regression}%% and ranged from [{min_regression},{max_regression}] %%.\n"
 
         comment_body += comparison_summary
         comment_body += "\n"
@@ -561,6 +574,8 @@ def compute_regression_table(
     comparison_target_branch=None,
     baseline_github_org="redis",
     comparison_github_org="redis",
+    regression_str="REGRESSION",
+    improvement_str="IMPROVEMENT",
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -665,6 +680,8 @@ def compute_regression_table(
         comparison_github_repo,
         baseline_github_org,
         comparison_github_org,
+        regression_str,
+        improvement_str,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -697,7 +714,7 @@ def compute_regression_table(
         writer_regressions.dump(mystdout, False)
         table_output += mystdout.getvalue()
         table_output += "\n\n"
-        test_names_str = "|".join(regressions_list)
+        test_names_str = "|".join([l[0] for l in regressions_list])
         table_output += f"Regressions test regexp names: {test_names_str}\n\n"
         mystdout.close()
         sys.stdout = old_stdout
@@ -750,7 +767,7 @@ def compute_regression_table(
         detected_regressions,
         table_output,
         total_improvements,
-        total_regressions,
+        regressions_list,
         total_stable,
         total_unstable,
         total_comparison_points,
@@ -919,6 +936,8 @@ def from_rts_to_regression_table(
     comparison_github_repo="redis",
     baseline_github_org="redis",
     comparison_github_org="redis",
+    regression_str="REGRESSION",
+    improvement_str="IMPROVEMENT",
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table_full = []
@@ -937,7 +956,7 @@ def from_rts_to_regression_table(
     regressions_list = []
     improvements_list = []
     for test_name in test_names:
-        compare_version = "v0.1.208"
+        compare_version = "main"
         github_link = "https://github.com/redis/redis-benchmarks-specification/blob"
         test_path = f"redis_benchmarks_specification/test-suites/{test_name}.yml"
         test_link = f"[{test_name}]({github_link}/{compare_version}/{test_path})"
@@ -965,7 +984,7 @@ def from_rts_to_regression_table(
             "triggering_env={}".format(tf_triggering_env),
         ]
         if comparison_github_org != "":
-            filters_baseline.append(f"github_org={comparison_github_org}")
+            filters_comparison.append(f"github_org={comparison_github_org}")
         if "hash" not in by_str_baseline:
             filters_baseline.append("hash==")
         if "hash" not in by_str_comparison:
@@ -1103,27 +1122,27 @@ def from_rts_to_regression_table(
         if baseline_v != "N/A" or comparison_v != "N/A":
             detected_regression = False
             detected_improvement = False
-            if percentage_change < 0.0 and not unstable:
+            if percentage_change < 0.0:
                 if -waterline >= percentage_change:
                     detected_regression = True
                     total_regressions = total_regressions + 1
-                    note = note + " REGRESSION"
+                    note = note + f" {regression_str}"
                     detected_regressions.append(test_name)
                 elif percentage_change < -noise_waterline:
                     if simplify_table is False:
-                        note = note + " potential REGRESSION"
+                        note = note + f" potential {regression_str}"
                 else:
                     if simplify_table is False:
                         note = note + " No Change"
 
-            if percentage_change > 0.0 and not unstable:
+            if percentage_change > 0.0:
                 if percentage_change > waterline:
                     detected_improvement = True
                     total_improvements = total_improvements + 1
-                    note = note + " IMPROVEMENT"
+                    note = note + f" {improvement_str}"
                 elif percentage_change > noise_waterline:
                     if simplify_table is False:
-                        note = note + " potential IMPROVEMENT"
+                        note = note + f" potential {improvement_str}"
                 else:
                     if simplify_table is False:
                         note = note + " No Change"
@@ -1147,7 +1166,7 @@ def from_rts_to_regression_table(
                 test_link,
             )
             if detected_regression:
-                regressions_list.append(test_name)
+                regressions_list.append([test_name, percentage_change])
                 table_regressions.append(line)
 
             if detected_improvement:

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -1375,7 +1375,7 @@ def process_self_contained_coordinator_stream(
                             detected_regressions,
                             table_output,
                             total_improvements,
-                            total_regressions,
+                            regressions_list,
                             total_stable,
                             total_unstable,
                             total_comparison_points,
@@ -1409,6 +1409,7 @@ def process_self_contained_coordinator_stream(
                             use_metric_context_path,
                             running_platform,
                         )
+                        total_regressions = len(regressions_list)
                         auto_approve = True
                         grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks"
 
@@ -1438,6 +1439,7 @@ def process_self_contained_coordinator_stream(
                             total_unstable,
                             verbose,
                             regressions_percent_lower_limit,
+                            regressions_list,
                         )
                     logging.info(
                         f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}"

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading SETs with 100 integer elements.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- sadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --pipeline 10 --test-time 180 -c 50 -t 4 --command "SADD __key__ 436858 768277 10664 860016 865292 841848 313147 896678 386308 977048 203069 283373 593503 457419 169542 391186 130304 916639 453967 273773 589383 657683 182813 641487 580095 195884 372142 774005 768470 95729 556839 771113 432139 613472 528415 174665 155343 215707 753892 81624 199200 877732 358153 337683 24664 481128 356011 28176 535519 724263 780697 435661 3623 6998 985912 696150 913066 171765 808279 608289 187563 300496 104333 684239 434669 111037 928553 714514 606168 190478 772363 393219 243921 549032 624950 743846 957491 864902 345918 846332 42207 243593 125622 915607 523476 226892 773003 840853 792775 582050 702242 194856 270254 617201 454742 543414 960526 136178 43558 172112" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-set-intset-with-100-elements
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading SETs with 100 integer elements.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- sadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 180 -c 50 -t 4 --command "SADD __key__ 436858 768277 10664 860016 865292 841848 313147 896678 386308 977048 203069 283373 593503 457419 169542 391186 130304 916639 453967 273773 589383 657683 182813 641487 580095 195884 372142 774005 768470 95729 556839 771113 432139 613472 528415 174665 155343 215707 753892 81624 199200 877732 358153 337683 24664 481128 356011 28176 535519 724263 780697 435661 3623 6998 985912 696150 913066 171765 808279 608289 187563 300496 104333 684239 434669 111037 928553 714514 606168 190478 772363 393219 243921 549032 624950 743846 957491 864902 345918 846332 42207 243593 125622 915607 523476 226892 773003 840853 792775 582050 702242 194856 270254 617201 454742 543414 960526 136178 43558 172112" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-zset-listpack-with-100-elements-double-score.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-zset-listpack-with-100-elements-double-score
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading ZSETs with encoding:listpack with 100 elements.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 180 -c 50 -t 4 --command "ZADD __key__ 0.652540306855235 UaZtxmrKBkxhBqJOlNJlwVUfHVALGkjnUemvubDvbGSVAaaQkXLlsg 0.8731899671198792 RKlVyoHMTvZcoh 0.0785627468533846 VZWJQlqLBHicktajowIvkyuaddTC 0.7688563664469605 YaXTxnRGWhENdaOFuXNAJmx 0.0033318113277969186 JjfFkvonkksjIfHS 0.49606648747577575 Q 0.8234766164292862 MZgqGWzeYWDAlplbxvlzllKR 0.42687597903639085 AGSAuhePDVmONKmViHovKsiIGSXQZdqX 0.34912978268081996 ZHIKgOWejxTOcrVCRxztCNqtVFEdqCbowiaROZfbhMmzFlr 0.15838599188422475 UuyeNtxYcAmkfjtovBmajhGcYvzucpgFjLnqKIw 0.4376432899068222 cfgHnJGQAHaXHjLIuLWYkDACbxQbywRIQMbuE 0.5536939917085721 uyZtiyKNdKhFhgNlcdtnHgDFEdBoovVYkyLIpREln 0.403546330051068 kRUDWaPmeYXeJZhgqKpnroFxUSCQ 0.8808108541089766 PCGgHbGIRxpzFRfADhupCtBKhknUNyqK 0.23044636914343264 OaHUUrbZEvSOmYOtbUzxnJq 0.6404180580960533 qJabjKLYYivniNNtGsbnBzRLzYP 0.6907030016224224 KJrGsIeQESEPkBdTgUhrQuAUzEfySPhVBoNaWDjOawkRrjyktuOMuBcwp 0.24900135884008867 bwQzFiMXWxSKtYJiFCSaDWxoAEoNGvFsMoAugjlIcWTRWhXyixemQnagvqL 0.7928656841898908 NKtwuzbLhLCCPupqgkRsRdGijHmSrnqwdfSj 0.9331302297178864 TcZibcjyBLAbljRGHjFgwVeVjbeZwi 0.640319798434186 nsmIvzevkqEJkJTiybCjqiXRjJVMqSqsSaHKxTzJmygg 0.5239886233297175 DZGleQLgYgAjWUcCOhzqsi 0.6136266546940706 ZVlOgBjrpZhQuQzJYVLYqnxytasTT 0.6591139297465682 AFgEEoLcwlGmoWeYLZZJnlilPagiaWdqhItaEMLovOfbaPAgtCeeyjCuEV 0.4468461455464968 pAjBRwToyaDwhSExhMreHmbgzPuJFPlpVrBbjhkyogmUAQamL 0.5614231865616031 KyArAdlPTTgxCatXdDQIPgZWHCBZicfkZjCKCbuAuvA 0.022487789155224203 qOuBflYpFEUWKIfvrIuZYEwJYtSIwSlvegDBDYCUTKnBoRWpwDuBAHgYbTwURCW 0.8510804209364501 QcjxLOamZbsRzGPlmSvuvBpYmDaooLoY 0.43460695876638156 WcTDiiDHQwNoubgkihXThvM 0.5610492156901229 XPGwnyfBmCnhkPlJxhZhPrjvXoGKWEWbqrFvZauVZGQ 0.7513407089150304 MTXpbeTMlXoxUsCDccRYHeKKfYunLBZuHbLgJmNStiiALTm 0.09175823221394674 vkAxXiwoCWTBUFUVtpBfpAlILPGaMvUrFaRuBqEgqvWnISaZaPTxiklQtzGS 0.8586634748104288 wyThKKWAeRg 0.36952587840155204 cRkudmpGSSMUnPOMhPSzjQ 0.538356756374977 TsRYebuuDgtamrxVbrxOaCBUGWEaGheHKWgzWsgPxVBfLwAev 0.5218317459277027 N 0.658938125456635 mAI 0.8474868095237909 JrYvYRtaFZldysTlMUVfZfVpHrSzayZGWysedghOV 0.20143282352735015 WjaRWpmPlWwPuepbXywfDmqIEPWnBh 0.23967093347793234 ZCLtDpzxZHqWUMzAJOfavhqZrGrPiGZQSBYDPqfyqqYAbGIHuyskvLk 0.8108325344648399 WhxmpALjwYdCOMwTeHRbFRSFfktMmQUVItotZiICjWfwebl 0.1322111892750637 dfVVGlvVTMufbfSsy 0.2285660170875491 DBqEkifRzIyvzTydaSqNrqalAXBZAkccYOuIBFEn 0.9396586731821924 NyIOQsgOAQevXYKYhGZXjVzTeqPiwjdJhtq 0.32374056012496966 TQYLPdswKkUntJEjvGWJTWxmVJGdBcbIclOrYzVqqtfFznolqcHYTVj 0.9080986140709963 IdUdmeLZNHLsCGaVkEdfKMemYRdzhQUqvhuQdXnYchGWXyYRAYZQWkb 0.3129458198716534 LJFPbCuN 0.39349745859330765 YTPdFVszdCVVgGGHKqoQxfjU 0.14704886640549086 fOFwwDNEsxjfpkUrwd 0.9168814654086035 rJSASiFxeYIINRbbgTBqPbATrcKOP 0.3100306836090321 VKKOyHveatcnmQR 0.2370937718635434 OCaIAjJkxSLYaWpMAVjmQoOXoZJMcAO 0.4941647120371836 neeyLswlmLnZXSCDtFx 0.269336828778751 UuIrk 0.12151325035284255 FmCgjvkoPmAcscWZixkzsFgsQBGAmDHqFSTaKcUdfxlfYWu 0.0035001439463812067 aThlZZWlDITljruiWa 0.5009568203132024 TBszlGVnMdDvMOLiCysTjjANuFqeq 0.4783242512285928 dafCundgmRFJLzTEexYHeprmCXFjxrJYLZAcsLXhTrgC 0.531664059031722 reqfWtmvctIfWtqVEuaTEGfarDa 0.47961187976147 CBvTozpAXrLpS 0.10226598211977789 eGQCebwBiUOPhMfrVRHiThsDeIFzPDEKHczj 0.721486119508813 nHhoxBCSgFFhJWBhTSVVXBpdFoLKYYKZmWEgvCfa 0.3044462915617381 AMu 0.8380361220680647 tQiutM 0.7791893412340167 jcKQrBTFeuiuHCprihKx 0.9140067046543505 bHuocqbTLhTHlHKIfDdemFDPwrcYFeJrXXafsATvSmFvKGYKRJYnqFmwKoRuaptI 0.6755477551341951 sJuZYHZFrVYyTvMdbMZJDuCqUmpRIsEgizBVplIPCkSAtGc 0.843803368180551 WgcCuSQuSzcvSPOzzXViuXfnOtBSxjm 0.4683803962889672 CqNaseFeIzhsgZdZDgySGfNawLz 0.300477038321727 pLiQVPjQuEROtmKceRjEcaiZxMvMSIeailtKgwUeVsgqQUdTUbWg 0.6782593142253811 bgbVdSCYQstqQQloWsJLCYYrmdvtmKeNXDfWGHvzdyTFuuiMwSxuSZa 0.09916181882671649 jHWIgQCltkPBgzLWWTfRJlLFrxesFUmaJzfeeKDsFglvPAwNIDCDdlvicM 0.031870198089671176 YfsobnenpUlKHCXNgKYVeWHNRFEPOsAU 0.9753495730511261 OHVmNYCiDZFlkmsqhwgc 0.6796131843206142 CPVsbsqYVeWPVRsTZKTvlcDwHnUzHpZyEchJqxnbrWM 0.41149806297291536 MGFGt 0.0635854843880973 xVozQKiljlffdYDherQcdYdEmEX 0.17349539011413317 SORWOElEDHqZuOvYslqzY 0.6957717601786134 XNVNKDBXRoFHDAu 0.514568844593022 GTagGWDGPcixUFOSdZTBaKpiJSPiKhmyricEAMLBwjQyEXmJIZn 0.5054082476784039 YOxGoUKeShnXhcdvTJQFGTukjSiRZFidGRfkttgXJBeKKnJebZNubpk 0.5791117949403571 PFfNxLyEMLCXtgEQVpFVGxadSZeOXdaSCaQmDBrMdJLSlIXAnerUpaF 0.6869490812905924 EDFLipKWqONzJ 0.5858117779265328 ufGNBddDuAsmqgFMQYqtMrOTUbOiexjEYqLKdPLscwavpbHWCbGvMSKfkp 0.3328807181469634 kIxgjCASHjQbqrvhxiPozqCtRVebWwuHCptoxBJ 0.5593178033061493 BHmhWwOEWhCelxVLRsUSiAsSicrsXyAYhbrOWaDtKVVDyVpgTtkbO 0.9568394270185203 XrsvMeAof 0.2895493213801318 eVIlCARlsKEygkqngxlVoJIhthE 0.36404485659899855 tAahXQIJDAbNWOgsSyAUkgwVcIzNvkawniDRxludhXrgdbReUJWKaDjPkLNCi 0.6751249599564046 zUNDGaygGLVztZyQgSMyevPstQKjDVTMafZnepYqPvANg 0.4002349143471098 blBkPgUMFnFxBlocRBolScezbONPxsKrGsdsMsiZWrhuRsINONXImqQlQY 0.4373588125087955 LpfFjgOkMnuixmgfGamVKyPICLQQzNKmdJaJRnflC 0.8323339473420782 TsvDvTRrfMUEskXqXTEHTKzYcuzaoBTAyKVkcnvvqRTgXItNQwJ 0.7464672802658118 GpXkYQokvVduxNQfcaSYTSiZsOMxrAlNMtPPdWCLKWWqbeunMDLgkIRRu 0.15234267491477727 bTapHmxvqZNdGNP 0.42610519579163275 DllLIwysTAcQqwVPjAGkGEedTflBevgZmdgwMbKpNdKJoWGteGWJwyhrI 0.05073435890699274 uoKiwzrJXTOqaDpeWknKEXdTkQziOGXjZZikNbaEPwOMFvlKoERaUq 0.8053618509879708 UYuKZw 0.21335197746306034 OqRQLePoVHjfSoZV 0.9955843393406656 s" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B-pipeline-10.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-append-1-100B-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 1 Bytes for each key initially. Then we append to the same keys with values ranging from 1 to 100B.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--data-size" "1" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- append
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 --pipeline 10 --data-size-range=1-100 --command "APPEND __key__ __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 1