diff --git a/src/it/java/io/deephaven/benchmark/tests/internal/workflow/PublishTest.java b/src/it/java/io/deephaven/benchmark/tests/internal/workflow/PublishTest.java index 03b8adce..712fa48f 100644 --- a/src/it/java/io/deephaven/benchmark/tests/internal/workflow/PublishTest.java +++ b/src/it/java/io/deephaven/benchmark/tests/internal/workflow/PublishTest.java @@ -36,18 +36,32 @@ public void verifyPublishQuery() throws Exception { api.query(q).fetchAfter("nightly_worst_score_small", table -> { assertEquals(""" - Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|ScrProb - AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|0.00% - ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|20.89% - WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|76.96% - VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|94.35% - ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|82.35% - SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|65.86% - Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|65.47% - Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|45.55% - WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|24.50% - CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|0.78%""", - table.toCsv("|"), "Wrong score table results"); + Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|Scr|ScrProb + AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|-44.9|0.00% + ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|-1.3|20.89% + WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|-0.3|76.96% + VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|-0.1|94.35% + ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|0.2|82.35% + SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|0.4|65.86% + Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|0.4|65.47% + Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|0.7|45.55% + WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|1.2|24.50% + CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|2.7|0.78%""", + table.toCsv("|"), "Wrong worst score table results"); + }).fetchAfter("nightly_best_score_small", table -> { + assertEquals(""" + Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|Scr|ScrProb + CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|2.7|0.78% + WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|1.2|24.50% + Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|0.7|45.55% + Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|0.4|65.47% + SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|0.4|65.86% + ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|0.2|82.35% + VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|-0.1|94.35% + WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|-0.3|76.96% + ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|-1.3|20.89% + AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|-44.9|0.00%""", + table.toCsv("|"), "Wrong best score table results"); }).execute(); api.awaitCompletion(); } diff --git a/src/main/java/io/deephaven/benchmark/run/PublishNotification.java b/src/main/java/io/deephaven/benchmark/run/PublishNotification.java index 1437e3b8..b7009792 100644 --- a/src/main/java/io/deephaven/benchmark/run/PublishNotification.java +++ b/src/main/java/io/deephaven/benchmark/run/PublishNotification.java @@ -19,7 +19,7 @@ */ public class PublishNotification { static final String rowIndent = " ".repeat(10); - final String[] tables = {"nightly_worst_score"}; + final String[] tables = {"nightly_worst_score", "nightly_best_score"}; final URL queryFile; final URL svgTemplate; final Path outputDir; @@ -78,10 +78,10 @@ public void publish() { void publishToSlack(Path outDir) { var message = "Nightly Benchmark Changes " + "\n"; - // "\n"; for (String table : tables) { - message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```"; + message += "*" + table.replace("_", " ") + "*\n"; + message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```\n"; } var payload = """ diff --git a/src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py b/src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py index 401837d4..a6ec2cc2 100644 --- a/src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py +++ b/src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py @@ -74,25 +74,44 @@ def zprob(zscore): 'prob=(float)zprob(score)' ]) -nightly_worst_score_large = nightly_score.head_by(20).view([ +nightly_worst_score_large = nightly_score.view([ 'Static_Benchmark=benchmark_name.replace(` -Static`,``)', 'Variability=(float)var_rate/100','Rate=op_rate', 'Change=(float)gain(avg_rate,op_rate)/100', 'Since_Release=(float)gain(prev_vers_avg_rate,op_rate)/100', 'Score=score','Score_Prob=prob' -]).sort([ - 'Score' -]).format_columns([ +]).sort(['Score']).head_by(20).format_columns([ 'Variability=Decimal(`0.0%`)','Rate=Decimal(`###,##0`)', - 'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)','Score_Prob=Decimal(`0.00%`)' + 'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)', + 'Score=Decimal(`0.0`)','Score_Prob=Decimal(`0.00%`)' ]) nightly_worst_score_small = nightly_worst_score_large.head_by(10).view([ 'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change', - 'Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob' + 'Var5d=Variability','Rate','ChngRls=Since_Release','Scr=Score','ScrProb=Score_Prob' +]).format_columns([ + 'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)', + 'ChngRls=Decimal(`0.0%`)','Scr=Decimal(`0.0`)','ScrProb=Decimal(`0.00%`)' +]) + +nightly_best_score_large = nightly_score.view([ + 'Static_Benchmark=benchmark_name.replace(` -Static`,``)', + 'Variability=(float)var_rate/100','Rate=op_rate', + 'Change=(float)gain(avg_rate,op_rate)/100', + 'Since_Release=(float)gain(prev_vers_avg_rate,op_rate)/100', + 'Score=score','Score_Prob=prob' +]).sort_descending(['Score']).head_by(20).format_columns([ + 'Variability=Decimal(`0.0%`)','Rate=Decimal(`###,##0`)', + 'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)', + 'Score=Decimal(`0.0`)','Score_Prob=Decimal(`0.00%`)' +]) + +nightly_best_score_small = nightly_best_score_large.head_by(10).view([ + 'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change', + 'Var5d=Variability','Rate','ChngRls=Since_Release','Scr=Score','ScrProb=Score_Prob' ]).format_columns([ 'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)', - 'ChngRls=Decimal(`0.0%`)','ScrProb=Decimal(`0.00%`)' + 'ChngRls=Decimal(`0.0%`)','Scr=Decimal(`0.0`)','ScrProb=Decimal(`0.00%`)' ]) bench_results = bench_metrics = bench_platforms = bench_metrics_diff = None