Skip to content

Commit 30c3e47

Browse files
stanbrubstanbrub
committed
Added nightly best score table. Fixed score issue
1 parent 2167860 commit 30c3e47

File tree

3 files changed

+55
-22
lines changed

3 files changed

+55
-22
lines changed

‎src/it/java/io/deephaven/benchmark/tests/internal/workflow/PublishTest.java

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,18 +36,32 @@ public void verifyPublishQuery() throws Exception {
3636

3737
api.query(q).fetchAfter("nightly_worst_score_small", table -> {
3838
assertEquals("""
39-
Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|ScrProb
40-
AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|0.00%
41-
ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|20.89%
42-
WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|76.96%
43-
VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|94.35%
44-
ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|82.35%
45-
SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|65.86%
46-
Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|65.47%
47-
Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|45.55%
48-
WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|24.50%
49-
CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|0.78%""",
50-
table.toCsv("|"), "Wrong score table results");
39+
Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|Scr|ScrProb
40+
AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|-44.9|0.00%
41+
ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|-1.3|20.89%
42+
WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|-0.3|76.96%
43+
VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|-0.1|94.35%
44+
ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|0.2|82.35%
45+
SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|0.4|65.86%
46+
Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|0.4|65.47%
47+
Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|0.7|45.55%
48+
WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|1.2|24.50%
49+
CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|2.7|0.78%""",
50+
table.toCsv("|"), "Wrong worst score table results");
51+
}).fetchAfter("nightly_best_score_small", table -> {
52+
assertEquals("""
53+
Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|Scr|ScrProb
54+
CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|2.7|0.78%
55+
WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|1.2|24.50%
56+
Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|0.7|45.55%
57+
Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|0.4|65.47%
58+
SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|0.4|65.86%
59+
ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|0.2|82.35%
60+
VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|-0.1|94.35%
61+
WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|-0.3|76.96%
62+
ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|-1.3|20.89%
63+
AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|-44.9|0.00%""",
64+
table.toCsv("|"), "Wrong best score table results");
5165
}).execute();
5266
api.awaitCompletion();
5367
}

‎src/main/java/io/deephaven/benchmark/run/PublishNotification.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
*/
2020
public class PublishNotification {
2121
static final String rowIndent = " ".repeat(10);
22-
final String[] tables = {"nightly_worst_score"};
22+
final String[] tables = {"nightly_worst_score", "nightly_best_score"};
2323
final URL queryFile;
2424
final URL svgTemplate;
2525
final Path outputDir;
@@ -78,10 +78,10 @@ public void publish() {
7878
void publishToSlack(Path outDir) {
7979
var message = "Nightly Benchmark Changes " +
8080
"<https://controller.try-dh.demo.community.deephaven.io/get_ide| (Benchmark Tables)>\n";
81-
// "<https://github.com/deephaven/benchmark/blob/main/docs/PublishedResults.md| (Dig Deeper)>\n";
8281

8382
for (String table : tables) {
84-
message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```";
83+
message += "*" + table.replace("_", " ") + "*\n";
84+
message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```\n";
8585
}
8686

8787
var payload = """

‎src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,25 +74,44 @@ def zprob(zscore):
7474
'prob=(float)zprob(score)'
7575
])
7676

77-
nightly_worst_score_large = nightly_score.head_by(20).view([
77+
nightly_worst_score_large = nightly_score.view([
7878
'Static_Benchmark=benchmark_name.replace(` -Static`,``)',
7979
'Variability=(float)var_rate/100','Rate=op_rate',
8080
'Change=(float)gain(avg_rate,op_rate)/100',
8181
'Since_Release=(float)gain(prev_vers_avg_rate,op_rate)/100',
8282
'Score=score','Score_Prob=prob'
83-
]).sort([
84-
'Score'
85-
]).format_columns([
83+
]).sort(['Score']).head_by(20).format_columns([
8684
'Variability=Decimal(`0.0%`)','Rate=Decimal(`###,##0`)',
87-
'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)','Score_Prob=Decimal(`0.00%`)'
85+
'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)',
86+
'Score=Decimal(`0.0`)','Score_Prob=Decimal(`0.00%`)'
8887
])
8988

9089
nightly_worst_score_small = nightly_worst_score_large.head_by(10).view([
9190
'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change',
92-
'Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob'
91+
'Var5d=Variability','Rate','ChngRls=Since_Release','Scr=Score','ScrProb=Score_Prob'
92+
]).format_columns([
93+
'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)',
94+
'ChngRls=Decimal(`0.0%`)','Scr=Decimal(`0.0`)','ScrProb=Decimal(`0.00%`)'
95+
])
96+
97+
nightly_best_score_large = nightly_score.view([
98+
'Static_Benchmark=benchmark_name.replace(` -Static`,``)',
99+
'Variability=(float)var_rate/100','Rate=op_rate',
100+
'Change=(float)gain(avg_rate,op_rate)/100',
101+
'Since_Release=(float)gain(prev_vers_avg_rate,op_rate)/100',
102+
'Score=score','Score_Prob=prob'
103+
]).sort_descending(['Score']).head_by(20).format_columns([
104+
'Variability=Decimal(`0.0%`)','Rate=Decimal(`###,##0`)',
105+
'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)',
106+
'Score=Decimal(`0.0`)','Score_Prob=Decimal(`0.00%`)'
107+
])
108+
109+
nightly_best_score_small = nightly_best_score_large.head_by(10).view([
110+
'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change',
111+
'Var5d=Variability','Rate','ChngRls=Since_Release','Scr=Score','ScrProb=Score_Prob'
93112
]).format_columns([
94113
'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)',
95-
'ChngRls=Decimal(`0.0%`)','ScrProb=Decimal(`0.00%`)'
114+
'ChngRls=Decimal(`0.0%`)','Scr=Decimal(`0.0`)','ScrProb=Decimal(`0.00%`)'
96115
])
97116

98117
bench_results = bench_metrics = bench_platforms = bench_metrics_diff = None

0 commit comments

Comments
 (0)