Skip to content

Commit

Permalink
Added nightly best score table. Fixed score issue (#207)
Browse files Browse the repository at this point in the history
  • Loading branch information
stanbrub authored Nov 1, 2023
1 parent 2167860 commit 0e1a40f
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,32 @@ public void verifyPublishQuery() throws Exception {

api.query(q).fetchAfter("nightly_worst_score_small", table -> {
assertEquals("""
Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|ScrProb
AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|0.00%
ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|20.89%
WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|76.96%
VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|94.35%
ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|82.35%
SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|65.86%
Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|65.47%
Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|45.55%
WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|24.50%
CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|0.78%""",
table.toCsv("|"), "Wrong score table results");
Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|Scr|ScrProb
AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|-44.9|0.00%
ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|-1.3|20.89%
WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|-0.3|76.96%
VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|-0.1|94.35%
ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|0.2|82.35%
SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|0.4|65.86%
Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|0.4|65.47%
Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|0.7|45.55%
WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|1.2|24.50%
CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|2.7|0.78%""",
table.toCsv("|"), "Wrong worst score table results");
}).fetchAfter("nightly_best_score_small", table -> {
assertEquals("""
Static_Benchmark|Chng5d|Var5d|Rate|ChngRls|Scr|ScrProb
CumCombo- 6 Ops No Groups|3.8%|1.4%|30,721,966|3.9%|2.7|0.78%
WhereOneOf- 2 Filters|6.7%|5.8%|325,609,160|9.0%|1.2|24.50%
Vector- 5 Calcs 1M Groups Dense Data|9.8%|13.2%|46,943,765|2.4%|0.7|45.55%
Where- 2 Filters|2.7%|6.0%|990,671,179|-1.0%|0.4|65.47%
SelectDistinct- 1 Group 250 Unique Vals|0.6%|1.3%|57,973,815|0.7%|0.4|65.86%
ParquetWrite- Lz4Raw Multi Col|0.8%|3.7%|2,869,769|1.0%|0.2|82.35%
VarBy- 2 Group 160K Unique Combos Float|-0.2%|2.2%|12,213,326|-0.7%|-0.1|94.35%
WhereNotIn- 1 Filter Col|-0.3%|1.0%|362,236,812|-0.7%|-0.3|76.96%
ReverseAsOfJoin- Join On 2 Cols 1 Match|-3.2%|2.5%|1,933,301|-3.2%|-1.3|20.89%
AsOfJoin- Join On 2 Cols 1 Match|-44.1%|1.0%|1,111,111|-44.1%|-44.9|0.00%""",
table.toCsv("|"), "Wrong best score table results");
}).execute();
api.awaitCompletion();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
*/
public class PublishNotification {
static final String rowIndent = " ".repeat(10);
final String[] tables = {"nightly_worst_score"};
final String[] tables = {"nightly_worst_score", "nightly_best_score"};
final URL queryFile;
final URL svgTemplate;
final Path outputDir;
Expand Down Expand Up @@ -78,10 +78,10 @@ public void publish() {
void publishToSlack(Path outDir) {
var message = "Nightly Benchmark Changes " +
"<https://controller.try-dh.demo.community.deephaven.io/get_ide| (Benchmark Tables)>\n";
// "<https://github.com/deephaven/benchmark/blob/main/docs/PublishedResults.md| (Dig Deeper)>\n";

for (String table : tables) {
message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```";
message += "*" + table.replace("_", " ") + "*\n";
message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```\n";
}

var payload = """
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,25 +74,44 @@ def zprob(zscore):
'prob=(float)zprob(score)'
])

nightly_worst_score_large = nightly_score.head_by(20).view([
nightly_worst_score_large = nightly_score.view([
'Static_Benchmark=benchmark_name.replace(` -Static`,``)',
'Variability=(float)var_rate/100','Rate=op_rate',
'Change=(float)gain(avg_rate,op_rate)/100',
'Since_Release=(float)gain(prev_vers_avg_rate,op_rate)/100',
'Score=score','Score_Prob=prob'
]).sort([
'Score'
]).format_columns([
]).sort(['Score']).head_by(20).format_columns([
'Variability=Decimal(`0.0%`)','Rate=Decimal(`###,##0`)',
'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)','Score_Prob=Decimal(`0.00%`)'
'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)',
'Score=Decimal(`0.0`)','Score_Prob=Decimal(`0.00%`)'
])

nightly_worst_score_small = nightly_worst_score_large.head_by(10).view([
'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change',
'Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob'
'Var5d=Variability','Rate','ChngRls=Since_Release','Scr=Score','ScrProb=Score_Prob'
]).format_columns([
'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)',
'ChngRls=Decimal(`0.0%`)','Scr=Decimal(`0.0`)','ScrProb=Decimal(`0.00%`)'
])

nightly_best_score_large = nightly_score.view([
'Static_Benchmark=benchmark_name.replace(` -Static`,``)',
'Variability=(float)var_rate/100','Rate=op_rate',
'Change=(float)gain(avg_rate,op_rate)/100',
'Since_Release=(float)gain(prev_vers_avg_rate,op_rate)/100',
'Score=score','Score_Prob=prob'
]).sort_descending(['Score']).head_by(20).format_columns([
'Variability=Decimal(`0.0%`)','Rate=Decimal(`###,##0`)',
'Change=Decimal(`0.0%`)','Since_Release=Decimal(`0.0%`)',
'Score=Decimal(`0.0`)','Score_Prob=Decimal(`0.00%`)'
])

nightly_best_score_small = nightly_best_score_large.head_by(10).view([
'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change',
'Var5d=Variability','Rate','ChngRls=Since_Release','Scr=Score','ScrProb=Score_Prob'
]).format_columns([
'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)',
'ChngRls=Decimal(`0.0%`)','ScrProb=Decimal(`0.00%`)'
'ChngRls=Decimal(`0.0%`)','Scr=Decimal(`0.0`)','ScrProb=Decimal(`0.00%`)'
])

bench_results = bench_metrics = bench_platforms = bench_metrics_diff = None
Expand Down

0 comments on commit 0e1a40f

Please sign in to comment.