forked from maxitg/SetReplace
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathperformanceTest.wls
executable file
·130 lines (113 loc) · 6.19 KB
/
performanceTest.wls
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#!/usr/bin/env wolframscript
Get["scripts/buildInit.wl"];
<< SetReplace`;
tests = Hold @ <|
"Single-input rule" -> WolframModel[{{1, 2}} -> {{1, 3}, {1, 3}, {3, 2}}, Automatic, 10],
"Medium rule" -> WolframModel[
{{1, 2, 3}, {4, 3, 5}, {3, 6}} -> {{6, 7, 8}, {6, 9, 10}, {11, 8, 10}, {5, 2, 9}, {9, 9}, {1, 9}, {7, 5}, {8, 5}},
Automatic, 13],
"Sequential rule" -> WolframModel[{{1, 2, 2}, {3, 2, 4}} -> {{5, 4, 4}, {4, 3, 5}, {3, 5, 1}}, Automatic, 10000],
"Large rule" -> WolframModel[
{{1, 2}, {2, 1}, {1, 3}, {2, 3}, {3, 1}, {3, 2}} ->
{{1, 2}, {2, 1}, {1, 3}, {2, 3}, {3, 1}, {3, 2}, {1, 4}, {2, 4}, {3, 4}, {4, 1}, {4, 2}, {4, 3}},
Automatic, 4],
"Exponential-match-count rule" -> WolframModel[{{1}, {1}, {1}} -> {{1}, {1}, {1}, {1}}, Automatic, 10],
"CA emulator" -> WolframModel[
{{{18, 18, 3}, {3, 19, 3}, {3, 3, 3, 3, 3}} ->
{{1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1, 1, 5}, {5, 5, 5, 5, 5}, {5, 4, 5}, {1, 19, 1}, {1}, {4, 12, 12},
{4}, {17, 14, 14}, {17}, {18, 18, 17}},
{{18, 16, 18}, {16, 16, 20}, {16, 16}} ->
{{1, 1, 14, 1}, {1, 1, 1, 12}, {1, 10, 10}, {1, 6, 1}, {6, 6}, {6, 6, 17}, {1, 1, 20}, {1}, {17, 14, 14}, {17},
{4, 12, 12}, {4}, {18, 4, 18}},
{{1, 1, 2}, {2, 11, 11}, {1, 15, 1}, {15, 13, 13}, {1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1, 1, 1, 1, 1, 1},
{2, 2, 2, 2, 2, 2}, {15, 15, 15, 15, 15, 15}} ->
{{10, 10, 11}, {10, 13, 10}, {10, 7, 7}, {10, 10, 10, 8}, {10, 10, 9, 10}, {10}, {12}, {14}, {12, 8, 8},
{14, 9, 9}},
{{1, 1, 2}, {2, 11, 11}, {1, 15, 1}, {15, 13, 13}, {1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1, 1, 1, 1, 1, 1},
{2, 2, 2, 2, 2, 2}, {15}} ->
{{10, 10, 11}, {10, 13, 10}, {10, 7, 7}, {10, 10, 10, 8}, {10, 10, 9, 10}, {10, 10, 10, 10, 10, 10},
{12, 12, 12, 12, 12, 12}, {14, 14, 14, 14, 14, 14}, {12, 8, 8}, {14, 9, 9}},
{{1, 1, 2}, {2, 11, 11}, {1, 15, 1}, {15, 13, 13}, {1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1},
{2, 2, 2, 2, 2, 2}, {15, 15, 15, 15, 15, 15}} ->
{{10, 10, 11}, {10, 13, 10}, {10, 7, 7}, {10, 10, 10, 8}, {10, 10, 9, 10}, {10, 10, 10, 10, 10, 10},
{12, 12, 12, 12, 12, 12}, {14, 14, 14, 14, 14, 14}, {12, 8, 8}, {14, 9, 9}},
{{1, 1, 2}, {2, 11, 11}, {1, 15, 1}, {15, 13, 13}, {1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1},
{2, 2, 2, 2, 2, 2}, {15}} ->
{{10, 10, 11}, {10, 13, 10}, {10, 7, 7}, {10, 10, 10, 8}, {10, 10, 9, 10}, {10}, {12}, {14}, {12, 8, 8},
{14, 9, 9}},
{{1, 1, 2}, {2, 11, 11}, {1, 15, 1}, {15, 13, 13}, {1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1, 1, 1, 1, 1, 1},
{2}, {15, 15, 15, 15, 15, 15}} ->
{{10, 10, 11}, {10, 13, 10}, {10, 7, 7}, {10, 10, 10, 8}, {10, 10, 9, 10}, {10, 10, 10, 10, 10, 10},
{12, 12, 12, 12, 12, 12}, {14, 14, 14, 14, 14, 14}, {12, 8, 8}, {14, 9, 9}},
{{1, 1, 2}, {2, 11, 11}, {1, 15, 1}, {15, 13, 13}, {1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1, 1, 1, 1, 1, 1},
{2}, {15}} ->
{{10, 10, 11}, {10, 13, 10}, {10, 7, 7}, {10, 10, 10, 8}, {10, 10, 9, 10}, {10, 10, 10, 10, 10, 10},
{12, 12, 12, 12, 12, 12}, {14, 14, 14, 14, 14, 14}, {12, 8, 8}, {14, 9, 9}},
{{1, 1, 2}, {2, 11, 11}, {1, 15, 1}, {15, 13, 13}, {1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1}, {2},
{15, 15, 15, 15, 15, 15}} ->
{{10, 10, 11}, {10, 13, 10}, {10, 7, 7}, {10, 10, 10, 8}, {10, 10, 9, 10}, {10, 10, 10, 10, 10, 10},
{12, 12, 12, 12, 12, 12}, {14, 14, 14, 14, 14, 14}, {12, 8, 8}, {14, 9, 9}},
{{1, 1, 2}, {2, 11, 11}, {1, 15, 1}, {15, 13, 13}, {1, 1, 1, 12}, {1, 1, 14, 1}, {1, 10, 10}, {1}, {2}, {15}} ->
{{10, 10, 11}, {10, 13, 10}, {10, 7, 7}, {10, 10, 10, 8}, {10, 10, 9, 10}, {10}, {12}, {14}, {12, 8, 8},
{14, 9, 9}}},
{{3, 3, 3, 6}, {3, 3, 7, 3}, {3, 5, 5}, {3, 3, 1}, {3, 2, 3}, {3, 3, 3, 3, 3, 3}, {4, 6, 6}, {4, 4, 4, 4, 4, 4},
{8, 7, 7}, {8, 8, 8, 8, 8, 8}, {1, 4, 1}, {1, 1, 1, 1, 1}, {2, 2, 8}, {2, 2}},
64]
|>;
$defaultMeasurementsCount = 5;
$measurementsCount = If[Length @ $ScriptCommandLine >= 4,
ToExpression[$ScriptCommandLine[[4]]],
$defaultMeasurementsCount];
If[!IntegerQ[$measurementsCount] || $measurementsCount < 2,
Print["The third argument should be an integer measurements count of at least 2."];
Exit[1];
];
Attributes[meanAroundTiming] = {HoldAll};
meanAroundTiming[expr_] := MeanAround @ Table[First @ AbsoluteTiming[expr], $measurementsCount]
runTests[repo_, sha_, tests_] := Module[{result, kernel},
Print["Testing ", sha];
GitCheckoutReference[repo, sha];
Run["./build.wls"];
Run["./install.wls"];
CloseKernels[];
{kernel} = LaunchKernels[1];
result = ParallelEvaluate[
<< SetReplace`; meanAroundTiming @@@ KeyMap[ReleaseHold, ReleaseHold @ Map[Hold, tests, {3}]],
kernel
];
Print[""];
result
]
speedupDelta[old_, new_] := (old - new) / old
$gitRepo = GitOpen[$repoRoot];
$currentSHA = GitSHA[$gitRepo, $gitRepo["HEAD"]];
$cleanQ = AllTrue[# === {} &] @ GitStatus[$gitRepo];
If[!$cleanQ,
Print["Current git tree must be clean."];
Exit[1];
];
$oldSHA = If[Length @ $ScriptCommandLine >= 2, $ScriptCommandLine[[2]], "master"];
$newSHA = If[Length @ $ScriptCommandLine >= 3, $ScriptCommandLine[[3]], $currentSHA];
$symbolicRef = RunProcess[{"git", "symbolic-ref", "--short", "HEAD"}];
$originalRef = If[$symbolicRef["ExitCode"] === 0, StringExtract[$symbolicRef["StandardOutput"], 1], $currentSHA];
{$oldResults, $newResults} = runTests[$gitRepo, #, tests] & /@ {$oldSHA, $newSHA};
RunProcess[{"git", "checkout", "-q", $originalRef}];
$speedup = speedupDelta[$oldResults, $newResults];
$redColor = "\033[0;31m";
$greenColor = "\033[0;32m";
$whiteColor = "\033[0;37m";
$endColor = "\033[0m";
differenceString[meanAround_] := With[{
magnitude = QuantityMagnitude[meanAround, "Percent"]},
If[5 * magnitude[[2]] < Abs[magnitude[[1]]], If[magnitude[[1]] > 0, $greenColor, $redColor], $whiteColor] <>
FirstCase[
MakeBoxes[magnitude, StandardForm],
TemplateBox[{value_, error_}, "Around"] :> value <> " \[PlusMinus] " <> error <> " %"] <>
$endColor
]
KeyValueMap[
Print[
#1,
StringJoin[ConstantArray[" ", Max[40 - StringLength[#1], 1]]],
differenceString[#2]] &,
$speedup];