-
Notifications
You must be signed in to change notification settings - Fork 2
/
parsebenchmarks.jl
131 lines (123 loc) · 4.17 KB
/
parsebenchmarks.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
using JSON
using DataFrames
using Statistics, Dates
using BenchmarkTools
const emptydf = DataFrame((
lang = String[],
label = String[],
dtime = DateTime[],
vlang = String[],
pcommit = Union{Missing,String}[],
op = String[],
ttype = String[],
mtype = String[],
memory = Union{Int,Missing}[],
allocs = Union{Int,Missing}[],
tmin = Float64[],
tmax = Float64[],
tmedian = Float64[],
tmean = Float64[],
tstddev = Float64[]
))
parsepybenchjson(df, filename; label = "numpy") = parsepybenchjson!(deepcopy(df), filename; label = label)
function parsepybenchjson!(df::DataFrame, filename::String; label = "numpy")
pjson = JSON.parsefile(filename)
vlang = pjson["machine_info"]["python_version"]
dtime = DateTime(pjson["datetime"][1:end-3])
pcommit = missing
foreach(pjson["benchmarks"]) do d
name = d["name"]
m = match(r"test_([a-z]*)\[([a-z]*)-([a-zA-Z0-9{}]*)\]", name)
stats = d["stats"]
op, mtype, ttype = string.(m.captures)
push!(df, (
lang = "python",
label = label,
dtime = dtime,
vlang = vlang,
pcommit = pcommit,
op=op,
ttype=ttype,
mtype=mtype,
memory = missing,
allocs = missing,
#python results are in seconds -> go to nanoseconds
tmin = 10^9*d["stats"]["min"],
tmax = 10^9*d["stats"]["max"],
tmedian = 10^9*d["stats"]["median"],
tmean = 10^9*d["stats"]["mean"],
tstddev = 10^9*d["stats"]["stddev"]
))
end
return df
end
parsejuliajson(df, filename; label = "") = parsejuliajson!(deepcopy(df), filename; label = label)
function parsejuliajson!(df::DataFrame, filename::String; label::String = "")
jjson = JSON.parsefile(filename)
vlang = match(r"Julia Version (\d*.\d*.\d*)",jjson["vinfo"]).captures[1]
isnothing(vlang) && error("version couldn't be read")
bmark = BenchmarkTools.load(IOBuffer(jjson["benchmarkgroup"]))[1]
dtime = DateTime(jjson["date"])
pcommit = jjson["commit"]
foreach(leaves(bmark)) do (labels, trial)
op, ttype, mtype = string.(labels)
times = trial.times
push!(df, (
lang = "julia",
label = label,
dtime = dtime,
vlang = vlang,
pcommit = pcommit,
op=op,
ttype=ttype,
mtype=mtype,
memory = trial.memory,
allocs = trial.allocs,
tmin = minimum(times),
tmax = maximum(times),
tmedian = median(times),
tmean = mean(times),
tstddev = sqrt(var(times))
))
end
df
end
@doc raw"
benchmarkscores(df::DataFrame, ref::String)
calculates a benchmarkscore for each label in `df` with the label `ref`
as a reference. The scores are calculated as:
``s_{\text{label}} = \Sum_{\text{ops},\text{sizes},\text{types}} w_\text{op} * log(t_\text{label}/t_\text{ref})``
where the weights can be found in the function definition.
Since missing terms are ignored, it is recommended to preparse the `df`.
"
function benchmarkscores(df, ref)
weights = Dict{String,Float64}(
"matmul" => 1,
"batchmul" => 1,
"dot" => 1,
"trace" => 0.5,
"ptrace" => 0.3,
"diag" => 0.5,
"perm" => 0.8,
"tcontract" => 1,
"star" => 0.7,
"starandcontract" => 0.2,
"indexsum" => 0.6,
"hadamard" => 0.6,
"outer" => 0.3)
fdf = DataFrame(label = String[], score = Float64[])
refdf = @where(df, :label .== ref)
for x in groupby(df, :label)
l = x.label[1]
score = 0.
for r in eachrow(x)
op, ttype, mtype, t = r.op, r.ttype, r.mtype, r.tmin
t0 = @where(refdf, :ttype .== ttype,
:op .== op,
:mtype .== mtype).tmin[]
score += weights[op] * log(t/t0)
end
push!(fdf, (label = l, score = score))
end
fdf
end