From a47ac526585b253accc390c7a8b5e2de4ac4fe55 Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Wed, 19 Dec 2018 21:03:19 -0500
Subject: [PATCH 1/2] Add postprocess::Function arg to benchmarkpkg()

postprocess is a function that takes the results `Benchmarkgroup` and
either modifies them or returns a new version.

This allows users to inject logic into the benchmarking pipeline,
including subtracting two runs, dividing by N, or other such operations.
---
 src/runbenchmark.jl | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/runbenchmark.jl b/src/runbenchmark.jl
index a02856a..e14e504 100644
--- a/src/runbenchmark.jl
+++ b/src/runbenchmark.jl
@@ -10,6 +10,7 @@ The argument `pkg` can be a name of a package or a path to a directory to a pack
 **Keyword arguments**:
 
 * `script` - The script with the benchmarks, if not given, defaults to `benchmark/benchmarks.jl` in the package folder.
+* `postprocess` - A function to post-process results. Will be passed the `BenchmarkGroup`, which it can modify, or return a new one.
 * `resultfile` - If set, saves the output to `resultfile`
 * `retune` - Force a re-tune, saving the new tuning to the tune file.
 
@@ -28,12 +29,15 @@ benchmarkpkg(pathof(MyPkg), "my-feature"; script="/home/me/mycustombenchmark.jl"
 benchmarkpkg(pathof(MyPkg), BenchmarkConfig(id = "my-feature",
                                             env = Dict("JULIA_NUM_THREADS" => 4),
                                             juliacmd = `julia -O3`))
+benchmarkpkg(pathof(MyPkg),  # Run the benchmarks and divide the (median of) results by 1000
+    postprocess=(results)->(results["g"] = median(results["g"])/1_000)
 ```
 """
 function benchmarkpkg(
         pkg::String,
         target=BenchmarkConfig();
         script=nothing,
+        postprocess=nothing,
         resultfile=nothing,
         retune=false,
         custom_loadpath="" #= used in tests =#
@@ -90,6 +94,12 @@ function benchmarkpkg(
         io = IOBuffer(results_local["results"])
         seek(io, 0)
         resgroup = BenchmarkTools.load(io)[1]
+        if postprocess != nothing
+            retval = postprocess(resgroup)
+            if retval != nothing
+                resgroup = retval
+            end
+        end
         juliasha = results_local["juliasha"]
         vinfo = results_local["vinfo"]
         results = BenchmarkResults(pkg, shastring, resgroup, now(), juliasha, vinfo, target)

From ad36c9817fe8e5ac7cfbf893fe7f613773a12aae Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Thu, 20 Dec 2018 12:53:08 -0500
Subject: [PATCH 2/2] Add unit test covering `postprocess=()->()`.

---
 test/runtests.jl | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/test/runtests.jl b/test/runtests.jl
index 0f8a123..a4d0dd1 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,5 +1,6 @@
 using PkgBenchmark
 using BenchmarkTools
+using Statistics
 using Test
 using Dates
 using LibGit2
@@ -73,6 +74,21 @@ temp_pkg_dir(;tmp_dir = tmp_dir) do
         end
     end
 
+    @testset "postprocess" begin
+        PkgBenchmark._withtemp(tempname()) do f
+            str = """
+            using BenchmarkTools
+            SUITE = BenchmarkGroup()
+            SUITE["foo"] = @benchmarkable for _ in 1:100; 1+1; end
+            """
+            open(f, "w") do file
+                print(file, str)
+            end
+            @test typeof(benchmarkpkg(TEST_PACKAGE_NAME, script=f;
+                postprocess=(r)->(r["foo"] = maximum(r["foo"]); return r))) == BenchmarkResults
+        end
+    end
+
     # Make a commit with a small benchmarks.jl file
     testpkg_path = Pkg.dir(TEST_PACKAGE_NAME)
     LibGit2.init(testpkg_path)