Randomize the order of benchmark suites by default

The intention behind this is to try to account for things that might put the
runtime into a state where it has major effects on specific benchmarks.  After
randomizing, we should see variation from run to run rather than e.g. having the
potential effects only shown in specific suites.

Author: polytypic

lib/cmd.ml

@@ -70,12 +70,26 @@ let build_filter = function
            | exception Not_found -> false
     end
 
+let shuffle xs =
+  let xs = Array.of_list xs in
+  let state = Random.State.make_self_init () in
+  let n = Array.length xs in
+  for i = 0 to n - 2 do
+    let j = Random.State.int state (n - i) + i in
+    let t = xs.(i) in
+    xs.(i) <- xs.(j);
+    xs.(j) <- t
+  done;
+  Array.to_list xs
+
 let run ~benchmarks ?(budgetf = 0.025) ?(filters = []) ?(debug = false)
-    ?(output = `JSON) ?(argv = Sys.argv) ?(flush = true) () =
+    ?(output = `JSON) ?(argv = Sys.argv) ?(flush = true) ?(randomize = true) ()
+    =
   let budgetf = ref budgetf in
   let filters = ref filters in
   let debug = ref debug in
   let output = ref output in
+  let randomize = ref randomize in
 
   let rec specs =
     [
@@ -119,7 +133,14 @@ let run ~benchmarks ?(budgetf = 0.025) ?(filters = []) ?(debug = false)
           `List
             (benchmarks
             |> List.filter (build_filter !filters)
-            |> List.map (run_benchmark ~debug:!debug ~budgetf:!budgetf)) );
+            |> (if !randomize then shuffle else Fun.id)
+            |> List.map (run_benchmark ~debug:!debug ~budgetf:!budgetf)
+            |> List.sort (fun l r ->
+                   let name_of = function
+                     | `Assoc (("name", `String name) :: _) -> name
+                     | _ -> failwith "bug"
+                   in
+                   String.compare (name_of l) (name_of r))) );
       ]
   in
 

lib/multicore_bench.mli

@@ -180,6 +180,7 @@ module Cmd : sig
     ?output:output ->
     ?argv:string array ->
     ?flush:bool ->
+    ?randomize:bool ->
     unit ->
     unit
   (** [run ~benchmarks ()] interprets command line arguments and runs the
@@ -205,6 +206,9 @@ module Cmd : sig
       - [~flush]: Whether to flush the standard output after writing it.
         Defaults to [true].
 
+      - [~randomize]: Whether to randomize the order of suites or not.  Defaults
+        to [true].
+
       Command line arguments take precedence over the optional arguments.  In
       other words, you can specify the optional arguments to give defaults for
       the benchmark executable. *)