bench over time

Quafadas · Sep 27, 2024 · dd59781 · dd59781
1 parent 0ad9c1c
commit dd59781
Show file tree

Hide file tree

Showing 8 changed files with 324 additions and 36 deletions.
diff --git a/build.sc b/build.sc
@@ -1,6 +1,6 @@
 
 import $ivy.`com.github.lolgab::mill-crossplatform::0.2.4`
-import $ivy.`io.github.quafadas::millSite::0.0.31-DIRTY1d54e88b`
+import $ivy.`io.github.quafadas::millSite::0.0.32`
 import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0`
 import $ivy.`com.lihaoyi::mill-contrib-jmh:`
 

diff --git a/site/docs/_assets/plots/countTrue.vg.json b/site/docs/_assets/plots/countTrue.vg.json
@@ -0,0 +1,141 @@
+{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "data": {
+    "url": "../../benchmarks/benchmark_history.json",
+    "format": {
+      "type": "json"
+    }
+  },
+  "transform": [
+    {
+      "calculate": "replace(datum.benchmark, 'vecxt.benchmark.', '')",
+      "as": "benchmark"
+    },
+    {
+      "calculate": "datetime(substring(datum.date,0, 4)+ '-' + substring(datum.date,4, 6) + '-' + substring(datum.date,6, 8))",
+      "as": "date"
+    },
+    {
+      "joinaggregate": [
+        {
+          "op": "max",
+          "field": "date",
+          "as": "maxDate"
+        }
+      ]
+    },
+    {
+      "filter": {
+        "field": "benchmark",
+        "oneOf": [
+          "CountTrueBenchmark.countTrue_loop",
+          "CountTrueBenchmark.countTrue_loop_vec"
+        ]
+      }
+    },
+    {
+      "window": [{ "op": "dense_rank", "as": "rank" }],
+      "sort": [{ "field": "date", "order": "descending" }]
+    },
+    { "filter": "datum.rank <= 1" }
+  ],
+  "vconcat": [
+    {
+      "layer": [
+        {
+          "mark": "errorbar",
+          "encoding": {
+            "x": {
+              "field": "scoreLowerConfidence",
+              "type": "quantitative",
+              "scale": { "zero": false },
+              "title": "ops / s"
+            },
+            "x2": { "field": "scoreUpperConfidence" },
+            "y": { "field": "benchmark", "type": "ordinal" }
+          }
+        },
+        {
+          "mark": {
+            "type": "point",
+            "filled": true,
+            "color": "black"
+          },
+          "encoding": {
+            "x": {
+              "field": "score",
+              "type": "quantitative"
+            },
+            "y": { "field": "benchmark", "type": "ordinal" }
+          }
+        }
+      ],
+      "transform": [{ "filter": "(datum.params.len == '3')" }]
+    },
+    {
+      "layer": [
+        {
+          "mark": "errorbar",
+          "encoding": {
+            "x": {
+              "field": "scoreLowerConfidence",
+              "type": "quantitative",
+              "scale": { "zero": false },
+              "title": "ops / s"
+            },
+            "x2": { "field": "scoreUpperConfidence" },
+            "y": { "field": "benchmark", "type": "ordinal" }
+          }
+        },
+        {
+          "mark": {
+            "type": "point",
+            "filled": true,
+            "color": "black"
+          },
+          "encoding": {
+            "x": {
+              "field": "score",
+              "type": "quantitative"
+            },
+            "y": { "field": "benchmark", "type": "ordinal" }
+          }
+        }
+      ],
+      "transform": [{ "filter": "(datum.params.len == '128')" }]
+    },
+
+    {
+      "layer": [
+        {
+          "mark": "errorbar",
+          "encoding": {
+            "x": {
+              "field": "scoreLowerConfidence",
+              "type": "quantitative",
+              "scale": { "zero": false },
+              "title": "ops / s"
+            },
+            "x2": { "field": "scoreUpperConfidence" },
+            "y": { "field": "benchmark", "type": "ordinal" }
+          }
+        },
+        {
+          "mark": {
+            "type": "point",
+            "filled": true,
+            "color": "black"
+          },
+          "encoding": {
+            "x": {
+              "field": "score",
+              "type": "quantitative"
+            },
+            "y": { "field": "benchmark", "type": "ordinal" }
+          }
+        }
+      ],
+      "transform": [{ "filter": "(datum.params.len == '100000')" }]
+    }
+  ]
+}
diff --git a/site/docs/_assets/plots/sum_over_time.vg.json b/site/docs/_assets/plots/sum_over_time.vg.json
@@ -0,0 +1,133 @@
+{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "data": {
+    "url": "../../benchmarks/benchmark_history.json",
+    "format": {
+      "type": "json"
+    }
+  },
+  "transform": [
+    {
+      "calculate": "replace(datum.benchmark, 'vecxt.benchmark.', '')",
+      "as": "benchmark"
+    },
+    {
+      "calculate": "datetime(substring(datum.date,0, 4)+ '-' + substring(datum.date,4, 6) + '-' + substring(datum.date,6, 8))",
+      "as": "date"
+    },
+    {
+      "filter": {
+        "field": "benchmark",
+        "oneOf": ["AddScalarBenchmark.vecxt_add_vec"]
+      }
+    },
+    {
+      "window": [{ "op": "dense_rank", "as": "rank" }],
+      "sort": [{ "field": "date", "order": "descending" }]
+    }
+  ],
+  "vconcat": [
+    {
+      "layer": [
+        {
+          "title": "n = 10",
+          "mark": {
+            "type": "line",
+            "color": "black"
+          },
+          "encoding": {
+            "y": {
+              "field": "score",
+              "type": "quantitative",
+              "scale": { "zero": false },
+              "title": "ops/s"
+            },
+            "x": { "field": "date", "timeUnit": "date" }
+          }
+        },
+        {
+          "mark": { "opacity": 0.3, "type": "area", "color": "#85C5A6" },
+          "encoding": {
+            "x": { "field": "date", "timeUnit": "date" },
+            "y": {
+              "field": "scoreUpperConfidence",
+              "type": "quantitative"
+            },
+            "y2": {
+              "field": "scoreLowerConfidence"
+            }
+          }
+        }
+      ],
+      "transform": [{ "filter": "(datum.params.n == '10')" }]
+    },
+    {
+      "layer": [
+        {
+          "title": "n = 1000",
+          "mark": {
+            "type": "line",
+            "color": "black"
+          },
+          "encoding": {
+            "y": {
+              "field": "score",
+              "type": "quantitative",
+              "scale": { "zero": false }
+            },
+            "x": { "field": "date", "timeUnit": "date" }
+          }
+        },
+        {
+          "mark": { "opacity": 0.3, "type": "area", "color": "#85C5A6" },
+          "encoding": {
+            "x": { "field": "date", "timeUnit": "date" },
+            "y": {
+              "field": "scoreUpperConfidence",
+              "type": "quantitative",
+              "title": "ops/s"
+            },
+            "y2": {
+              "field": "scoreLowerConfidence"
+            }
+          }
+        }
+      ],
+      "transform": [{ "filter": "(datum.params.n == '1000')" }]
+    },
+    {
+      "layer": [
+        {
+          "title": "n = 1000000",
+          "mark": {
+            "type": "line",
+            "color": "black"
+          },
+          "encoding": {
+            "y": {
+              "field": "score",
+              "type": "quantitative",
+              "scale": { "zero": false },
+              "title": "ops/s"
+            },
+            "x": { "field": "date", "timeUnit": "date" }
+          }
+        },
+        {
+          "mark": { "opacity": 0.3, "type": "area", "color": "#85C5A6" },
+          "encoding": {
+            "x": { "field": "date", "timeUnit": "date" },
+            "y": {
+              "field": "scoreUpperConfidence",
+              "type": "quantitative"
+            },
+            "y2": {
+              "field": "scoreLowerConfidence"
+            }
+          }
+        }
+      ],
+      "transform": [{ "filter": "(datum.params.n == '100000')" }]
+    }
+  ]
+}
diff --git a/site/docs/_docs/benchmarks/countTrue.md b/site/docs/_docs/benchmarks/countTrue.md
@@ -0,0 +1,16 @@
+
+You may need to refresh the page.
+https://github.com/scala/scala3/issues/21637
+
+Here is the comparison of the standard `while` loop with the vectorised version.
+<div id="vis" style="width: 50vw;height: 10vh"></div>
+
+<script type="module">
+  import vegaEmbed from "https://cdn.jsdelivr.net/npm/vega-embed@6/+esm?bundle-deps=true";
+  var spec = "../../plots/sum.vg.json";
+  vegaEmbed('#vis', spec).then(function(result) {
+    // Access the Vega view instance (https://vega.github.io/vega/docs/api/view/) as result.view
+  }).catch(console.error);
+</script>
+
+# Conclusion
diff --git a/site/docs/_docs/benchmarks/sum.md b/site/docs/_docs/benchmarks/sum.md
@@ -6,13 +6,22 @@ Here is the comparison of the standard `while` loop with the vectorised version.
 <div id="vis" style="width: 50vw;height: 10vh"></div>
 
 <script type="module">
-  import vegaEmbed from "https://cdn.jsdelivr.net/npm/vega-embed@6/+esm?bundle-deps=true";
+  import vegaEmbed from "https://cdn.jsdelivr.net/npm/vega-embed@6/+esm";
   var spec = "../../plots/sum.vg.json";
-  vegaEmbed('#vis', spec).then(function(result) {
-    // Access the Vega view instance (https://vega.github.io/vega/docs/api/view/) as result.view
-  }).catch(console.error);
+  vegaEmbed('#vis', spec)
 </script>
 
+And the function left in vexct over time (against regressions)
+
+<div id="visTime" style="width: 50vw;height: 10vh"></div>
+
+<script type="module">
+  import vegaEmbed from "https://cdn.jsdelivr.net/npm/vega-embed@6/+esm";
+  var spec = "../../plots/sum_over_time.vg.json";
+  vegaEmbed('#visTime', spec)
+</script>
+
+
 The two implementations are;
 
 ```scala sc:nocompile