benchmark

.github/workflows/benchmark.yml

@@ -0,0 +1,36 @@
+name: Benchmark
+
+on:
+  workflow_dispatch:  # Manual trigger
+
+jobs:
+  benchmark:
+    name: Performance check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: coursier/setup-action@main
+        with:
+          jvm: temurin@17
+          apps: mill
+      # Run benchmark with `go test -bench` and stores the output to a file
+      - name: Run benchmark
+        run:  mill benchmark.runJmh -jvmArgs --add-modules=jdk.incubator.vector -rf json
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark-${{ github.head_ref }} // cache against branch name
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'jmh'
+          # Where the output from the benchmark tool is stored
+          output-file-path: out/benchmark/runJmh.dest/jmh-result.json
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          # Workflow will fail when an alert happens
+          fail-on-alert: true
+      # Upload the updated cache file for the next job by actions/cache

benchmark/src/Benchmark.scala

@@ -0,0 +1,50 @@
+package vecxt.benchmark
+
+import dev.ludovic.netlib.blas.*;
+
+import org.openjdk.jmh.annotations.*;
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Array(Mode.Throughput))
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Thread)
+@Fork(value = 1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 3)
+abstract class BLASBenchmark:
+
+  var blas: BLAS = _;
+
+  @Setup
+  def setupImplementation: Unit =
+    blas = JavaBLAS.getInstance();
+    ()
+
+    // System.out.println("implementation = " + blas.getClass().getName());
+  end setupImplementation
+
+  private final val rand: Random = new Random(0);
+
+  protected def randomDouble(): Double =
+    return rand.nextDouble();
+
+  protected def randomDoubleArray(n: Int): Array[Double] =
+    val res = new Array[Double](n);
+
+    for i <- 0 until n do res(i) = rand.nextDouble();
+    end for
+    return res;
+  end randomDoubleArray
+
+  protected def randomFloat(): Float =
+    return rand.nextFloat();
+
+  protected def randomFloatArray(n: Int): Array[Float] =
+    val res = new Array[Float](n);
+    for i <- 0 until n do res(i) = rand.nextFloat();
+    end for
+    return res;
+  end randomFloatArray
+end BLASBenchmark

benchmark/src/matmul.scala

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020, 2021, Ludovic Henry
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Please contact git@ludovic.dev or visit ludovic.dev if you need additional
+ * information or have any questions.
+ */
+
+package vecxt.benchmark
+
+import org.openjdk.jmh.annotations.*
+import org.openjdk.jmh.infra.Blackhole
+import vecxt.Matrix.*
+import vecxt.BoundsCheck
+
+@State(Scope.Thread)
+class DgemmBenchmark extends BLASBenchmark:
+
+  var m, n, k: Int = _
+  var transa: String = _
+  var transb: String = _
+  var alpha: Double = _
+  var a: Array[Double] = _
+  var lda: Int = _
+  var b: Array[Double] = _
+  var ldb: Int = _
+  var beta: Double = _
+  var c, cclone: Array[Double] = _
+  var ldc: Int = _
+
+  var matA: Matrix = _
+  var matB: Matrix = _
+
+  // format: off
+  @Setup(Level.Trial)
+  def setup: Unit =
+    transa = "N"
+    transb = "N"
+    m = 100
+    n = 100
+    k = 100
+    alpha = randomDouble();
+    a = randomDoubleArray(k * m);
+    b = randomDoubleArray(k * n);
+
+    beta = randomDouble();
+    // c = randomDoubleArray(m * n);
+    matA = Matrix(a, (m, k))(using BoundsCheck.DoBoundsCheck.no)
+    matB = Matrix(b, (k, n))(using BoundsCheck.DoBoundsCheck.no)
+    ()
+
+  end setup
+
+  @Benchmark
+  def runblas(bh: Blackhole) =
+    val cclone = Array.fill[Double](m*n)(0)
+    blas.dgemm(
+      transa,
+      transb,
+      m,
+      n,
+      k,
+      alpha,
+      a,
+      if transa.equals("N") then m else k,
+      b,
+      if transb.equals("N") then k else n,
+      beta,
+      cclone,
+      m
+    );
+    bh.consume(cclone);
+  end runblas
+
+
+  @Benchmark
+  def mmult(bh: Blackhole) =
+
+    val cclone = (matA @@ matB)(using BoundsCheck.DoBoundsCheck.no)
+    bh.consume(cclone);
+  end mmult
+
+
+
+end DgemmBenchmark

build.sc

@@ -2,6 +2,8 @@
 import $ivy.`com.github.lolgab::mill-crossplatform::0.2.4`
 import $ivy.`io.github.quafadas::millSite::0.0.24`
 import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0`
+import $ivy.`com.lihaoyi::mill-contrib-jmh:`
+
 
 import de.tobiasroeser.mill.vcs.version._
 import com.github.lolgab.mill.crossplatform._
@@ -10,6 +12,7 @@ import io.github.quafadas.millSite._
 import mill._, scalalib._, publish._
 import mill.scalajslib.api._
 import mill.scalanativelib._
+import contrib.jmh.JmhModule
 
 import mill.api.Result
 
@@ -39,6 +42,7 @@ trait Common extends ScalaModule  with PublishModule {
 }
 
 val vecIncubatorFlag = Seq("""--add-modules=jdk.incubator.vector""")
+
 trait CommonJS extends ScalaJSModule {
   def scalaJSVersion = "1.16.0"
   // def ivyDeps = super.ivyDeps() ++ Seq(ivy"com.raquo::ew::0.2.0")
@@ -92,6 +96,12 @@ object vecxt extends CrossPlatform {
 
 }
 
+object benchmark extends JmhModule with ScalaModule {
+    def scalaVersion = vecxt.jvm.scalaVersion
+    def jmhCoreVersion = "1.37"
+    override def moduleDeps: Seq[JavaModule] = Seq(vecxt.jvm)
+}
+
 object jsSite extends SiteJSModule {
 
   override def moduleDeps = Seq(vecxt.js)
@@ -109,7 +119,7 @@ object site extends SiteModule {
 
   override def moduleDeps = Seq(vecxt.jvm)
 
-  override def allScalacOptions: Target[Seq[String]] = super.allScalacOptions() ++ vecIncubatorFlag
+  // override def allScalacOptions: Target[Seq[String]] = super.allScalacOptions() ++ vecIncubatorFlag
 
   override def scalaDocOptions = super.scalaDocOptions
 

justfile

@@ -1,4 +1,7 @@
 
 
 format:
-  mill mill.scalalib.scalafmt.ScalafmtModule/reformatAll __.sources
+  mill mill.scalalib.scalafmt.ScalafmtModule/reformatAll __.sources
+
+benchmark:
+  mill benchmark.runJmh -jvmArgs --add-modules=jdk.incubator.vector