From 9d8ba8f2064b356eb4c0393bd84a8d5c552ae246 Mon Sep 17 00:00:00 2001 From: Simon Parten Date: Tue, 17 Sep 2024 09:37:46 +0200 Subject: [PATCH] benchmark --- .github/workflows/benchmark.yml | 36 +++++++++++ benchmark/src/Benchmark.scala | 50 ++++++++++++++++ benchmark/src/matmul.scala | 102 ++++++++++++++++++++++++++++++++ build.sc | 12 +++- justfile | 5 +- 5 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/src/Benchmark.scala create mode 100644 benchmark/src/matmul.scala diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..cd1d0f0 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,36 @@ +name: Benchmark + +on: + workflow_dispatch: # Manual trigger + +jobs: + benchmark: + name: Performance check + runs-on: ubuntu-latest + steps: + - uses: coursier/setup-action@main + with: + jvm: temurin@17 + apps: mill + # Run benchmark with `go test -bench` and stores the output to a file + - name: Run benchmark + run: mill benchmark.runJmh -jvmArgs --add-modules=jdk.incubator.vector -rf json + # Download previous benchmark result from cache (if exists) + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark-${{ github.head_ref }} // cache against branch name + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool the output.txt came from + tool: 'jmh' + # Where the output from the benchmark tool is stored + output-file-path: out/benchmark/runJmh.dest/jmh-result.json + # Where the previous data file is stored + external-data-json-path: ./cache/benchmark-data.json + # Workflow will fail when an alert happens + fail-on-alert: true + # Upload the updated cache file for the next job by actions/cache \ No newline at end of file diff --git a/benchmark/src/Benchmark.scala b/benchmark/src/Benchmark.scala new file mode 100644 index 0000000..fbbb81f --- /dev/null +++ b/benchmark/src/Benchmark.scala @@ -0,0 +1,50 @@ +package vecxt.benchmark + +import dev.ludovic.netlib.blas.*; + +import org.openjdk.jmh.annotations.*; + +import java.util.Random; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Array(Mode.Throughput)) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +@Fork(value = 1) +@Warmup(iterations = 3) +@Measurement(iterations = 3) +abstract class BLASBenchmark: + + var blas: BLAS = _; + + @Setup + def setupImplementation: Unit = + blas = JavaBLAS.getInstance(); + () + + // System.out.println("implementation = " + blas.getClass().getName()); + end setupImplementation + + private final val rand: Random = new Random(0); + + protected def randomDouble(): Double = + return rand.nextDouble(); + + protected def randomDoubleArray(n: Int): Array[Double] = + val res = new Array[Double](n); + + for i <- 0 until n do res(i) = rand.nextDouble(); + end for + return res; + end randomDoubleArray + + protected def randomFloat(): Float = + return rand.nextFloat(); + + protected def randomFloatArray(n: Int): Array[Float] = + val res = new Array[Float](n); + for i <- 0 until n do res(i) = rand.nextFloat(); + end for + return res; + end randomFloatArray +end BLASBenchmark diff --git a/benchmark/src/matmul.scala b/benchmark/src/matmul.scala new file mode 100644 index 0000000..7002f19 --- /dev/null +++ b/benchmark/src/matmul.scala @@ -0,0 +1,102 @@ +/* + * Copyright 2020, 2021, Ludovic Henry + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Please contact git@ludovic.dev or visit ludovic.dev if you need additional + * information or have any questions. + */ + +package vecxt.benchmark + +import org.openjdk.jmh.annotations.* +import org.openjdk.jmh.infra.Blackhole +import vecxt.Matrix.* +import vecxt.BoundsCheck + +@State(Scope.Thread) +class DgemmBenchmark extends BLASBenchmark: + + var m, n, k: Int = _ + var transa: String = _ + var transb: String = _ + var alpha: Double = _ + var a: Array[Double] = _ + var lda: Int = _ + var b: Array[Double] = _ + var ldb: Int = _ + var beta: Double = _ + var c, cclone: Array[Double] = _ + var ldc: Int = _ + + var matA: Matrix = _ + var matB: Matrix = _ + + // format: off + @Setup(Level.Trial) + def setup: Unit = + transa = "N" + transb = "N" + m = 100 + n = 100 + k = 100 + alpha = randomDouble(); + a = randomDoubleArray(k * m); + b = randomDoubleArray(k * n); + + beta = randomDouble(); + // c = randomDoubleArray(m * n); + matA = Matrix(a, (m, k))(using BoundsCheck.DoBoundsCheck.no) + matB = Matrix(b, (k, n))(using BoundsCheck.DoBoundsCheck.no) + () + + end setup + + @Benchmark + def runblas(bh: Blackhole) = + val cclone = Array.fill[Double](m*n)(0) + blas.dgemm( + transa, + transb, + m, + n, + k, + alpha, + a, + if transa.equals("N") then m else k, + b, + if transb.equals("N") then k else n, + beta, + cclone, + m + ); + bh.consume(cclone); + end runblas + + + @Benchmark + def mmult(bh: Blackhole) = + + val cclone = (matA @@ matB)(using BoundsCheck.DoBoundsCheck.no) + bh.consume(cclone); + end mmult + + + +end DgemmBenchmark diff --git a/build.sc b/build.sc index 1211627..642572c 100644 --- a/build.sc +++ b/build.sc @@ -2,6 +2,8 @@ import $ivy.`com.github.lolgab::mill-crossplatform::0.2.4` import $ivy.`io.github.quafadas::millSite::0.0.24` import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0` +import $ivy.`com.lihaoyi::mill-contrib-jmh:` + import de.tobiasroeser.mill.vcs.version._ import com.github.lolgab.mill.crossplatform._ @@ -10,6 +12,7 @@ import io.github.quafadas.millSite._ import mill._, scalalib._, publish._ import mill.scalajslib.api._ import mill.scalanativelib._ +import contrib.jmh.JmhModule import mill.api.Result @@ -39,6 +42,7 @@ trait Common extends ScalaModule with PublishModule { } val vecIncubatorFlag = Seq("""--add-modules=jdk.incubator.vector""") + trait CommonJS extends ScalaJSModule { def scalaJSVersion = "1.16.0" // def ivyDeps = super.ivyDeps() ++ Seq(ivy"com.raquo::ew::0.2.0") @@ -92,6 +96,12 @@ object vecxt extends CrossPlatform { } +object benchmark extends JmhModule with ScalaModule { + def scalaVersion = vecxt.jvm.scalaVersion + def jmhCoreVersion = "1.37" + override def moduleDeps: Seq[JavaModule] = Seq(vecxt.jvm) +} + object jsSite extends SiteJSModule { override def moduleDeps = Seq(vecxt.js) @@ -109,7 +119,7 @@ object site extends SiteModule { override def moduleDeps = Seq(vecxt.jvm) - override def allScalacOptions: Target[Seq[String]] = super.allScalacOptions() ++ vecIncubatorFlag + // override def allScalacOptions: Target[Seq[String]] = super.allScalacOptions() ++ vecIncubatorFlag override def scalaDocOptions = super.scalaDocOptions diff --git a/justfile b/justfile index ba4e20d..b52113c 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,7 @@ format: - mill mill.scalalib.scalafmt.ScalafmtModule/reformatAll __.sources \ No newline at end of file + mill mill.scalalib.scalafmt.ScalafmtModule/reformatAll __.sources + +benchmark: + mill benchmark.runJmh -jvmArgs --add-modules=jdk.incubator.vector \ No newline at end of file