[prototype] fsim interpreter

.github/workflows/test.yml

@@ -47,6 +47,35 @@ jobs:
         timeout-minutes: 6
         run: sbt ++${{ matrix.scala }} "testOnly integration.**"
 
+  benchmarks:
+    name: Benchmarks
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        scala: [ 2.13.10 ]
+        jvm: [ 8, 11, 20 ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Scala
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: ${{ matrix.jvm }}
+      - name: Check for Warnings
+        run: sbt "project benchmark ; set ThisBuild / scalacOptions ++= Seq(\"-Xfatal-warnings\") ; compile"
+      - name: Check Formatting
+        run: sbt "project benchmark ; scalafmtCheckAll"
+      - name: Create JAR
+        run: sbt "project benchmark ; assembly"
+      - name: Run Benchmark
+        run: ./benchmark/benchmark.py --bench=gcd_64 --sim=fsim
+      - name: Archive Flamegraph
+        uses: actions/upload-artifact@v3
+        if: matrix.jvm == 8
+        with:
+          name: flamegraph
+          path: out.svg
 
   test-mac:
     name: sbt test on mac
@@ -206,7 +235,7 @@ jobs:
   # When adding new jobs, please add them to `needs` below
   all_tests_passed:
     name: "all tests passed"
-    needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, no-warn, integration-test]
+    needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, no-warn, integration-test, benchmarks]
     runs-on: ubuntu-latest
     steps:
       - run: echo Success!
@@ -215,7 +244,7 @@ jobs:
   # separate from a Scala versions build matrix to avoid duplicate publishing
   publish:
     # note: we do not require a warning check for publishing!
-    needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, test-treadle, integration-test]
+    needs: [test, doc, verilator, formal, formal-mac, icarus, test-mac, test-treadle, integration-test, benchmarks]
     runs-on: ubuntu-20.04
     if: github.event_name == 'push'
 

.gitignore

@@ -20,3 +20,8 @@
 # Synopsys Verdi
 novas_dump.log
 ucli.key
+
+# JVM Profiling
+/out.hprof
+/out-folded.txt
+/out.svg

benchmark/.gitignore

@@ -0,0 +1,11 @@
+/project/target
+/project/project
+/target/
+/benchmark.jar
+
+# JVM Profiling
+/out.hprof
+/out-folded.txt
+/out.svg
+/FlameGraph/
+/hprof2flamegraph/

benchmark/benchmark.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# Copyright 2023 The Regents of the University of California
+# released under BSD 3-Clause License
+# author: Kevin Laeufer <laeufer@cs.berkeley.edu>
+import os
+import sys
+from pathlib import Path
+import subprocess
+
+# possible locations for Java 8
+java_8_paths = [
+    "/usr/lib/jvm/java-1.8.0/bin/java"
+]
+
+_script_dir = Path(__file__).parent.resolve()
+jar_file = _script_dir / "benchmark.jar"
+main_class = "fsim.Benchmark"
+hprof = "-agentlib:hprof=cpu=samples,depth=100,interval=20,lineno=y,thread=y,file=out.hprof"
+
+
+
+
+def get_jvm_major_version(java: str) -> int:
+    r = subprocess.run([java, "-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out = r.stdout.decode('utf-8').split('\n') + r.stderr.decode('utf-8').split('\n')
+    needle = 'version "'
+    for line in out:
+        if needle in line:
+            numbers = [int(n) for n in line.split(needle)[1].split('.')[0:2]]
+            if numbers[0] == 1:
+                return numbers[1]
+            else:
+                return numbers[0]
+    raise RuntimeError("Unexpected version output:\n" + '\n'.join(out))
+
+def try_to_find_java_8() -> str:
+    for path in java_8_paths:
+        path = Path(path).resolve()
+        if path.exists() and path.is_file():
+            version = get_jvm_major_version(str(path))
+            if version == 8:
+                return str(path)
+            else:
+                print(f"{path} points to Java {version}")
+        else:
+            print(f"{path} does not exist")
+    return "java"
+
+
+hprof_2_flamegraph = _script_dir / "hprof2flamegraph"
+hprof_2_flamegraph_repo = "https://github.com/cykl/hprof2flamegraph.git"
+def get_hprof_2_flamegraph() -> Path:
+    if not hprof_2_flamegraph.exists():
+        # clone
+        cmd = ['git', 'clone', hprof_2_flamegraph_repo, str(hprof_2_flamegraph.resolve())]
+        subprocess.run(cmd, check=True)
+    assert hprof_2_flamegraph.exists()
+    return hprof_2_flamegraph
+
+def run_hprof_2_flamegraph(out: Path) -> Path:
+    script = get_hprof_2_flamegraph() / "stackcollapse_hprof.py"
+    collapsed = Path("out-folded.txt")
+    with open(collapsed, 'wb') as ff:
+        subprocess.run([str(script.resolve()), str(out.resolve())], stdout=ff, check=True)
+    assert collapsed.exists()
+    return collapsed
+
+flamegraph_dir = _script_dir / "FlameGraph"
+flamegraph_repo = "https://github.com/brendangregg/FlameGraph.git"
+def get_flamegraph() -> Path:
+    if not flamegraph_dir.exists():
+        # clone
+        cmd = ['git', 'clone', flamegraph_repo, str(flamegraph_dir.resolve())]
+        subprocess.run(cmd, check=True)
+    assert flamegraph_dir.exists()
+    return flamegraph_dir
+
+def run_flamegraph(collapsed: Path) -> Path:
+    script = get_flamegraph() / "flamegraph.pl"
+    svg = Path("out.svg")
+    with open(svg, 'wb') as ff:
+        subprocess.run([str(script.resolve()), str(collapsed.resolve())], stdout=ff, check=True)
+    assert svg.exists()
+    return svg
+
+
+def convert_hprof_out(out: Path):
+    assert out.exists(), f"{out} does not exists"
+    collapsed = run_hprof_2_flamegraph(out)
+    svg = run_flamegraph(collapsed)
+    # cleanup
+    os.remove(collapsed)
+    os.remove(out)
+    return svg
+
+def main():
+    assert jar_file.exists(), f"{jar_file} not found. Did you run sbt assembly?"
+
+    do_profile = True
+
+    java = try_to_find_java_8() if do_profile else "java"
+    version = get_jvm_major_version(java)
+    cmd = ["-cp", jar_file, main_class]
+    if do_profile:
+        if version == 8:
+            cmd = [hprof] + cmd
+        else:
+            print(f"WARN: we currently only support profiling with JVM 8, but you are running {version}")
+
+    subprocess.run([java] + cmd + sys.argv[1:])
+
+    if do_profile and version == 8:
+        svg = convert_hprof_out(Path("out.hprof"))
+        print(f"Wrote flame graph to {svg}")
+
+
+
+if __name__ == '__main__':
+    main()

benchmark/src/fsim/Benchmark.scala

@@ -0,0 +1,139 @@
+// Copyright 2023 The Regents of the University of California
+// released under BSD 3-Clause License
+// author: Kevin Laeufer <laeufer@cs.berkeley.edu>
+
+package fsim
+
+import firrtl2.options.Dependency
+import firrtl2.stage.{FirrtlSourceAnnotation, Forms}
+import scopt.OptionParser
+import treadle2.TreadleTester
+
+case class Config(benches: Seq[String], sim: String, warmupRun: Int, repsRun: Int)
+
+class ArgumentParser extends OptionParser[Config]("synthesizer") {
+  head("fsim benchmark", "0.1")
+  opt[String]("bench").required().action((a, config) => config.copy(benches = config.benches :+ a))
+  opt[String]("sim").action((a, config) => config.copy(sim = a))
+  opt[Int]("warmup").action((a, config) => config.copy(warmupRun = a))
+  opt[Int]("reps").action((a, config) => config.copy(repsRun = a))
+}
+
+case class Bench(
+  name:           String,
+  getCircuit:     String => String,
+  runTest:        Simulation => Unit,
+  runTreadleTest: TreadleTester => Unit)
+case class Result(nsCompile: Long, nsRun: Long, steps: Int)
+object Benchmark {
+  val Benches = Seq(
+    Bench("gcd_16", _ => GCDBench.circuitSrc(16), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500)),
+    Bench("gcd_44", _ => GCDBench.circuitSrc(44), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500)),
+    Bench("gcd_64", _ => GCDBench.circuitSrc(64), GCDBench.fsimTest(_, 10, 500), GCDBench.treadleTest(_, 10, 500))
+  )
+
+  private val DefaultConfig = Config(benches = Seq(), sim = "fsim", warmupRun = 1, repsRun = 1)
+
+  def main(args: Array[String]): Unit = {
+    val parser = new ArgumentParser()
+    val conf = parser.parse(args, DefaultConfig).get
+    conf.benches.foreach { benchName =>
+      val bench =
+        Benches.find(_.name == benchName).getOrElse(throw new RuntimeException(s"Unknown benchmark: $benchName"))
+      val res = runBench(conf, bench)
+      printResult(bench.name, conf.sim, res)
+    }
+
+  }
+
+  def printResult(bench: String, sim: String, res: Result): Unit = {
+    println(
+      s"${bench} on ${sim}: " +
+        s"${secondString(res.nsRun)}, ${res.steps} cycles, ${freqString(res.nsRun, res.steps)}, " +
+        s"${secondString(res.nsCompile)} to compile"
+    )
+  }
+
+  private def secondString(ns: Long): String = {
+    val elapsedSeconds = ns.toDouble / Giga
+    f"$elapsedSeconds%.6fs"
+  }
+
+  private val Kilo: Double = 1000.0
+  private val Mega: Double = 1000000.0
+  private val Giga: Double = 1000000000.0
+  private def freqString(ns: Long, steps: Int): String = {
+    val elapsedSeconds = ns.toDouble / Giga
+    val hz = steps.toDouble / elapsedSeconds
+    if (hz > Giga) {
+      f"${hz / Giga}%.6fGHz"
+    } else if (hz > Mega) {
+      f"${hz / Mega}%.6fMHz"
+    } else if (hz > Kilo) {
+      f"${hz / Kilo}%.6fkHz"
+    } else {
+      f"${hz}%.6fHz"
+    }
+  }
+
+  def runBench(conf: Config, bench: Bench): Result = {
+    val src = bench.getCircuit(conf.sim)
+    conf.sim match {
+      case "fsim"    => runFSimBench(conf, bench, src)
+      case "treadle" => runTreadleBench(conf, bench, src)
+      case other     => throw new RuntimeException(s"Unsupported simulator: $other")
+    }
+  }
+
+  private def runFSimBench(conf: Config, bench: Bench, src: String): Result = {
+    val compileStart = System.nanoTime()
+    val sim = new Simulation(Compiler.run(FirrtlCompiler.toLow(src)))
+    val compileEnd = System.nanoTime()
+    for (_ <- 0 until conf.warmupRun) {
+      bench.runTest(sim)
+    }
+    val runtimeAndSteps = (0 until conf.repsRun).map { _ =>
+      val startCycles = sim.getStepCount
+      val testStart = System.nanoTime()
+      bench.runTest(sim)
+      val testEnd = System.nanoTime()
+      val steps = sim.getStepCount - startCycles
+      (testEnd - testStart, steps)
+    }
+    sim.finish()
+    val nsRun = runtimeAndSteps.map(_._1).sum / runtimeAndSteps.length
+    val steps = runtimeAndSteps.head._2
+    Result(nsCompile = compileEnd - compileStart, nsRun = nsRun, steps = steps)
+  }
+
+  private def runTreadleBench(conf: Config, bench: Bench, src: String): Result = {
+    val compileStart = System.nanoTime()
+    val tester = TreadleTester(Seq(FirrtlSourceAnnotation(src)))
+    val compileEnd = System.nanoTime()
+    for (_ <- 0 until conf.warmupRun) {
+      bench.runTreadleTest(tester)
+    }
+    val runtimeAndSteps = (0 until conf.repsRun).map { _ =>
+      val startCycles = tester.cycleCount.toInt
+      val testStart = System.nanoTime()
+      bench.runTreadleTest(tester)
+      val testEnd = System.nanoTime()
+      val steps = tester.cycleCount.toInt - startCycles
+      (testEnd - testStart, steps)
+    }
+    tester.finish
+    val nsRun = runtimeAndSteps.map(_._1).sum / runtimeAndSteps.length
+    val steps = runtimeAndSteps.head._2
+    Result(nsCompile = compileEnd - compileStart, nsRun = nsRun, steps = steps)
+  }
+}
+
+object FirrtlCompiler {
+  private val loFirrtlCompiler =
+    new firrtl2.stage.transforms.Compiler(Seq(Dependency[firrtl2.LowFirrtlEmitter]) ++ Forms.LowFormOptimized)
+  def toLow(src: String): firrtl2.ir.Circuit = {
+    val hi = firrtl2.Parser.parse(src)
+    val lo = loFirrtlCompiler.execute(firrtl2.CircuitState(hi))
+    lo.circuit
+  }
+}