[ci] add VCS CI

.github/workflows/vcs.yml

@@ -0,0 +1,105 @@
+name: VCS
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+      - labeled
+env:
+  USER: runner
+
+# Cancel the current workflow when new commit pushed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  gen-test-plan:
+    if: '! github.event.pull_request.draft'
+    name: "Generate test plan"
+    runs-on: [self-hosted, linux, nixos]
+    outputs:
+      testplan: ${{ steps.get-all-configs.outputs.out }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - id: "get-all-configs"
+        run: echo "out=$(nix run .#ci-helper generateTestPlan)" > $GITHUB_OUTPUT
+
+  build-emulators:
+    name: "Build VCS Emulators"
+    needs: [gen-test-plan]
+    runs-on: [self-hosted, linux, nixos, BIGRAM]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.gen-test-plan.outputs.testplan) }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: "Build verilator emulator"
+        run: |
+          nix build '.#t1.${{ matrix.config }}.ip.vcs-emu-trace' --impure --no-link --cores 64
+      - name: "Build all testcases"
+        run: |
+          # Build testcases with vlen 1024 and vlen 4096
+          nix build ".#t1.${{ matrix.config }}.cases.all" --max-jobs auto --no-link --cores 64
+
+  gen-matrix:
+    name: "Prepare for running testcases"
+    needs: [build-emulators]
+    runs-on: [self-hosted, linux, nixos, BIGRAM]
+    env:
+      RUNNERS: 70
+    outputs:
+      ci-tests: ${{ steps.gen-matrix.outputs.matrix }}
+    steps:
+      # actions/checkout will use the "event" commit to checkout repository,
+      # which will lead to an unexpected issue that the "event" commit doesn't belongs to the repository,
+      # and causing the derivation build output cannot be cache correctly.
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - id: gen-matrix
+        name: "Generate test matrix"
+        run: |
+          echo -n matrix= >> "$GITHUB_OUTPUT"
+          nix run ".#ci-helper" -- generateCiMatrix --runnersAmount "$RUNNERS" >> "$GITHUB_OUTPUT"
+
+  run-testcases:
+    name: "Run VCS"
+    needs: [gen-matrix]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.gen-matrix.outputs.ci-tests) }}
+    runs-on: [self-hosted, linux, nixos]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: "Run testcases"
+        run: |
+          nix run ".#ci-helper" -- runTests --jobs "${{ matrix.jobs }}" --testType "vcs"
+
+  report:
+    name: "Report VCS CI result"
+    # Don't run report when:
+    #   - user cancel ( we don't need report at this case )
+    #   - PR from outside repository ( we don't have permission to push commit into fork repository )
+    if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
+    needs: [run-testcases]
+    runs-on: [self-hosted, linux, nixos]
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.head_ref }}
+      - name: "Print step summary"
+        run: |
+          nix run ".#ci-helper" -- postCI --failed-tests-file-path ./failed-tests.md --emuType vcs
+          cat ./failed-tests.md >> $GITHUB_STEP_SUMMARY

nix/t1/default.nix

@@ -52,7 +52,7 @@ lib.makeScope newScope
         elaborateConfig = builtins.fromJSON (lib.readFile configPath);
 
         cases = innerSelf.callPackage ../../tests {
-          inherit (ip) verilator-emu verilator-emu-trace vcs-emu-trace;
+          inherit (ip) verilator-emu verilator-emu-trace vcs-emu vcs-emu-trace;
         };
 
         # for the convenience to use x86 cases on non-x86 machines, avoiding the extra build time

script/ci/src/Main.scala

@@ -144,7 +144,7 @@ object Main:
   @main
   def runTests(
       jobs: String,
-      dontBail: Flag = Flag(false)
+      testType: String = "verilator"
   ): Unit =
     if jobs == "" then
       Logger.info("No test found, exiting")
@@ -157,32 +157,31 @@ object Main:
         val Array(config, caseName) = testName.split(",")
         println("\n")
         Logger.info(
-          s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running test case $caseName with config $config"
+          s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running VCS for test case $caseName with config $config"
         )
 
+        val testAttr = testType.toLowerCase() match
+          case "verilator" =>
+            s".#t1.$config.cases.$caseName.emu-result.with-offline"
+          case "vcs" => s".#t1.$config.cases.$caseName.emu-result.with-vcs"
+          case _     => Logger.fatal(s"Invalid test type ${testType}")
         val testResultPath =
           try
             os.Path(
               nixResolvePath(
-                s".#t1.$config.cases.$caseName.emu-result.with-offline"
+                testAttr,
+                if testType == "vcs" then Seq("--impure") else Seq()
               )
             )
           catch
             case _ =>
-              Logger.error(s"Emulation for config $config, case $caseName fail")
+              Logger.error(s"VCS emulation for config $config, case $caseName fail")
               println("-" * 50)
-              println(
-                os.proc(
-                  "nix",
-                  "log",
-                  s".#t1.$config.cases.$caseName.emu-result"
-                ).call()
-                  .out
-              )
+              println(os.proc("nix", "log", testAttr).call().out)
               println("-" * 50)
               Logger.fatal("Got error from emulation, exiting CI")
 
-        Logger.info("Checking RTL event with offline difftest")
+        Logger.info("Checking RTL event from VCS")
         val testSuccess =
           os.read(testResultPath / "offline-check-status").trim() == "0"
         if !testSuccess then
@@ -193,10 +192,6 @@ object Main:
 
     val failedTests = findFailedTests()
     if failedTests.isEmpty then Logger.info(s"All tests passed")
-    else if dontBail.value then
-      Logger.error(
-        s"${BOLD}${failedTests.length} tests failed${RESET}"
-      )
     else
       Logger.fatal(
         s"${BOLD}${failedTests.length} tests failed${RESET}"
@@ -249,13 +244,17 @@ object Main:
       @arg(
         name = "cycle-update-file-path",
         doc = "specify the cycle update markdown file output path"
-      ) cycleUpdateFilePath: String
+      ) cycleUpdateFilePath: Option[String],
+      emuType: String = "verilator"
   ) =
     val failedTestsFile = os.Path(failedTestsFilePath, os.pwd)
     os.write.over(failedTestsFile, "## Failed Tests\n")
 
-    val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd)
-    os.write.over(cycleUpdateRecordFile, "## Cycle Update\n")
+    if cycleUpdateFilePath.nonEmpty then
+      os.write.over(
+        os.Path(cycleUpdateFilePath.get, os.pwd),
+        "## Cycle Update\n"
+      )
 
     os.walk(os.pwd / ".github" / "cases")
       .filter(_.last == "default.json")
@@ -264,6 +263,11 @@ object Main:
         var cycleRecord = ujson.read(os.read(file))
 
         Logger.info("Fetching CI results")
+        val resultAttr = emuType.toLowerCase() match
+          case "verilator" =>
+            s".#t1.$config.cases._allEmuResult"
+          case "vcs" => s".#t1.$config.cases._allVCSEmuResult"
+          case _     => Logger.fatal(s"Invalid test type ${emuType}")
         val emuResultPath =
           os.Path(nixResolvePath(s".#t1.$config.cases._allEmuResult"))
 
@@ -278,30 +282,34 @@ object Main:
             os.write.append(failedTestsFile, s"```text\n${journal}\n```\n")
           })
 
-        Logger.info("Collecting cycle update info")
-        val perfCycleRegex = raw"total_cycles:\s(\d+)".r
-        val allCycleUpdates = os
-          .walk(emuResultPath)
-          .filter(path => path.last == "perf.txt")
-          .map(path => {
-            val cycle = os.read.lines(path).head match
-              case perfCycleRegex(cycle) => cycle.toInt
-              case _ =>
-                throw new Exception("perf.txt file is not format as expected")
-            val caseName = path.segments.toSeq.reverse.drop(1).head
-            (caseName, cycle, cycleRecord.obj(caseName).num.toInt)
-          })
-          .filter((_, newCycle, oldCycle) => newCycle != oldCycle)
-          .map:
-            case (caseName, newCycle, oldCycle) =>
-              cycleRecord(caseName) = newCycle
-              if oldCycle == -1 then s"* 🆕 ${caseName}($config): NaN -> ${newCycle}"
-              else if oldCycle > newCycle then
-                s"* 🚀 $caseName($config): $oldCycle -> $newCycle"
-              else s"* 🐢 $caseName($config): $oldCycle -> $newCycle"
-
-        os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n"))
-        os.write.append(cycleUpdateRecordFile, "\n")
+        if cycleUpdateFilePath.nonEmpty then
+          Logger.info("Collecting cycle update info")
+          val perfCycleRegex = raw"total_cycles:\s(\d+)".r
+          val allCycleUpdates = os
+            .walk(emuResultPath)
+            .filter(path => path.last == "perf.txt")
+            .map(path => {
+              val cycle = os.read.lines(path).head match
+                case perfCycleRegex(cycle) => cycle.toInt
+                case _ =>
+                  throw new Exception("perf.txt file is not format as expected")
+              val caseName = path.segments.toSeq.reverse.drop(1).head
+              (caseName, cycle, cycleRecord.obj(caseName).num.toInt)
+            })
+            .filter((_, newCycle, oldCycle) => newCycle != oldCycle)
+            .map:
+              case (caseName, newCycle, oldCycle) =>
+                cycleRecord(caseName) = newCycle
+                if oldCycle == -1 then
+                  s"* 🆕 ${caseName}($config): NaN -> ${newCycle}"
+                else if oldCycle > newCycle then
+                  s"* 🚀 $caseName($config): $oldCycle -> $newCycle"
+                else s"* 🐢 $caseName($config): $oldCycle -> $newCycle"
+
+          os.write.append(
+            os.Path(cycleUpdateFilePath.get, os.pwd),
+            allCycleUpdates.mkString("\n") + "\n"
+          )
 
         os.write.over(file, ujson.write(cycleRecord, indent = 2))
   end postCI
@@ -316,14 +324,16 @@ object Main:
     println(ujson.write(Map("config" -> testPlans)))
   end generateTestPlan
 
-  def nixResolvePath(attr: String): String =
+  def nixResolvePath(attr: String, extraArgs: Seq[String] = Seq()): String =
     os.proc(
-      "nix",
-      "build",
-      "--no-link",
-      "--no-warn-dirty",
-      "--print-out-paths",
-      attr
+      Seq(
+        "nix",
+        "build",
+        "--no-link",
+        "--no-warn-dirty",
+        "--print-out-paths",
+        attr
+      ) ++ extraArgs
     ).call()
       .out
       .trim()

tests/default.nix

@@ -6,6 +6,7 @@
 , runCommand
 , verilator-emu
 , verilator-emu-trace
+, vcs-emu
 , vcs-emu-trace
 }:
 
@@ -21,7 +22,7 @@ let
   scope = lib.recurseIntoAttrs (lib.makeScope newScope (casesSelf: {
     recurseForDerivations = true;
 
-    inherit xLen vLen isFp verilator-emu verilator-emu-trace vcs-emu-trace;
+    inherit xLen vLen isFp verilator-emu verilator-emu-trace vcs-emu vcs-emu-trace;
 
     makeEmuResult = casesSelf.callPackage ./make-emu-result.nix { };
 
@@ -87,6 +88,25 @@ let
     in
     runCommand "catch-${configName}-all-emu-result-for-ci" { } script;
 
+  _allVCSEmuResult =
+    let
+      testPlan = builtins.fromJSON (lib.readFile ../.github/cases/${configName}/default.json);
+      # flattern the attr set to a list of test case derivations
+      # AttrSet (AttrSet Derivation) -> List Derivation
+      allCases = lib.filter (val: lib.isDerivation val && lib.hasAttr val.pname testPlan)
+        (lib.concatLists (map lib.attrValues (lib.attrValues scopeStripped)));
+      script = ''
+        mkdir -p $out
+      '' + (lib.concatMapStringsSep "\n"
+        (caseDrv: ''
+          _caseOutDir=$out/${caseDrv.pname}
+          mkdir -p "$_caseOutDir"
+          cp ${caseDrv.emu-result.with-vcs}/offline-check-* "$_caseOutDir"/
+        '')
+        allCases);
+    in
+    runCommand "catch-${configName}-all-vcs-emu-result-for-ci" { } script;
+
   all =
     let
       allCases = lib.filter lib.isDerivation
@@ -103,4 +123,4 @@ let
     in
     runCommand "build-all-testcases" { } script;
 in
-lib.recurseIntoAttrs (scopeStripped // { inherit all _allEmuResult; })
+lib.recurseIntoAttrs (scopeStripped // { inherit all _allEmuResult _allVCSEmuResult; })