From 1d58b45f470f6dc1b80a22b59041736120d43a4c Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Fri, 20 Dec 2024 16:25:24 -0500
Subject: [PATCH] dry run stolen from Henry (#765)

---
 .github/workflows/frontier/build.sh |  2 +-
 .github/workflows/phoenix/test.sh   |  2 +-
 .github/workflows/test.yml          |  2 +-
 toolchain/mfc/args.py               |  2 ++
 toolchain/mfc/test/test.py          | 14 ++++++++++++--
 5 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/frontier/build.sh b/.github/workflows/frontier/build.sh
index a6a51b65f..65d93d537 100644
--- a/.github/workflows/frontier/build.sh
+++ b/.github/workflows/frontier/build.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 . ./mfc.sh load -c f -m g
-./mfc.sh build -j 8 --gpu
+./mfc.sh test --dry-run -j 8 --gpu
diff --git a/.github/workflows/phoenix/test.sh b/.github/workflows/phoenix/test.sh
index 5cdc57e78..e89af4721 100644
--- a/.github/workflows/phoenix/test.sh
+++ b/.github/workflows/phoenix/test.sh
@@ -5,7 +5,7 @@ if [ "$job_device" == "gpu" ]; then
     build_opts="--gpu"
 fi
 
-./mfc.sh build -j 8 $build_opts
+./mfc.sh test --dry-run -j 8 $build_opts
 
 n_test_threads=8
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3bcde5114..205bfbbd9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -78,7 +78,7 @@ jobs:
 
       - name: Build
         run:  |
-          /bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} --${{ matrix.precision }} 
+          /bin/bash mfc.sh test --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} --${{ matrix.precision }} 
 
       - name: Test
         run:  |
diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py
index bda830d54..848868bdd 100644
--- a/toolchain/mfc/args.py
+++ b/toolchain/mfc/args.py
@@ -83,6 +83,8 @@ def add_common_arguments(p, mask = None):
     test.add_argument(      "--no-build",     action="store_true",                    default=False,      help="(Testing) Do not rebuild MFC.")
     test.add_argument(      "--no-examples",  action="store_true",                    default=False,      help="Do not test example cases." )
     test.add_argument("--case-optimization",  action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.")
+    test.add_argument(      "--dry-run",      action="store_true",                    default=False,      help="Build and generate case files but do not run tests.")
+
     test_meg = test.add_mutually_exclusive_group()
     test_meg.add_argument("--generate",          action="store_true", default=False, help="(Test Generation) Generate golden files.")
     test_meg.add_argument("--add-new-variables", action="store_true", default=False, help="(Test Generation) If new variables are found in D/ when running tests, add them to the golden files.")
diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py
index 04cf173f2..ca8c9137d 100644
--- a/toolchain/mfc/test/test.py
+++ b/toolchain/mfc/test/test.py
@@ -1,5 +1,5 @@
 import os, typing, shutil, time, itertools
-from random import sample
+from random import sample, seed
 
 import rich, rich.table
 
@@ -71,6 +71,8 @@ def __filter(cases_) -> typing.List[TestCase]:
     if ARG("percent") == 100:
         return cases, skipped_cases
 
+    seed(time.time())
+
     selected_cases = sample(cases, k=int(len(cases)*ARG("percent")/100.0))
     skipped_cases = [item for item in cases if item not in selected_cases]
 
@@ -173,6 +175,11 @@ def _handle_case(case: TestCase, devices: typing.Set[int]):
     tol = case.compute_tolerance()
     case.delete_output()
     case.create_directory()
+
+    if ARG("dry_run"):
+        cons.print(f"  [bold magenta]{case.get_uuid()}[/bold magenta]     SKIP     {case.trace}")
+        return
+
     cmd = case.run([PRE_PROCESS, SIMULATION], gpus=devices)
     out_filepath = os.path.join(case.get_dirpath(), "out_pre_sim.txt")
 
@@ -265,7 +272,10 @@ def handle_case(case: TestCase, devices: typing.Set[int]):
 
         try:
             _handle_case(case, devices)
-            nPASS += 1
+            if ARG("dry_run"):
+                nSKIP += 1
+            else:
+                nPASS += 1
         except Exception as exc:
             if nAttempts < max_attempts:
                 continue