From b2f178abee65264317793179edba92cee951b4fc Mon Sep 17 00:00:00 2001
From: Sebastian Grimberg <sjg@amazon.com>
Date: Tue, 4 Jun 2024 16:34:06 -0700
Subject: [PATCH] Add serial refinement capability for debugging

---
 docs/src/config/model.md         |  1 +
 palace/utils/configfile.cpp      |  5 ++++-
 palace/utils/configfile.hpp      |  3 +++
 palace/utils/geodata.cpp         | 16 ++++++++++++++++
 scripts/schema/config/model.json |  1 +
 5 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/docs/src/config/model.md b/docs/src/config/model.md
index 7a674e804..abb165898 100644
--- a/docs/src/config/model.md
+++ b/docs/src/config/model.md
@@ -125,3 +125,4 @@ mesh length units.
   - `"MaximumImbalance" [1.1]`
   - `"SaveAdaptIterations" [true]`
   - `"SaveAdaptMesh" [false]`
+  - `"SerialUniformLevels" [0]`
diff --git a/palace/utils/configfile.cpp b/palace/utils/configfile.cpp
index 72e8486a5..5f929324f 100644
--- a/palace/utils/configfile.cpp
+++ b/palace/utils/configfile.cpp
@@ -337,7 +337,8 @@ void RefinementData::SetUp(json &model)
 
   // Options for a priori refinement.
   uniform_ref_levels = refinement->value("UniformLevels", uniform_ref_levels);
-  MFEM_VERIFY(uniform_ref_levels >= 0,
+  ser_uniform_ref_levels = refinement->value("SerialUniformLevels", ser_uniform_ref_levels);
+  MFEM_VERIFY(uniform_ref_levels >= 0 && ser_uniform_ref_levels >= 0,
               "Number of uniform mesh refinement levels must be non-negative!");
   auto boxes = refinement->find("Boxes");
   if (boxes != refinement->end())
@@ -430,6 +431,7 @@ void RefinementData::SetUp(json &model)
   refinement->erase("SaveAdaptIterations");
   refinement->erase("SaveAdaptMesh");
   refinement->erase("UniformLevels");
+  refinement->erase("SerialUniformLevels");
   refinement->erase("Boxes");
   refinement->erase("Spheres");
   MFEM_VERIFY(refinement->empty(),
@@ -449,6 +451,7 @@ void RefinementData::SetUp(json &model)
     std::cout << "SaveAdaptIterations: " << save_adapt_iterations << '\n';
     std::cout << "SaveAdaptMesh: " << save_adapt_mesh << '\n';
     std::cout << "UniformLevels: " << uniform_ref_levels << '\n';
+    std::cout << "SerialUniformLevels: " << ser_uniform_ref_levels << '\n';
   }
 }
 
diff --git a/palace/utils/configfile.hpp b/palace/utils/configfile.hpp
index 3e65af1cc..c567d2382 100644
--- a/palace/utils/configfile.hpp
+++ b/palace/utils/configfile.hpp
@@ -171,6 +171,9 @@ struct RefinementData
   // Parallel uniform mesh refinement levels.
   int uniform_ref_levels = 0;
 
+  // Serial uniform mesh refinement levels.
+  int ser_uniform_ref_levels = 0;
+
 private:
   // Refinement data for mesh regions.
   std::vector<BoxRefinementData> box_list = {};
diff --git a/palace/utils/geodata.cpp b/palace/utils/geodata.cpp
index baecd8a3c..626a45ba1 100644
--- a/palace/utils/geodata.cpp
+++ b/palace/utils/geodata.cpp
@@ -123,6 +123,22 @@ std::unique_ptr<mfem::ParMesh> ReadMesh(MPI_Comm comm, const IoData &iodata)
       ReorderMeshElements(*smesh);
     }
 
+    // Refine the serial mesh (not typically used, prefer parallel uniform refinement
+    // instead).
+    {
+      int ne = smesh->GetNE();
+      for (int l = 0; l < iodata.model.refinement.ser_uniform_ref_levels; l++)
+      {
+        smesh->UniformRefinement();
+      }
+      if (iodata.model.refinement.ser_uniform_ref_levels > 0)
+      {
+        Mpi::Print("Serial uniform mesh refinement levels added {:d} elements (initial = "
+                   "{:d}, final = {:d})\n",
+                   smesh->GetNE() - ne, ne, smesh->GetNE());
+      }
+    }
+
     // Clean up unused domain elements from the mesh, add new boundary elements for material
     // interfaces if not present. Can only clean up conforming meshes, assumes that any
     // nonconformal mesh was generated by adaptation and thus does not need checking.
diff --git a/scripts/schema/config/model.json b/scripts/schema/config/model.json
index 556464dcd..cab779794 100644
--- a/scripts/schema/config/model.json
+++ b/scripts/schema/config/model.json
@@ -34,6 +34,7 @@
         "SaveAdaptIterations": {"type": "boolean"},
         "SaveAdaptMesh": {"type": "boolean"},
         "UniformLevels": { "type": "integer", "minimum": 0 },
+        "SerialUniformLevels": { "type": "integer", "minimum": 0 },
         "Boxes":
         {
           "type": "array",