Add flash test

data/cav5_comb4/3D/scalability/mkmsh

@@ -18,6 +18,7 @@ SAMPFAC="4"
 OUTPUT="actii"
 MKLINK="false"
 OVRWT="false"
+NELEM="0"
 
 while [[ $# -gt 0 ]]; do
   arg=$1
@@ -65,6 +66,9 @@ while [[ $# -gt 0 ]]; do
     --output=*)
         OUTPUT=${arg#*=}
         ;;
+    --nelem=*)
+        NELEM=${arg#*=}
+        ;;
     --link)
         MKLINK="true"
         ;;
@@ -94,7 +98,9 @@ printf "Cavity factor: ${CAVFAC}\n"
 printf "Injector factor: ${INJFAC}\n"
 printf "Shear layer factor: ${SHEARFAC}\n"
 printf "Sample factor: ${SAMPFAC}\n"
+
 if [[ "${OVRWT}" == "true" ]]; then
+    rm -f ${OUTPUT}_${NELEM}.msh
     rm -f ${OUTPUT}.msh
     MKLINK="true"
 fi
@@ -106,6 +112,13 @@ if [[ "${MKLINK}" == "true" ]]; then
     fi
     printf "Make link ${OUTPUT}.msh for finished mesh.\n"
 fi
+if [[ ! -f ${OUTPUT}_${NELEM}.msh ]]; then
+    date
+    set -x
+    rm -f tmp.msh
+    gmsh -setnumber size ${SIZE} -setnumber blratio ${BLRAT} -setnumber cavityfac ${CAVFAC} -setnumber isofac ${ISOFAC} -setnumber samplefac ${SAMPFAC} -setnumber injectorfac ${INJFAC} -setnumber blratiocavity ${BLCAVRAT} -setnumber blratioinjector ${BLINJRAT} -setnumber blratiosample ${BLSAMPRAT} -setnumber blratiosurround ${BLSURRRAT} -setnumber shearfac ${SHEARFAC} -o tmp.msh -nopopup -format msh2 ./actii_from_brep.geo -3 -nt ${NCPUS}
+    set +x
+    date
 
 date
 set -x
@@ -120,13 +133,11 @@ if [ -f "tmp.msh" ]; then
         printf "Mesh creation failed.\n"
         exit 1
     fi
-    mv tmp.msh ${OUTPUT}_${NCELLS}.msh
-    printf "Created ${OUTPUT}_${NCELLS}.msh with ${NCELLS} tets.\n"
+else
+    printf "Expected mesh file already exists, skipping mesh creation.\n"
     if [[ "${MKLINK}" == "true" ]]; then
-        ln -s ${OUTPUT}_${NCELLS}.msh ${OUTPUT}.msh
+        ln -s ${OUTPUT}_${NELEM}.msh ${OUTPUT}.msh
     fi
-else
-    printf "Mesh creation failed."
-    exit 1
 fi
 
+

data/cav5_comb4/3D/scalability/sizes.txt

@@ -9,4 +9,4 @@
 | 7.45                |  1,536k |      1,536,571 |
 | 5.8545              |  3,072k |      3,071,835 |
 | 4.61                |  6,144k |      6,142,150 |
-| 3.6389              | 12,288k |     12,283,032 |
+| 3.6389              | 12,288k |     12,283,032 |

flash_test/driver.py

@@ -0,0 +1 @@
+../driver.py

flash_test/flash_test.bsub.sh

@@ -0,0 +1,20 @@
+#! /bin/bash
+#BSUB -nnodes 1
+#BSUB -G uiuc
+#BSUB -W 360
+#BSUB -J flash9
+#BSUB -q pbatch
+#BSUB -o flash9.out
+
+source ../emirge/config/activate_env.sh
+source ../emirge/mirgecom/scripts/mirge-testing-env.sh
+
+set -x
+$MIRGE_MPI_EXEC -n 1 $MIRGE_PARALLEL_SPAWNER python -u -O -m mpi4py driver.py -c ${casename} -i run_params.yaml -r restart_data/prediction-000965000 --log --lazy
+set +x
+
+# flash5 000483000
+# flash6 000604000
+# flash7 000726000
+# flash8 000847000
+# flash9 000965000

flash_test/run_params.yaml

@@ -0,0 +1,56 @@
+# params to vary
+init_name: Flash1D
+order:  2
+mu: 1.e-3
+mesh_size: 0.0002
+current_dt: 1.e-9
+nviz: 1000
+# everything else
+init_shock: False
+init_flame: True
+viz_level: 3
+nrestart: 1000
+mesh_angle: 0
+periodic: False
+bl_ratio: 4.0
+interface_ratio: 1.0
+transfinite: True
+nhealth: 100
+nstatus: 100
+t_final: 1.e-2
+mach: 4.0
+pressure_background: 101325
+temperature_background: 300
+use_species_limiter: 1
+vel_sigma: 0
+temp_sigma: 0
+nspecies:  7
+use_av: False
+use_species_limiter: True
+noslip: True
+adiabatic: False
+#mu: 1.e-5
+#kappa: 2.0
+alpha_sc: 0.05
+s0_sc: -5.0
+kappa_sc: 0.5
+integrator: compiled_lsrk54
+#integrator: euler
+#inviscid_numerical_flux: hll
+health_pres_min: 0.001
+health_pres_max: 1000000
+#
+#  wall settings
+#
+wall_penalty_amount: 0
+wall_time_scale: 1
+use_wall_ox: False
+use_wall_mass: False
+use_sponge: False
+#
+# filter settings
+#
+soln_nfilter: -1
+use_rhs_filter: False
+soln_filter_cutoff: 4
+rhs_filter_cutoff: 3

scalability_test/data

@@ -0,0 +1 @@
+../data/cav5_comb4/3D/scalability

scalability_test/driver.py

@@ -0,0 +1 @@
+../driver.py

scalability_test/generate_lassen_scaling_job_script.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+NONOPT_ARGS=()
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -n|--nodes)
+            NUM_NODES="$2"
+            shift
+            shift
+            ;;
+        -e|--end)
+            NUM_PROCS="$2"
+            shift 
+            shift
+            ;;
+        -s|--start)
+            NUM_PROCS_1="$2"
+            shift 
+            shift
+            ;;
+        -o|--output)
+            OUTPUT_PATH="$2"
+            shift 
+            shift
+            ;;
+        -q|--queue)
+            QUEUE_NAME="$2"
+            shift 
+            shift
+            ;;
+        -t|--time)
+            TIME_LIMIT="$2"
+            shift 
+            shift
+            ;;
+        -h|--help)
+            printf "Generates batch script for scaling tests on Lassen.\n\nUsage:\n"
+            printf "./generate_lassen_scaling_job_script.sh -n <nnodes> -s <nproc_start> -e <nproc_end> -o <batch script name> -q <queue name> -t <walltime limit>\n\n"
+            printf "Default: scaling_test_lassen.bsub.sh = single node scaling job to run on 1-4 GPUs on batch queue with 120m time limit.\n\n"
+            printf "Submit the resulting script with the \`bsub\` command.\n"
+            exit 1
+            ;;
+        -*|--*)
+            echo "generate_lassen_scaling_job_script: Unknown option $1"
+            exit 1
+            ;;
+        *)
+            NONOPT_ARGS+=("$1")
+            shift
+            ;;
+    esac
+done
+set -- "${NONOPT_ARGS[@]}"
+
+
+NUM_NODES=${NUM_NODES:-"1"}
+NUM_PROCS_MAX=$(( 4 * ${NUM_NODES} ))
+NUM_PROCS=${NUM_PROCS:-"${NUM_PROCS_MAX}"}
+NUM_PROCS_1=${NUM_PROCS_1:-"1"}
+OUTPUT_PATH=${OUTPUT_PATH:-"scaling_test_lassen.bsub.sh"}
+QUEUE_NAME=${QUEUE_NAME:-"pbatch"}
+TIME_LIMIT=${TIME_LIMIT:-"120"}
+
+rm -f ${OUTPUT_PATH}
+cat <<EOF > ${OUTPUT_PATH}
+#!/bin/bash
+
+#BSUB -nnodes ${NUM_NODES}
+#BSUB -G uiuc
+#BSUB -W ${TIME_LIMIT}
+#BSUB -J scale${NUM_PROCS}
+#BSUB -q ${QUEUE_NAME}
+#BSUB -o scal${NUM_PROCS}.txt
+
+source ../emirge/config/activate_env.sh
+source ../emirge/mirgecom/scripts/mirge-testing-env.sh
+source ../scripts/multi_scalability.sh -p ../ -s ${NUM_PROCS_1} -n ${NUM_PROCS}
+
+EOF

scalability_test/runLassenBatch.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+#BSUB -nnodes 4
+#BSUB -G uiuc
+#BSUB -W 180
+#BSUB -J scale16
+#BSUB -q pbatch
+#BSUB -o scal16.txt
+
+source ../emirge/config/activate_env.sh
+source ../emirge/mirgecom/scripts/mirge-testing-env.sh
+source ../scripts/multi_scalability.sh -p ../ -s 1 -n 16

scalability_test/run_params.yaml

@@ -0,0 +1,69 @@
+##############
+# run control
+##############
+nviz: 1000
+nrestart: 1000
+nhealth: -1
+nstatus: -1
+ngarbage: 10
+viz_level: 1
+viz_interval_type: 0
+#viz_interval_type: 1
+#viz_interval_type: 2
+t_viz_interval: 1.e-9
+constant_cfl: 0
+#constant_cfl: 1
+current_cfl: 0.05
+current_dt: 1.0e-13
+t_final: 2.e-11
+mesh_filename: data/actii.msh
+dimen: 3
+
+##############
+# wall model
+##############
+wall_time_scale: 100
+wall_penalty_amount: 0
+wall_material: 2
+
+##############
+# model control
+##############
+use_species_limiter: True
+use_av: True
+alpha_sc: 0.01
+s0_sc: -5.0
+kappa_sc: 0.5
+order:  4
+# use_rhs_filter: True
+# rhs_filter_cutoff: 3
+
+##############
+# equation of state
+##############
+nspecies:  7
+pyro_temp_iter: 1
+pyro_temp_tol: 1.
+transport: 1
+
+##############
+# ignition parameters
+##############
+use_ignition: 2
+ignition_loc_x: 0.68
+ignition_loc_y: -0.01
+ignition_init_time: 5.e-8
+ignition_strength: 600
+ignition_duration: 1.e-8
+ignition_diameter: 0.02
+integrator: compiled_lsrk54
+#health_pres_min: 2828
+#health_pres_max: 274259
+
+##############
+# initialization
+##############
+vel_sigma: 500
+temp_sigma: 1250
+vel_sigma_inj: 5000
+temp_sigma_inj: 5000