diff --git a/experiments/frontier_metg_compute/metg_legion.sh b/experiments/frontier_metg_compute/metg_legion_rank1.sh similarity index 86% rename from experiments/frontier_metg_compute/metg_legion.sh rename to experiments/frontier_metg_compute/metg_legion_rank1.sh index 9ede83c2..de35627e 100644 --- a/experiments/frontier_metg_compute/metg_legion.sh +++ b/experiments/frontier_metg_compute/metg_legion_rank1.sh @@ -61,9 +61,9 @@ function sweep { for n in $SLURM_JOB_NUM_NODES; do for g in ${NGRAPHS:-1}; do for t in ${PATTERN:-stencil_1d}; do - sweep launch_util_0 $n $g $t > legion_util_0_ngraphs_${g}_type_${t}_nodes_${n}.log - # sweep launch_util_1 $n $g $t > legion_util_1_ngraphs_${g}_type_${t}_nodes_${n}.log - # sweep launch_util_2 $n $g $t > legion_util_2_ngraphs_${g}_type_${t}_nodes_${n}.log + sweep launch_util_0 $n $g $t > legion_util_0_rank1_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_1 $n $g $t > legion_util_1_rank1_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_2 $n $g $t > legion_util_2_rank1_ngraphs_${g}_type_${t}_nodes_${n}.log done done done diff --git a/experiments/frontier_metg_compute/metg_legion_socket.sh b/experiments/frontier_metg_compute/metg_legion_rank2.sh similarity index 80% rename from experiments/frontier_metg_compute/metg_legion_socket.sh rename to experiments/frontier_metg_compute/metg_legion_rank2.sh index 428d1aa0..774f31fc 100644 --- a/experiments/frontier_metg_compute/metg_legion_socket.sh +++ b/experiments/frontier_metg_compute/metg_legion_rank2.sh @@ -7,6 +7,11 @@ total_cores=56 cores=$(( $total_cores - 4 )) +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_DEVICE_0=dummy # https://gasnet-bugs.lbl.gov/bugzilla/show_bug.cgi?id=4669 +export GASNET_OFI_DEVICE_0_1=cxi1 +export GASNET_OFI_DEVICE_2_3=cxi0 + function launch_util_0 { memoize="-dm:memoize -lg:parallel_replay $(( cores / 2 ))" srun_flags= @@ -61,9 +66,9 @@ function sweep { for n in $SLURM_JOB_NUM_NODES; do for g in ${NGRAPHS:-1}; do for t in ${PATTERN:-stencil_1d}; do - # sweep launch_util_0 $n $g $t > legion_util_0_socket_ngraphs_${g}_type_${t}_nodes_${n}.log - # sweep launch_util_1 $n $g $t > legion_util_1_socket_ngraphs_${g}_type_${t}_nodes_${n}.log - sweep launch_util_2 $n $g $t > legion_util_2_socket_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_0 $n $g $t > legion_util_0_rank2_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_1 $n $g $t > legion_util_1_rank2_ngraphs_${g}_type_${t}_nodes_${n}.log + sweep launch_util_2 $n $g $t > legion_util_2_rank2_ngraphs_${g}_type_${t}_nodes_${n}.log done done done diff --git a/experiments/frontier_metg_compute/metg_legion_quad.sh b/experiments/frontier_metg_compute/metg_legion_rank4.sh similarity index 81% rename from experiments/frontier_metg_compute/metg_legion_quad.sh rename to experiments/frontier_metg_compute/metg_legion_rank4.sh index b8aa4073..fc248181 100644 --- a/experiments/frontier_metg_compute/metg_legion_quad.sh +++ b/experiments/frontier_metg_compute/metg_legion_rank4.sh @@ -7,6 +7,12 @@ total_cores=56 cores=$(( $total_cores - 8 )) +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_DEVICE_0=cxi2 +export GASNET_OFI_DEVICE_1=cxi1 +export GASNET_OFI_DEVICE_2=cxi3 +export GASNET_OFI_DEVICE_3=cxi0 + function launch_util_0 { memoize="-dm:memoize -lg:parallel_replay $(( cores / 4 ))" srun_flags= @@ -61,9 +67,9 @@ function sweep { for n in $SLURM_JOB_NUM_NODES; do for g in ${NGRAPHS:-1}; do for t in ${PATTERN:-stencil_1d}; do - # sweep launch_util_0 $n $g $t > legion_util_0_quad_ngraphs_${g}_type_${t}_nodes_${n}.log - # sweep launch_util_1 $n $g $t > legion_util_1_quad_ngraphs_${g}_type_${t}_nodes_${n}.log - sweep launch_util_2 $n $g $t > legion_util_2_quad_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_0 $n $g $t > legion_util_0_rank4_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_1 $n $g $t > legion_util_1_rank4_ngraphs_${g}_type_${t}_nodes_${n}.log + sweep launch_util_2 $n $g $t > legion_util_2_rank4_ngraphs_${g}_type_${t}_nodes_${n}.log done done done diff --git a/experiments/frontier_metg_compute/metg_realm.sh b/experiments/frontier_metg_compute/metg_realm_rank1.sh similarity index 100% rename from experiments/frontier_metg_compute/metg_realm.sh rename to experiments/frontier_metg_compute/metg_realm_rank1.sh diff --git a/experiments/frontier_metg_compute/metg_realm_quad.sh b/experiments/frontier_metg_compute/metg_realm_rank4.sh similarity index 89% rename from experiments/frontier_metg_compute/metg_realm_quad.sh rename to experiments/frontier_metg_compute/metg_realm_rank4.sh index b382b7b1..b61ad332 100644 --- a/experiments/frontier_metg_compute/metg_realm_quad.sh +++ b/experiments/frontier_metg_compute/metg_realm_rank4.sh @@ -7,6 +7,12 @@ total_cores=56 cores=$(( $total_cores - 4 )) +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_DEVICE_0=cxi2 +export GASNET_OFI_DEVICE_1=cxi1 +export GASNET_OFI_DEVICE_2=cxi3 +export GASNET_OFI_DEVICE_3=cxi0 + function launch { srun_flags= if (( $1 == 1 )); then diff --git a/experiments/frontier_metg_compute/metg_realm_socket.sh b/experiments/frontier_metg_compute/metg_realm_ranks2.sh similarity index 87% rename from experiments/frontier_metg_compute/metg_realm_socket.sh rename to experiments/frontier_metg_compute/metg_realm_ranks2.sh index e814d0aa..9bf12f13 100644 --- a/experiments/frontier_metg_compute/metg_realm_socket.sh +++ b/experiments/frontier_metg_compute/metg_realm_ranks2.sh @@ -7,6 +7,11 @@ total_cores=56 cores=$(( $total_cores - 2 )) +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_DEVICE_0=dummy # https://gasnet-bugs.lbl.gov/bugzilla/show_bug.cgi?id=4669 +export GASNET_OFI_DEVICE_0_1=cxi1 +export GASNET_OFI_DEVICE_2_3=cxi0 + function launch { srun_flags= if (( $1 == 1 )); then diff --git a/experiments/frontier_metg_compute/metg_regent.sh b/experiments/frontier_metg_compute/metg_regent_rank1.sh similarity index 92% rename from experiments/frontier_metg_compute/metg_regent.sh rename to experiments/frontier_metg_compute/metg_regent_rank1.sh index febce3f0..b2cd5923 100644 --- a/experiments/frontier_metg_compute/metg_regent.sh +++ b/experiments/frontier_metg_compute/metg_regent_rank1.sh @@ -61,9 +61,9 @@ function sweep { for n in $SLURM_JOB_NUM_NODES; do for g in ${NGRAPHS:-1}; do for t in ${PATTERN:-stencil_1d}; do - # sweep launch_util_0 $n $g $t > regent${VARIANT+_}${VARIANT}_util_0_ngraphs_${g}_type_${t}_nodes_${n}.log - # sweep launch_util_1 $n $g $t > regent${VARIANT+_}${VARIANT_}_util_1_ngraphs_${g}_type_${t}_nodes_${n}.log - sweep launch_util_2 $n $g $t > regent${VARIANT+_}${VARIANT}_util_2_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_0 $n $g $t > regent${VARIANT+_}${VARIANT}_util_0_rank1_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_1 $n $g $t > regent${VARIANT+_}${VARIANT_}_util_1_rank1_ngraphs_${g}_type_${t}_nodes_${n}.log + sweep launch_util_2 $n $g $t > regent${VARIANT+_}${VARIANT}_util_2_rank1_ngraphs_${g}_type_${t}_nodes_${n}.log done done done diff --git a/experiments/frontier_metg_compute/metg_regent_socket.sh b/experiments/frontier_metg_compute/metg_regent_rank2.sh similarity index 86% rename from experiments/frontier_metg_compute/metg_regent_socket.sh rename to experiments/frontier_metg_compute/metg_regent_rank2.sh index 1a3d4efd..150c82c0 100644 --- a/experiments/frontier_metg_compute/metg_regent_socket.sh +++ b/experiments/frontier_metg_compute/metg_regent_rank2.sh @@ -7,6 +7,11 @@ total_cores=56 cores=$(( $total_cores - 4 )) +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_DEVICE_0=dummy # https://gasnet-bugs.lbl.gov/bugzilla/show_bug.cgi?id=4669 +export GASNET_OFI_DEVICE_0_1=cxi1 +export GASNET_OFI_DEVICE_2_3=cxi0 + function launch_util_0 { memoize="-dm:memoize -lg:parallel_replay $(( cores / 2 ))" srun_flags= @@ -61,9 +66,9 @@ function sweep { for n in $SLURM_JOB_NUM_NODES; do for g in ${NGRAPHS:-1}; do for t in ${PATTERN:-stencil_1d}; do - # sweep launch_util_0 $n $g $t > regent${VARIANT+_}${VARIANT}_util_0_socket_ngraphs_${g}_type_${t}_nodes_${n}.log - # sweep launch_util_1 $n $g $t > regent${VARIANT+_}${VARIANT}_util_1_socket_ngraphs_${g}_type_${t}_nodes_${n}.log - sweep launch_util_2 $n $g $t > regent${VARIANT+_}${VARIANT}_util_2_socket_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_0 $n $g $t > regent${VARIANT+_}${VARIANT}_util_0_rank2_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_1 $n $g $t > regent${VARIANT+_}${VARIANT}_util_1_rank2_ngraphs_${g}_type_${t}_nodes_${n}.log + sweep launch_util_2 $n $g $t > regent${VARIANT+_}${VARIANT}_util_2_rank2_ngraphs_${g}_type_${t}_nodes_${n}.log done done done diff --git a/experiments/frontier_metg_compute/metg_regent_quad.sh b/experiments/frontier_metg_compute/metg_regent_rank4.sh similarity index 87% rename from experiments/frontier_metg_compute/metg_regent_quad.sh rename to experiments/frontier_metg_compute/metg_regent_rank4.sh index 8dd36f17..17011a7b 100644 --- a/experiments/frontier_metg_compute/metg_regent_quad.sh +++ b/experiments/frontier_metg_compute/metg_regent_rank4.sh @@ -7,6 +7,12 @@ total_cores=56 cores=$(( $total_cores - 8 )) +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_DEVICE_0=cxi2 +export GASNET_OFI_DEVICE_1=cxi1 +export GASNET_OFI_DEVICE_2=cxi3 +export GASNET_OFI_DEVICE_3=cxi0 + function launch_util_0 { memoize="-dm:memoize -lg:parallel_replay $(( cores / 4 ))" srun_flags= @@ -61,9 +67,9 @@ function sweep { for n in $SLURM_JOB_NUM_NODES; do for g in ${NGRAPHS:-1}; do for t in ${PATTERN:-stencil_1d}; do - # sweep launch_util_0 $n $g $t > regent${VARIANT+_}${VARIANT}_util_0_quad_ngraphs_${g}_type_${t}_nodes_${n}.log - # sweep launch_util_1 $n $g $t > regent${VARIANT+_}${VARIANT}_util_1_quad_ngraphs_${g}_type_${t}_nodes_${n}.log - sweep launch_util_2 $n $g $t > regent${VARIANT+_}${VARIANT}_util_2_quad_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_0 $n $g $t > regent${VARIANT+_}${VARIANT}_util_0_rank4_ngraphs_${g}_type_${t}_nodes_${n}.log + # sweep launch_util_1 $n $g $t > regent${VARIANT+_}${VARIANT}_util_1_rank4_ngraphs_${g}_type_${t}_nodes_${n}.log + sweep launch_util_2 $n $g $t > regent${VARIANT+_}${VARIANT}_util_2_rank4_ngraphs_${g}_type_${t}_nodes_${n}.log done done done