Merge pull request #333 from glotzerlab/delta-openmpi-update

Updates for latest Delta software.

CHANGELOG.rst

@@ -8,6 +8,14 @@ and optionally the day separated by periods or hyphens.
 2023
 ----
 
+2023-10-12
+++++++++++
+
+*Changed*
+
+* Require openmpi/4.1.4 on Delta.
+* Recommend ``export OMPI_MCA_btl=self`` and ``srun`` when launching MPI jobs on Delta.
+
 2023-09-22
 ++++++++++
 

doc/clusters/delta.rst

@@ -37,24 +37,35 @@ container:
 
 Serial (or multithreaded) CPU jobs (``cpu`` partition)::
 
-    module load gcc/11.2.0 openmpi/4.1.2
-    mpirun -n 1 -x UCX_POSIX_USE_PROC_LINK=n singularity exec --bind /scratch /scratch/<your-account>/$USER/software.sif command arguments
+    module load gcc/11.2.0 openmpi/4.1.4
+    srun -n 1 singularity exec --bind /scratch /scratch/<your-account>/$USER/software.sif command arguments
 
 Single GPU jobs (``gpuA100x4`` and similar partitions)::
 
-    module load gcc/11.2.0 openmpi/4.1.2
-    mpirun -n 1 -x UCX_POSIX_USE_PROC_LINK=n singularity exec --nv --bind /scratch /scratch/<your-account>/$USER/software.sif command arguments
+    module load gcc/11.2.0 openmpi/4.1.4
+    srun -n 1 singularity exec --nv --bind /scratch /scratch/<your-account>/$USER/software.sif command arguments
 
 MPI parallel CPU jobs (``cpu`` partition with more than 1 core)::
 
-    module load gcc/11.2.0 openmpi/4.1.2
-    mpirun -x UCX_POSIX_USE_PROC_LINK=n singularity exec --bind /scratch /scratch/<your-account>/$USER/software.sif command arguments
+    module load gcc/11.2.0 openmpi/4.1.4
+    export OMPI_MCA_btl=self
+    srun singularity exec --bind /scratch /scratch/<your-account>/$USER/software.sif command arguments
 
 MPI parallel GPU jobs (``gpuA100x4`` and similar partitions with more than 1 GPU)::
 
-    module load gcc/11.2.0 openmpi/4.1.2
-    mpirun -x UCX_POSIX_USE_PROC_LINK=n singularity exec --nv --bind /scratch /scratch/<your-account>/$USER/software.sif command arguments
+    module load gcc/11.2.0 openmpi/4.1.4
+    export OMPI_MCA_btl=self
+    srun singularity exec --nv --bind /scratch /scratch/<your-account>/$USER/software.sif command arguments
+
+.. note::
+
+    Setting ``OMPI_MCA_btl=self`` prevents the warning:
+    ```
+    UCX  WARN  IB: ibv_fork_init() was disabled or failed, yet a fork() has been issued.
+    UCX  WARN  IB: data corruption might occur when using registered memory.
+    ```
+    OpenMPI uses ``ucx`` for internode communication and skips ``btl`` on Delta.
 
 .. tip::
 
-    You may use ``srun`` in place of ``mpirun`` on Delta.
+    You may use ``mpirun -x UCX_POSIX_USE_PROC_LINK=n`` in place of ``srun`` on Delta.

docker/delta/test/job-cpu.sh

@@ -2,21 +2,22 @@
 #SBATCH --job-name="test-cpu"
 #SBATCH --partition=cpu
 #SBATCH --nodes=2
-#SBATCH --ntasks=2
-#SBATCH --cpus-per-task=1
+#SBATCH --ntasks-per-node=1
 #SBATCH --export=ALL
 #SBATCH -t 0:10:00
 
-module load gcc/11.2.0 openmpi/4.1.2
+module load gcc/11.2.0 openmpi/4.1.4
 
 set -x
 
+export OMPI_MCA_btl=self
+
 singularity exec software.sif bash -c "set" | grep GLOTZERLAB
 
-mpirun -n 1 singularity exec software.sif python3 serial-cpu.py
+srun -n 1 singularity exec software.sif python3 serial-cpu.py
 
-mpirun --npernode 1 singularity exec software.sif python3 mpi-cpu.py
+srun singularity exec software.sif python3 mpi-cpu.py
 
-mpirun --npernode 1 singularity exec software.sif /opt/osu-micro-benchmarks/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_bibw
+srun singularity exec software.sif /opt/osu-micro-benchmarks/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_bibw
 
 echo "Tests complete."

docker/delta/test/job-gpu.sh

@@ -7,16 +7,18 @@
 #SBATCH --export=ALL
 #SBATCH -t 0:10:00
 
-module load gcc/11.2.0 openmpi/4.1.2
+module load gcc/11.2.0 openmpi/4.1.4
 
 set -x
 
+export OMPI_MCA_btl=self
+
 singularity exec software.sif bash -c "set" | grep GLOTZERLAB
 
-singularity exec --nv software.sif python3 serial-gpu.py
+srun -n 1 singularity exec --nv software.sif python3 serial-gpu.py
 
-mpirun -v -x UCX_POSIX_USE_PROC_LINK=n singularity exec --nv software.sif python3 mpi-gpu.py
+srun singularity exec --nv software.sif python3 mpi-gpu.py
 
-mpirun -v -x UCX_POSIX_USE_PROC_LINK=n singularity exec software.sif /opt/osu-micro-benchmarks/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_bibw
+srun singularity exec software.sif /opt/osu-micro-benchmarks/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_bibw
 
 echo "Tests complete."

make_dockerfiles.py

@@ -73,13 +73,13 @@ def write(fname, templates, **kwargs):
                                       openmpi_template,
                                       glotzerlab_software_template,
                                       finalize_template],
-          FROM='nvidia/cuda:11.6.2-devel-ubuntu20.04',
+          FROM='nvidia/cuda:11.7.1-devel-ubuntu20.04',
           system='delta',
-          CUDA_VERSION='11.6',
+          CUDA_VERSION='11.7',
           OPENMPI_VERSION='4.1',
-          OPENMPI_PATCHLEVEL='2',
-          UCX_VERSION='1.11.2',
-          PMIX_VERSION='3.2.3',
+          OPENMPI_PATCHLEVEL='4',
+          UCX_VERSION='1.12.1',
+          PMIX_VERSION='3.2.5',
           LIBFABRIC_VERSION='1.13.2',
           ENABLE_MPI='on',
           MAKEJOBS=multiprocessing.cpu_count()+2,

template/base.jinja

@@ -36,6 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   python3.9-dev \
   python3.9-venv \
   python3.9-distutils \
+  strace \
   zlib1g-dev \
   ca-certificates \
   && rm -rf /var/lib/apt/lists/*