From ff5c879b1593cabed7d380593300542d20312499 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Wed, 14 Feb 2024 09:38:45 -0500 Subject: [PATCH 01/24] Add hoomd conda recipe. --- conda/conda-build-hodges.sh | 6 ++++ conda/conda_build_config.yaml | 24 +++++++++++++++ conda/hoomd/build_hoomd.sh | 16 ++++++++++ conda/hoomd/meta.yaml | 55 +++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+) create mode 100755 conda/conda-build-hodges.sh create mode 100644 conda/conda_build_config.yaml create mode 100644 conda/hoomd/build_hoomd.sh create mode 100644 conda/hoomd/meta.yaml diff --git a/conda/conda-build-hodges.sh b/conda/conda-build-hodges.sh new file mode 100755 index 00000000..d7c27808 --- /dev/null +++ b/conda/conda-build-hodges.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +conda build "$@" \ + --skip-existing \ + --variants "{'cluster': ['hodges'], 'device': ['gpu']}" \ + --output-folder $HOME/test/channel diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml new file mode 100644 index 00000000..35cf5970 --- /dev/null +++ b/conda/conda_build_config.yaml @@ -0,0 +1,24 @@ +# From conda-forge-pinning +zip_keys: + - + - python + - numpy + +python: + # part of a zip_keys: python, numpy +# - 3.8.* *_cpython +# - 3.9.* *_cpython + - 3.10.* *_cpython + - 3.11.* *_cpython + - 3.12.* *_cpython + +numpy: + # part of a zip_keys: python, numpy +# - 1.22 +# - 1.22 + - 1.22 + - 1.23 + - 1.26 + +pybind11_abi: + - 4 diff --git a/conda/hoomd/build_hoomd.sh b/conda/hoomd/build_hoomd.sh new file mode 100644 index 00000000..5be01dcd --- /dev/null +++ b/conda/hoomd/build_hoomd.sh @@ -0,0 +1,16 @@ +if [[ $1 == "gpu" ]]; then + ENABLE_GPU="on" +fi + +cmake -B build \ + ${CMAKE_ARGS} \ + -DENABLE_MPI=on \ + -DENABLE_GPU=${ENABLE_GPU:-off} \ + -DBUILD_TESTING=off \ + -DENABLE_TBB=off \ + -DENABLE_LLVM=off \ + -DPLUGINS="" \ + -GNinja + +ninja -C build +ninja -C build install diff --git a/conda/hoomd/meta.yaml b/conda/hoomd/meta.yaml new file mode 100644 index 00000000..ffc920e2 --- /dev/null +++ b/conda/hoomd/meta.yaml @@ -0,0 +1,55 @@ +{% set name = "hoomd" %} +{% set version = "4.5.0" %} +{% set build = 0 %} +{% set sha256 = "b44f3dc9aa6b60c6bb5fe7c668d72af7eb1a52a4ebf5abe1849e9bb92da88667" %} + + +package: + name: {{ name }} + version: {{ version }} + +source: + fn: {{ name }}-{{ version }}.tar.gz + url: https://github.com/glotzerlab/hoomd-blue/releases/download/v{{ version }}/{{ name }}-{{ version }}.tar.gz + sha256: {{ sha256 }} + +build: + number: {{ build }} + string: {{ cluster }}_{{ device }}_py{{ py }}h{{ PKG_HASH }}_{{ build }} + script: bash ${RECIPE_DIR}/build_hoomd.sh {{ device }} + +# rpaths: +# - lib/ +# - lib/python3.12/site-packages/{{ name }} # [py==312] +# - lib/python3.11/site-packages/{{ name }} # [py==311] +# - lib/python3.10/site-packages/{{ name }} # [py==310] +# - lib/python3.9/site-packages/{{ name }} # [py==39] + +requirements: + build: + - cmake + - ninja + - pkg-config + + host: + - cereal + - eigen + - numpy + - pybind11 + - pybind11-abi + - python + + run: + - python + - numpy + +test: + requires: + - pytest + - rowan + imports: + - hoomd + - hoomd.md + - hoomd.hpmc + commands: + - pytest --pyargs hoomd -v From 6744d8984f4a361d4b1e5cfd236fc131ba4a9a4e Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Wed, 14 Feb 2024 10:38:58 -0500 Subject: [PATCH 02/24] Add Great Lakes build script. --- conda/conda-build-greatlakes.sh | 22 ++++++++++++++++++++++ conda/hoomd/build_hoomd.sh | 1 + 2 files changed, 23 insertions(+) create mode 100755 conda/conda-build-greatlakes.sh diff --git a/conda/conda-build-greatlakes.sh b/conda/conda-build-greatlakes.sh new file mode 100755 index 00000000..ab91f12a --- /dev/null +++ b/conda/conda-build-greatlakes.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Please use a compute node to build packages. +# srun -Asglotzer9 --nodes=1 --ntasks-per-node=1 --cpus-per-task=36 --partition standard -t 8:00:00 --mem=144G --pty $SHELL + +# Load modules used to build packages with native MPI and CUDA support. +module reset +module load gcc/10.3.0 openmpi/4.1.6 cuda/12.3.0 + +# Build on a node local SSD. +export CONDA_BLD_PATH=`mktemp --directory --tmpdir=/tmpssd conda-build-${USER}.XXXXXXXX` + +function cleanup { + rm -rf "$CONDA_BLD_PATH" +} + +trap cleanup EXIT + +conda build "$@" \ + --skip-existing \ + --variants "{'cluster': ['greatlakes'], 'device': ['gpu']}" \ + --output-folder /nfs/turbo/glotzer/software/conda diff --git a/conda/hoomd/build_hoomd.sh b/conda/hoomd/build_hoomd.sh index 5be01dcd..b2a6c511 100644 --- a/conda/hoomd/build_hoomd.sh +++ b/conda/hoomd/build_hoomd.sh @@ -10,6 +10,7 @@ cmake -B build \ -DENABLE_TBB=off \ -DENABLE_LLVM=off \ -DPLUGINS="" \ + -DCMAKE_CXX_FLAGS=-march=native -DCMAKE_C_FLAGS=-march=native \ -GNinja ninja -C build From 0d650a4e93eb365762e7a7b665dca5fddeb91aca Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Wed, 14 Feb 2024 10:44:19 -0500 Subject: [PATCH 03/24] Skip pytest tests. --- conda/hoomd/meta.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conda/hoomd/meta.yaml b/conda/hoomd/meta.yaml index ffc920e2..442f3188 100644 --- a/conda/hoomd/meta.yaml +++ b/conda/hoomd/meta.yaml @@ -51,5 +51,6 @@ test: - hoomd - hoomd.md - hoomd.hpmc - commands: - - pytest --pyargs hoomd -v +# pytest hangs for unknown reasons on Great Lakes +# commands: +# - pytest --pyargs hoomd -v From 80c555c4479a11578d8555c8e9342d379a89b68f Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Wed, 14 Feb 2024 12:20:35 -0500 Subject: [PATCH 04/24] Working build on Great Lakes. --- conda/.condarc | 2 ++ conda/conda-build-greatlakes.sh | 3 +++ conda/example-condarc | 9 +++++++++ conda/hoomd/meta.yaml | 18 +++++++++--------- 4 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 conda/.condarc create mode 100644 conda/example-condarc diff --git a/conda/.condarc b/conda/.condarc new file mode 100644 index 00000000..28816df7 --- /dev/null +++ b/conda/.condarc @@ -0,0 +1,2 @@ +conda-build: + pkg_format: 2 diff --git a/conda/conda-build-greatlakes.sh b/conda/conda-build-greatlakes.sh index ab91f12a..6399aba9 100755 --- a/conda/conda-build-greatlakes.sh +++ b/conda/conda-build-greatlakes.sh @@ -7,6 +7,9 @@ module reset module load gcc/10.3.0 openmpi/4.1.6 cuda/12.3.0 +# Build package format 2 +export CONDARC=.condarc + # Build on a node local SSD. export CONDA_BLD_PATH=`mktemp --directory --tmpdir=/tmpssd conda-build-${USER}.XXXXXXXX` diff --git a/conda/example-condarc b/conda/example-condarc new file mode 100644 index 00000000..dbcd2121 --- /dev/null +++ b/conda/example-condarc @@ -0,0 +1,9 @@ +channel_priority: strict +channels: + - file://nfs/turbo/glotzer/software/conda + - conda-forge + +disallow: +- openmpi +- mpich +- cuda-cudart-dev diff --git a/conda/hoomd/meta.yaml b/conda/hoomd/meta.yaml index 442f3188..9f9daa6b 100644 --- a/conda/hoomd/meta.yaml +++ b/conda/hoomd/meta.yaml @@ -43,14 +43,14 @@ requirements: - python - numpy -test: - requires: - - pytest - - rowan - imports: - - hoomd - - hoomd.md - - hoomd.hpmc -# pytest hangs for unknown reasons on Great Lakes +# Tests hang for unknown reasons on great lakes. +# test: +# requires: +# - pytest +# - rowan +# imports: +# - hoomd +# - hoomd.md +# - hoomd.hpmc # commands: # - pytest --pyargs hoomd -v From 89354343da44f37a043efe80e918e9b619a46c6e Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Thu, 22 Feb 2024 10:21:08 -0500 Subject: [PATCH 05/24] Mention salloc. --- conda/conda-build-greatlakes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/conda-build-greatlakes.sh b/conda/conda-build-greatlakes.sh index 6399aba9..0ed96f18 100755 --- a/conda/conda-build-greatlakes.sh +++ b/conda/conda-build-greatlakes.sh @@ -1,7 +1,7 @@ #!/bin/bash # Please use a compute node to build packages. -# srun -Asglotzer9 --nodes=1 --ntasks-per-node=1 --cpus-per-task=36 --partition standard -t 8:00:00 --mem=144G --pty $SHELL +# salloc -Asglotzer9 --nodes=1 --ntasks-per-node=1 --cpus-per-task=36 --partition standard -t 8:00:00 --mem=144G # Load modules used to build packages with native MPI and CUDA support. module reset From ebaf7bdaf5fe1b85da800bd577ce6554820441ea Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Thu, 22 Feb 2024 18:06:31 -0600 Subject: [PATCH 06/24] Add delta build script. --- .gitignore | 1 + conda/conda-build-delta.sh | 25 +++++++++++++++++++++++++ conda/conda-build-hodges.sh | 6 ------ conda/conda_build_config.yaml | 8 ++++---- conda/job-build-all-delta.sh | 14 ++++++++++++++ 5 files changed, 44 insertions(+), 10 deletions(-) create mode 100755 conda/conda-build-delta.sh delete mode 100755 conda/conda-build-hodges.sh create mode 100644 conda/job-build-all-delta.sh diff --git a/.gitignore b/.gitignore index 49f11899..7b25f58c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ *.sublime-workspace *.simg Dockerfile +slurm*.out diff --git a/conda/conda-build-delta.sh b/conda/conda-build-delta.sh new file mode 100755 index 00000000..483348ee --- /dev/null +++ b/conda/conda-build-delta.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Please use a compute node to build packages. +# salloc --account=bbgw-delta-gpu --partition=gpuA40x4 --nodes=1 --tasks=1 --tasks-per-node=1 --cpus-per-task=16 --mem=48g --gpus=1 --time 4:00:00 + +# Load modules used to build packages with native MPI and CUDA support. +module reset +module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0 + +# Build package format 2 +export CONDARC=.condarc + +# Build on a node local SSD. +export CONDA_BLD_PATH=`mktemp --directory --tmpdir=/tmp conda-build-${USER}.XXXXXXXX` + +function cleanup { + rm -rf "$CONDA_BLD_PATH" +} + +trap cleanup EXIT + +conda build "$@" \ + --skip-existing \ + --variants "{'cluster': ['delta'], 'device': ['gpu']}" \ + --output-folder /projects/bbgw/software/conda diff --git a/conda/conda-build-hodges.sh b/conda/conda-build-hodges.sh deleted file mode 100755 index d7c27808..00000000 --- a/conda/conda-build-hodges.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -conda build "$@" \ - --skip-existing \ - --variants "{'cluster': ['hodges'], 'device': ['gpu']}" \ - --output-folder $HOME/test/channel diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml index 35cf5970..0c9e1af2 100644 --- a/conda/conda_build_config.yaml +++ b/conda/conda_build_config.yaml @@ -6,16 +6,16 @@ zip_keys: python: # part of a zip_keys: python, numpy -# - 3.8.* *_cpython -# - 3.9.* *_cpython + - 3.8.* *_cpython + - 3.9.* *_cpython - 3.10.* *_cpython - 3.11.* *_cpython - 3.12.* *_cpython numpy: # part of a zip_keys: python, numpy -# - 1.22 -# - 1.22 + - 1.22 + - 1.22 - 1.22 - 1.23 - 1.26 diff --git a/conda/job-build-all-delta.sh b/conda/job-build-all-delta.sh new file mode 100644 index 00000000..010afa65 --- /dev/null +++ b/conda/job-build-all-delta.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +#SBATCH --job-name="glotzerlab-software build" +#SBATCH -A bbgw-delta-gpu +#SBATCH --partition=gpuA40x4 +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=48g +#SBATCH --gpus=1 +#SBATCH --time 4:00:00 + +# TODO: rewrite to use CPU account when it is renewed +./conda-build-delta.sh hoomd From 5442822794455f2a24045b9a793d5cc68f385152 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Thu, 22 Feb 2024 19:44:15 -0500 Subject: [PATCH 07/24] Add great lakes build job. --- conda/job-build-all-greatlakes.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 conda/job-build-all-greatlakes.sh diff --git a/conda/job-build-all-greatlakes.sh b/conda/job-build-all-greatlakes.sh new file mode 100644 index 00000000..5d6fb738 --- /dev/null +++ b/conda/job-build-all-greatlakes.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +#SBATCH --job-name="glotzerlab-software build" +#SBATCH -A sglotzer9 +#SBATCH --partition=standard +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=1 +#SBATCH --cpus-per-task=36 +#SBATCH --mem=144g +#SBATCH --time 4:00:00 + +./conda-build-greatlakes.sh hoomd From fe380188e43403cbc6e14fa74e8ccda81c55419b Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Fri, 23 Feb 2024 09:25:01 -0500 Subject: [PATCH 08/24] Add mpi4py. --- conda/conda-build-anvil.sh | 22 +++++++++++++++++++++ conda/job-build-all-anvil.sh | 11 +++++++++++ conda/job-build-all-delta.sh | 6 +++--- conda/job-build-all-greatlakes.sh | 7 ++++--- conda/mpi4py/meta.yaml | 32 +++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 6 deletions(-) create mode 100755 conda/conda-build-anvil.sh create mode 100644 conda/job-build-all-anvil.sh create mode 100644 conda/mpi4py/meta.yaml diff --git a/conda/conda-build-anvil.sh b/conda/conda-build-anvil.sh new file mode 100755 index 00000000..8230ba45 --- /dev/null +++ b/conda/conda-build-anvil.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Load modules used to build packages with native MPI support (no CUDA). +module reset +module load gcc/11.2.0 openmpi/4.1.6 + +# Build package format 2 +export CONDARC=.condarc + +# Build on a node local SSD. +export CONDA_BLD_PATH=`mktemp --directory --tmpdir=/tmp conda-build-${USER}.XXXXXXXX` + +function cleanup { + rm -rf "$CONDA_BLD_PATH" +} + +trap cleanup EXIT + +conda build "$@" \ + --skip-existing \ + --variants "{'cluster': ['anvil'], 'device': ['cpu']}" \ + --output-folder /anvil/projects/x-dmr140129/software/conda diff --git a/conda/job-build-all-anvil.sh b/conda/job-build-all-anvil.sh new file mode 100644 index 00000000..f74508da --- /dev/null +++ b/conda/job-build-all-anvil.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +#SBATCH --job-name="glotzerlab-software build" +#SBATCH --account=dmr140129 +#SBATCH --partition=shared +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --time=8:00:00 + +./conda-build-anvil.sh hoomd mpi4py + diff --git a/conda/job-build-all-delta.sh b/conda/job-build-all-delta.sh index 010afa65..f54e801f 100644 --- a/conda/job-build-all-delta.sh +++ b/conda/job-build-all-delta.sh @@ -1,14 +1,14 @@ #!/bin/bash #SBATCH --job-name="glotzerlab-software build" -#SBATCH -A bbgw-delta-gpu +#SBATCH --account=bbgw-delta-gpu #SBATCH --partition=gpuA40x4 #SBATCH --nodes=1 #SBATCH --tasks-per-node=1 #SBATCH --cpus-per-task=16 #SBATCH --mem=48g #SBATCH --gpus=1 -#SBATCH --time 4:00:00 +#SBATCH --time=8:00:00 # TODO: rewrite to use CPU account when it is renewed -./conda-build-delta.sh hoomd +./conda-build-delta.sh hoomd mpi4py diff --git a/conda/job-build-all-greatlakes.sh b/conda/job-build-all-greatlakes.sh index 5d6fb738..15e5c38e 100644 --- a/conda/job-build-all-greatlakes.sh +++ b/conda/job-build-all-greatlakes.sh @@ -1,12 +1,13 @@ #!/bin/bash #SBATCH --job-name="glotzerlab-software build" -#SBATCH -A sglotzer9 +#SBATCH --account=sglotzer9 #SBATCH --partition=standard #SBATCH --nodes=1 #SBATCH --tasks-per-node=1 #SBATCH --cpus-per-task=36 #SBATCH --mem=144g -#SBATCH --time 4:00:00 +#SBATCH --time=8:00:00 + +./conda-build-greatlakes.sh hoomd mpi4py -./conda-build-greatlakes.sh hoomd diff --git a/conda/mpi4py/meta.yaml b/conda/mpi4py/meta.yaml new file mode 100644 index 00000000..d080a396 --- /dev/null +++ b/conda/mpi4py/meta.yaml @@ -0,0 +1,32 @@ +# Adapted from: https://github.com/conda-forge/mpi4py-feedstock/blob/main/recipe/meta.yaml + +{% set version = "3.1.5" %} +{% set build = 0 %} + +package: + name: mpi4py + version: {{ version }} + +source: + - fn: mpi4py-{{ version }}.tar.gz + url: https://github.com/mpi4py/mpi4py/releases/download/{{ version }}/mpi4py-{{ version }}.tar.gz + sha256: a706e76db9255135c2fb5d1ef54cb4f7b0e4ad9e33cbada7de27626205f2a153 + +build: + number: {{ build }} + string: {{ cluster }}_py{{ py }}h{{ PKG_HASH }}_{{ build }} + script: + - {{ PYTHON }} conf/cythonize.py # TODO: remove for mpi4py 4.0.0 + - {{ PYTHON }} -m pip install --no-deps . -vv + +requirements: + build: + - python # [build_platform != target_platform] + - cython <3 # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + host: + - python + - pip + - cython <3 + run: + - python From 5deb31f3c58a629757d99f1c9a1713dc9a8fab77 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Fri, 23 Feb 2024 14:37:16 -0500 Subject: [PATCH 09/24] Document conda packages. --- .vscode/settings.json | 5 -- README.md | 62 +++++++++++++++----- conda/example-condarc | 9 --- doc/cluster.rst | 2 +- doc/clusters/anvil.rst | 8 +-- doc/clusters/bridges2.rst | 8 +-- doc/clusters/crusher.rst | 2 +- doc/clusters/delta.rst | 8 +-- doc/clusters/expanse.rst | 16 +---- doc/clusters/frontier.rst | 2 +- doc/clusters/greatlakes.rst | 8 +-- doc/community.rst | 8 +-- doc/conda-build.rst | 33 +++++++++++ doc/conda-install.rst | 92 +++++++++++++++++++++++++++++ doc/conda.rst | 23 ++++++++ doc/container.rst | 85 +++++++++++++++++++++++++++ doc/docker.rst | 2 +- doc/files.rst | 6 +- doc/index.rst | 93 ++++-------------------------- doc/profiling.rst | 6 +- doc/software.rst | 2 +- doc/test.rst | 2 +- glotzerlab-software.code-workspace | 7 --- 23 files changed, 310 insertions(+), 179 deletions(-) delete mode 100644 .vscode/settings.json delete mode 100644 conda/example-condarc create mode 100644 doc/conda-build.rst create mode 100644 doc/conda-install.rst create mode 100644 doc/conda.rst create mode 100644 doc/container.rst delete mode 100644 glotzerlab-software.code-workspace diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 172b1e8d..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "editor.rulers": [ - 100 - ] -} diff --git a/README.md b/README.md index df8e8ba5..bdb0941a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,57 @@ # Glotzerlab software -**glotzerlab-software** is a set of container images that provide software developed by the [Glotzer +**glotzerlab-software** deploys software developed by the [Glotzer group]() at the [University of Michigan](https://www.umich.edu/) -along with related software commonly used in simulation and data analysis workflows. +to HPC resources via conda packages and container images. Use **glotzerlab-software** to install +a MPI and GPU enabled build of HOOMD-blue on a *HPC resource*. Use the standard conda-forge +provided HOOMD-blue (`mamba install hoomd` with no special configuration) for serial execution +on individual workstations. ## Resources * [Documentation](https://glotzerlab-software.readthedocs.io/): - Tutorial, usage information, and list of supported clusters. + Tutorial, usage information, and list of supported HPC resources. * [glotzerlab-software discussion board](https://github.com/glotzerlab/software/discussions): Ask the **glotzerlab-software** user community for help. -* [glotzerlab/software on Docker Hub](https://hub.docker.com/r/glotzerlab/software): - See the list of published images. +* [HOOMD-blue](https://glotzerlab.engin.umich.edu/hoomd-blue/): The general-purpose particle + simulation toolkit that **glotzerlab-software** packages. -## Quick start +## Quick start [conda] -First, download the stable **glotzerlab/software** image using +1. Install [miniforge](https://github.com/conda-forge/miniforge). +2. Replace `$HOME/miniforge3/.condarc` with: + + channel_priority: strict + channels: + - file://path/to/conda/channel + - conda-forge + + disallow: + - openmpi + - mpich + - cuda-cudart-dev + +3. Install HOOMD-blue: + + mamba install hoomd + +4. Load prerequisite modules. For example: + + module load openmpi cuda + +5. Execute Python scripts with MPI and GPU support. For example: + + srun -n 8 python3 script.py + +You can confirm that the correct package was installed with `mamba list hoomd`. The full package +build will contain the cluster name and a `gpu` or `cpu` suffix. For example: + + # Name Version Build Channel + hoomd X.Y.Z cluster_gpu_py311h1b32822_0 file://path/to/conda/channel + +## Quick start [container] + +Download the stable **glotzerlab/software** image using [Singularity](https://sylabs.io/singularity/): $ singularity pull software.sif docker://glotzerlab/software @@ -29,8 +65,6 @@ Add the ``--nv`` option to **enable NVIDIA GPUs** inside the container: $ singularity exec --nv software.sif nvidia-smi -## Updates - **glotzerlab-software** updates regularly with the latest versions of included software. Download the latest image to update: @@ -42,9 +76,7 @@ See [CHANGELOG.rst](CHANGELOG.rst). ## Contributing -Contributions are welcomed via -[pull requests](https://github.com/glotzerlab/software/pulls). -Please report bugs and suggest feature enhancements via the -[issue tracker](https://github.com/glotzerlab/software/issues). -See [ARCHITECTURE.md](ARCHITECTURE.md) for information on how the repository is structured, -including how to modify the containers. +Contributions are welcomed via [pull requests](https://github.com/glotzerlab/software/pulls). Please +report bugs and suggest feature enhancements via the [issue +tracker](https://github.com/glotzerlab/software/issues). See [ARCHITECTURE.md](ARCHITECTURE.md) for +information on how the repository is structured, including how to modify the containers. diff --git a/conda/example-condarc b/conda/example-condarc deleted file mode 100644 index dbcd2121..00000000 --- a/conda/example-condarc +++ /dev/null @@ -1,9 +0,0 @@ -channel_priority: strict -channels: - - file://nfs/turbo/glotzer/software/conda - - conda-forge - -disallow: -- openmpi -- mpich -- cuda-cudart-dev diff --git a/doc/cluster.rst b/doc/cluster.rst index 041da915..3b033b19 100644 --- a/doc/cluster.rst +++ b/doc/cluster.rst @@ -1,5 +1,5 @@ HPC clusters -============ +------------ **glotzerlab-software** provides a number of cluster-specific images, needed to enable high performance CPU, CUDA, and multi-node MPI support matching the drivers and hardware available on diff --git a/doc/clusters/anvil.rst b/doc/clusters/anvil.rst index a098c8ab..7d5a9393 100644 --- a/doc/clusters/anvil.rst +++ b/doc/clusters/anvil.rst @@ -1,5 +1,5 @@ Anvil (Purdue) ---------------- +************** Anvil_ is an HPC cluster at NCSA with GPU and CPU nodes. Apply for resources on Anvil through the ACCESS_ program. @@ -7,9 +7,6 @@ the ACCESS_ program. .. _Anvil: https://www.rcac.purdue.edu/knowledge/anvil .. _ACCESS: https://allocations.access-ci.org -Downloading -*********** - The **glotzerlab-software** image and the singularity cache are large, store them in your scratch directory:: @@ -20,9 +17,6 @@ Download the image with support for Anvil:: $ singularity pull software.sif docker://glotzerlab/software:anvil -Using -***** - Use the following commands in your job scripts or interactively to execute software inside the container: diff --git a/doc/clusters/bridges2.rst b/doc/clusters/bridges2.rst index ed2613a3..ebc1caae 100644 --- a/doc/clusters/bridges2.rst +++ b/doc/clusters/bridges2.rst @@ -1,12 +1,9 @@ Bridges-2 (PSC) ---------------- +*************** `Bridges-2 `_ is an HPC cluster at PSC with GPU and CPU nodes. Apply for resources on Bridges through the `ACCESS `_ program. -Downloading -*********** - The **glotzerlab-software** image and the singularity cache are large, store them in your project directory:: @@ -17,9 +14,6 @@ Download the image with support for Bridges-2:: $ singularity pull software.sif docker://glotzerlab/software:bridges2 -Using -***** - Use the following commands in your job scripts or interactively to execute software inside the container: diff --git a/doc/clusters/crusher.rst b/doc/clusters/crusher.rst index 9e4971ab..61a60a76 100644 --- a/doc/clusters/crusher.rst +++ b/doc/clusters/crusher.rst @@ -1,5 +1,5 @@ Crusher (OLCF) --------------- +************** `Crusher `_ is a system at ORNL with identical hardware and similar software as the upcoming Frontier system. diff --git a/doc/clusters/delta.rst b/doc/clusters/delta.rst index 7b5ff121..c3840193 100644 --- a/doc/clusters/delta.rst +++ b/doc/clusters/delta.rst @@ -1,5 +1,5 @@ Delta (NCSA) ------------- +************* Delta_ is an HPC cluster at NCSA with GPU and CPU nodes. Apply for resources on Delta through the ACCESS_ program. @@ -7,9 +7,6 @@ the ACCESS_ program. .. _Delta: https://ncsa-delta-doc.readthedocs-hosted.com .. _ACCESS: https://allocations.access-ci.org -Downloading -*********** - The **glotzerlab-software** image and the singularity cache are large, store them in your scratch directory:: @@ -24,9 +21,6 @@ Download the image with support for Delta:: $ singularity pull software.sif docker://glotzerlab/software:delta -Using -***** - Use the following commands in your job scripts or interactively to execute software inside the container: diff --git a/doc/clusters/expanse.rst b/doc/clusters/expanse.rst index ed2d32ef..a20c85c0 100644 --- a/doc/clusters/expanse.rst +++ b/doc/clusters/expanse.rst @@ -1,5 +1,5 @@ Expanse (SDSC) ---------------- +************** Expanse_ is an HPC cluster at SDSC with GPU and CPU nodes. Apply for resources on Expanse through the ACCESS_ program. @@ -7,9 +7,6 @@ the ACCESS_ program. .. _Expanse: https://www.sdsc.edu/support/user_guides/expanse.html .. _ACCESS: https://allocations.access-ci.org/ -Downloading -*********** - For unknown reasons, ``singularity pull`` generates corrupt images when run on Expanse. You need to use another Linux system to pull the ``expanse`` or ``expanse-gpu`` image, then copy that image to Expanse. @@ -19,19 +16,13 @@ to Expanse. The corrupt image causes ``/usr/.so: file too short`` error messages when you attempt to use software in the container. -CPU -+++ - Download the image with support for Expanse's CPU nodes:: $ singularity pull software.sif docker://glotzerlab/software:expanse Then copy ``software.sif`` to ``/expanse/lustre/scratch/$USER/temp_project/`` on Expanse. -GPU -+++ - -Download the image with support for Expanse's GPU nodes:: +Or: download the image with support for Expanse's GPU nodes:: $ singularity pull software.sif docker://glotzerlab/software:expanse-gpu @@ -44,9 +35,6 @@ Then copy ``software.sif`` to ``/expanse/lustre/scratch/$USER/temp_project/`` on * ``expanse`` on CPU nodes. * ``expanse-gpu`` on GPU nodes. -Using -***** - Use the following commands in your job scripts or interactively to execute software inside the container: diff --git a/doc/clusters/frontier.rst b/doc/clusters/frontier.rst index c628c3fa..f9a9561c 100644 --- a/doc/clusters/frontier.rst +++ b/doc/clusters/frontier.rst @@ -1,5 +1,5 @@ Frontier (OLCF) ---------------- +*************** `frontier `_ is a system at ORNL. diff --git a/doc/clusters/greatlakes.rst b/doc/clusters/greatlakes.rst index 6fbc68a0..34775748 100644 --- a/doc/clusters/greatlakes.rst +++ b/doc/clusters/greatlakes.rst @@ -1,11 +1,8 @@ Great Lakes (UMich) -------------------- +******************* `Great Lakes `_ is the University of Michigan campus cluster. -Downloading -*********** - The **glotzerlab-software** image and the singularity cache are large, store them in your scratch directory:: @@ -21,9 +18,6 @@ Download the image with support for Great Lakes:: $ module load singularity $ singularity pull software.sif docker://glotzerlab/software:greatlakes -Using -***** - Use the following commands in your job scripts or interactively to execute software inside the container: .. note:: diff --git a/doc/community.rst b/doc/community.rst index 0a32d15b..506d9828 100644 --- a/doc/community.rst +++ b/doc/community.rst @@ -1,15 +1,9 @@ User community -============== - -Discussion board ----------------- +-------------- Use the `glotzerlab-software discussion board `_ to post questions, ask for support, and discuss potential new features. -Issue tracker -------------- - File bug reports and feature requests on `glotzerlab-software's issue tracker `_. diff --git a/doc/conda-build.rst b/doc/conda-build.rst new file mode 100644 index 00000000..ad623654 --- /dev/null +++ b/doc/conda-build.rst @@ -0,0 +1,33 @@ +Building packages +----------------- + +If you are not a Glotzer Lab member, or would like to build these packages on a new resource: + +1. Install ``conda-build`` into your conda environment:: + + mamba install conda-build + +2. Obtain the **glotzerlab-software** source code:: + + git clone https://github.com/glotzerlab/software + +3. Change to the ``conda`` directory:: + + cd software/conda + +4. On Great Lakes, ANVIL, or Delta: Modify the ``output-folder`` in + ``conda-build-.sh`` and the account in ``job-build-all-.sh``. + Then submit ``job-build-all-.sh`` to build the packages:: + + sbatch job-build-all-.sh + +5. On a different HPC resource, use ``conda-build-greatlakes.sh`` as a starting point for your own + build script. Modify the ``module load`` line, the cluster name (and device, if needed) + ``--variants``, and the ``--output-folder``. Run the build script to build the packages:: + + ./conda-build-.sh hoomd mpi4py + +.. important:: + + The ``output-folder`` is the directory where ``conda build`` will write the packages. Set the + channel path ``file://path/to/conda/channel`` in ``.condarc`` to the ``output-folder``. diff --git a/doc/conda-install.rst b/doc/conda-install.rst new file mode 100644 index 00000000..6af7d00f --- /dev/null +++ b/doc/conda-install.rst @@ -0,0 +1,92 @@ +Installing packages +------------------- + +1. `Install miniforge`_ or your preferred conda environment. + +2. `Configure condarc`_ to use only the local HPC resource channel and **conda-forge**. + +3. `Install packages`_:: + + mamba install hoomd mpi4py + +4. `Load prerequisite modules`_. For example:: + + module load openmpi cuda + +5. `Execute Python scripts`_. For example:: + + srun -n 8 python3 script.py + +.. _Install miniforge: + +.. rubric:: Install miniforge. + +Follow the `miniforge`_ installation instructions for your platform. +This document assumes that you installed to the default path ``$HOME/miniforge3``. + +.. _miniforge: https://github.com/conda-forge/miniforge + + +.. _Configure condarc: + +.. rubric:: Configure condarc. + +Replace ``$HOME/miniforge3/.condarc`` with: + +.. code:: + + channel_priority: strict + channels: + - file://path/to/conda/channel + - conda-forge + + disallow: + - openmpi + - mpich + - cuda-cudart-dev + +.. tip:: + + Glotzer Lab members should use the following channel paths: + + * UMich Great Lakes: ``file://nfs/turbo/glotzer/software/conda`` + * Purdue ANVIL: ``file://anvil/projects/x-dmr140129/software/conda`` + * NCSA Delta: ``file://projects/bbgw/software/conda`` + +.. seealso:: + + See :doc:`conda-build` if you are not a Glotzer Lab member. + +.. note:: + + The ``disallow`` section prevents you from accidentally installing *MPI* and *GPU* packages + from conda-forge. + +.. _Install packages: + +.. rubric:: Install packages + +You should also install any non-MPI and non-GPU packages you need. + + +.. _Load prerequisite modules: + +.. rubric:: Load prerequisite modules + +Load the specific module versions that match those used to build the package. + +.. tip:: + + Glotzer Lab members should load the following modules: + + * UMich Great Lakes: ``module load gcc/10.3.0 openmpi/4.1.6 cuda/12.3.0`` + * Purdue ANVIL: ``module load gcc/11.2.0 openmpi/4.1.6`` + * NCSA Delta: ``module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0`` + + +.. _Execute Python scripts: + +.. rubric:: Execute Python scripts + +Use your HPC resource scheduler to execute scripts on the appropriate compute nodes. Use ``srun``, +``mpirun``, or the appropriate launcher for MPI execution. diff --git a/doc/conda.rst b/doc/conda.rst new file mode 100644 index 00000000..2b916e87 --- /dev/null +++ b/doc/conda.rst @@ -0,0 +1,23 @@ +Conda packages +============== + +**glotzerlab-software** provides conda packages built with cluster-specific *MPI* and *CUDA* +libraries. Use it to add a fully *MPI-* and *GPU-enabled* build of **HOOMD-blue** (and related +software) to your conda environment on a HPC resource. + +**glotzerlab-software** provides the following packages: + +* hoomd +* mpi4py + +.. danger:: + + Do not install any MPI or GPU enabled packages from the **conda-forge** channel. The generic + *MPI* and *CUDA* libraries provided by **conda-forge** will take precedence and prevent the + cluster-specific libraries from operating correctly. + +.. toctree:: + :maxdepth: 1 + + conda-install + conda-build diff --git a/doc/container.rst b/doc/container.rst new file mode 100644 index 00000000..b3ab1c36 --- /dev/null +++ b/doc/container.rst @@ -0,0 +1,85 @@ +Container images +================ + +**glotzerlab-software** is a set of container images that provide software developed by the `Glotzer +group `_ at the `University of Michigan +`_ along with related software commonly used in simulation and data analysis +workflows. An *image* bundles these tools into a single file which can be copied, moved, shared with +others, and published to provide a completely reproducible workflow. A *host* system provides the +resources to execute software inside containers. `Singularity `_ +provides a high performance solution, supporting NVIDIA GPUs and MPI parallel execution. Singularity +is available on most HPC resources. You can install it on your linux system if you have root access. + +To use a container, download the **glotzerlab/software** image:: + + $ singularity pull software.sif docker://glotzerlab/software + +.. note:: + + On HPC clusters, download the cluster specific image. See :doc:`cluster` for details. + +**singularity exec** executes software from inside the container. For example, run a Python script +with:: + + $ singularity exec software.sif python3 script.py + +Add the ``--nv`` option to **enable NVIDIA GPUs** inside the container:: + + $ singularity exec --nv software.sif nvidia-smi + +-----------------------------------------------------------------------------+ + | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | + |-------------------------------+----------------------+----------------------+ + | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | + | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | + | | | MIG M. | + |===============================+======================+======================| + | 0 Quadro RTX 5000 On | 00000000:02:00.0 Off | Off | + | 33% 29C P8 16W / 230W | 1MiB / 16124MiB | 0% Default | + | | | N/A | + +-------------------------------+----------------------+----------------------+ + +**singularity shell** launches an interactive shell:: + + $ singularity shell --nv software.sif + Singularity: Invoking an interactive shell within container... + + Singularity> python3 + Python 3.6.9 (default, Oct 8 2020, 12:12:24) + [GCC 8.4.0] on linux + Type "help", "copyright", "credits" or "license" for more information. + >>> import hoomd + >>> import freud + >>> import signac + >>> gpu = hoomd.device.GPU() + >>> print(gpu.devices) + ['[0] Quadro RTX 5000 48 SM_7.5 @ 1.82 GHz, 16124 MiB DRAM'] + >>> + +**glotzerlab-software** updates weekly with the latest versions of included +software. Download the latest image to update:: + + $ singularity pull software.sif docker://glotzerlab/software + +Environment variables set in the image identify the build:: + + $ singularity exec software.sif bash -c "set" | grep GLOTZERLAB + GLOTZERLAB_SOFTWARE_CONFIGURATION=nompi + GLOTZERLAB_SOFTWARE_GIT_BRANCH=trunk + GLOTZERLAB_SOFTWARE_GIT_SHA=f2debd75280c98ed0a4e46e3e2d381e953b80b6f + GLOTZERLAB_SOFTWARE_TAG=2021.03.24 + +.. seealso:: + + See the `Singularity documentation `_ for more information on + ``singularity`` commands. + +.. toctree:: + :maxdepth: 1 + + software + cluster + files + profiling + test + docker + community diff --git a/doc/docker.rst b/doc/docker.rst index 69c462d8..d4bdee1e 100644 --- a/doc/docker.rst +++ b/doc/docker.rst @@ -1,5 +1,5 @@ Docker -====== +------ **glotzerlab-software** is available on the `Docker Hub `_ for use on docker based systems (for example: diff --git a/doc/files.rst b/doc/files.rst index 20aa610b..0ca8d22d 100644 --- a/doc/files.rst +++ b/doc/files.rst @@ -1,8 +1,8 @@ Accessing files -=============== +---------------- The container's filesystem ---------------------------- +************************** A container has its own filesystem baked into the image. The container's software is installed in it's filesystem. Software executing inside the container will see different files than on the host. @@ -22,7 +22,7 @@ This starts at the root of the filesystem:: With Singularity, the container's filesystem is **read-only**. Bind mounting -------------- +************* Specific directories may be *bind mounted* from the host into the container so it can access the contents directly. On most systems, Singularity is configured to bind mount your home directory by diff --git a/doc/index.rst b/doc/index.rst index 9ff10cd2..98475324 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,93 +1,22 @@ Glotzerlab software images ++++++++++++++++++++++++++ -**glotzerlab-software** is a set of container images that provide software developed by the `Glotzer -group `_ at the `University of Michigan -`_ along with related software commonly used in simulation and data analysis -workflows. An *image* bundles these tools into a single file which can be copied, moved, shared with -others, and published to provide a completely reproducible workflow. A *host* system provides the -resources to execute software inside containers. `Singularity `_ -provides a high performance solution, supporting NVIDIA GPUs and MPI parallel execution. Singularity -is available on most HPC resources. You can install it on your linux system if you have root access. - -Quick start -=========== - -First, download the stable **glotzerlab/software** image:: - - $ singularity pull software.sif docker://glotzerlab/software +**glotzerlab-software** deploys software developed by the `Glotzer group +`_ at the `University of Michigan `_ to +HPC resources via conda packages and container images. Use **glotzerlab-software** to install a MPI +and GPU enabled build of HOOMD-blue on a *HPC resource*. Use the standard **conda-forge** provided +HOOMD-blue (`mamba install hoomd` with no special configuration) for serial execution on individual +workstations. .. note:: - On HPC clusters, download the cluster specific image. See :doc:`cluster` for details. - -**singularity exec** executes software from inside the container. For example, run a Python script -with:: - - $ singularity exec software.sif python3 script.py - -Add the ``--nv`` option to **enable NVIDIA GPUs** inside the container:: - - $ singularity exec --nv software.sif nvidia-smi - +-----------------------------------------------------------------------------+ - | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | - |-------------------------------+----------------------+----------------------+ - | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | - | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | - | | | MIG M. | - |===============================+======================+======================| - | 0 Quadro RTX 5000 On | 00000000:02:00.0 Off | Off | - | 33% 29C P8 16W / 230W | 1MiB / 16124MiB | 0% Default | - | | | N/A | - +-------------------------------+----------------------+----------------------+ - -**singularity shell** launches an interactive shell:: - - $ singularity shell --nv software.sif - Singularity: Invoking an interactive shell within container... - - Singularity> python3 - Python 3.6.9 (default, Oct 8 2020, 12:12:24) - [GCC 8.4.0] on linux - Type "help", "copyright", "credits" or "license" for more information. - >>> import hoomd - >>> import freud - >>> import signac - >>> gpu = hoomd.device.GPU() - >>> print(gpu.devices) - ['[0] Quadro RTX 5000 48 SM_7.5 @ 1.82 GHz, 16124 MiB DRAM'] - >>> - -**glotzerlab-software** updates weekly with the latest versions of included -software. Download the latest image to update:: - - $ singularity pull software.sif docker://glotzerlab/software - -Environment variables set in the image identify the build:: - - $ singularity exec software.sif bash -c "set" | grep GLOTZERLAB - GLOTZERLAB_SOFTWARE_CONFIGURATION=nompi - GLOTZERLAB_SOFTWARE_GIT_BRANCH=trunk - GLOTZERLAB_SOFTWARE_GIT_SHA=f2debd75280c98ed0a4e46e3e2d381e953b80b6f - GLOTZERLAB_SOFTWARE_TAG=2021.03.24 - -.. seealso:: - - See the `Singularity documentation `_ for more information on - ``singularity`` commands. - -Documentation -_____________ + Use conda packages for new installations. The container images are deprecated and will no longer + be maintained after 2024-06-01. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - software - cluster - files - profiling - test - docker - community + conda + container changes license diff --git a/doc/profiling.rst b/doc/profiling.rst index e791a02c..ed488944 100644 --- a/doc/profiling.rst +++ b/doc/profiling.rst @@ -1,12 +1,12 @@ Profiling -========= +--------- You can run profiling tools to determine what operations take time while running executables within the container. Some profiling tools must be executed on the host as they use tight integrations with the Linux kernel. GPU -___ +*** Use ``nvprof`` with the option ``--profile-child-processes``:: @@ -22,7 +22,7 @@ On platforms where you need to launch HOOMD with MPI, call ``nvprof`` after ``mp You may need to add the ``nvprof`` option ``--openacc-profiling off`` on some systems. CPU ---- +*** Build HOOMD from source to profile it on the CPU. The CPU profiler `perf` does not interoperate with Singularity containers. diff --git a/doc/software.rst b/doc/software.rst index bd627531..1e05cc8d 100644 --- a/doc/software.rst +++ b/doc/software.rst @@ -1,5 +1,5 @@ Installed software -================== +------------------ **glotzerlab-software** includes the following installed inside the container. diff --git a/doc/test.rst b/doc/test.rst index a530f1cc..873c39b7 100644 --- a/doc/test.rst +++ b/doc/test.rst @@ -1,5 +1,5 @@ Tests -============ +----- **glotzerlab-software** provides a set of test jobs to determine the installed software functions and that MPI-enabled software inter-operates with the host system. diff --git a/glotzerlab-software.code-workspace b/glotzerlab-software.code-workspace deleted file mode 100644 index 362d7c25..00000000 --- a/glotzerlab-software.code-workspace +++ /dev/null @@ -1,7 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ] -} \ No newline at end of file From bae73a7533947c7a5d48124738547652bacd7f6f Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Fri, 23 Feb 2024 15:05:52 -0500 Subject: [PATCH 10/24] Misc updates. --- CHANGELOG.rst | 11 +++++++++++ doc/index.rst | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d2c10b7e..906c25bf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,17 @@ Find the complete archive of releases on `Docker Hub`_. 2024 ---- +Next release +++++++++++++ + +*Added* + +* Conda package distribution for UMich Great Lakes, Purdue ANVIL, and NCSA Delta. + +*Deprecated* + +* Docker images + 2024-02-02 ++++++++++ diff --git a/doc/index.rst b/doc/index.rst index 98475324..73a84b99 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -5,7 +5,7 @@ Glotzerlab software images `_ at the `University of Michigan `_ to HPC resources via conda packages and container images. Use **glotzerlab-software** to install a MPI and GPU enabled build of HOOMD-blue on a *HPC resource*. Use the standard **conda-forge** provided -HOOMD-blue (`mamba install hoomd` with no special configuration) for serial execution on individual +HOOMD-blue (``mamba install hoomd`` with no special configuration) for serial execution on individual workstations. .. note:: From 2f8068c27f5631a9e2a13c840dc11e3ba616dc24 Mon Sep 17 00:00:00 2001 From: janbridley Date: Fri, 23 Feb 2024 15:55:52 -0500 Subject: [PATCH 11/24] Fixed salloc call in conda-build-greatlakes.sh --- conda/conda-build-greatlakes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/conda-build-greatlakes.sh b/conda/conda-build-greatlakes.sh index 0ed96f18..8dafe4f2 100755 --- a/conda/conda-build-greatlakes.sh +++ b/conda/conda-build-greatlakes.sh @@ -1,7 +1,7 @@ #!/bin/bash # Please use a compute node to build packages. -# salloc -Asglotzer9 --nodes=1 --ntasks-per-node=1 --cpus-per-task=36 --partition standard -t 8:00:00 --mem=144G +# salloc --account=sglotzer9 --nodes=1 --ntasks-per-node=1 --cpus-per-task=36 --partition standard -t 8:00:00 --mem=144G # Load modules used to build packages with native MPI and CUDA support. module reset From 1dde7e450335ea35a604c9115c6d4bb3d1955167 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Sat, 24 Feb 2024 11:10:11 -0500 Subject: [PATCH 12/24] Reduce code duplication in conda build scripts. --- conda/build.sh | 15 +++++++++++++++ conda/conda-build-anvil.sh | 22 ---------------------- conda/conda-build-delta.sh | 25 ------------------------- conda/conda-build-greatlakes.sh | 25 ------------------------- conda/job-build-all-anvil.sh | 9 ++++++++- conda/job-build-all-delta.sh | 11 ++++++++++- conda/job-build-all-greatlakes.sh | 10 +++++++++- 7 files changed, 42 insertions(+), 75 deletions(-) create mode 100755 conda/build.sh delete mode 100755 conda/conda-build-anvil.sh delete mode 100755 conda/conda-build-delta.sh delete mode 100755 conda/conda-build-greatlakes.sh diff --git a/conda/build.sh b/conda/build.sh new file mode 100755 index 00000000..fa272602 --- /dev/null +++ b/conda/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Build package format 2 +export CONDARC=.condarc + +# Build on a node local SSD. +export CONDA_BLD_PATH=`mktemp --directory conda-build-${USER}.XXXXXXXX` + +function cleanup { + rm -rf "$CONDA_BLD_PATH" +} + +trap cleanup EXIT + +conda mambabuild "$@" diff --git a/conda/conda-build-anvil.sh b/conda/conda-build-anvil.sh deleted file mode 100755 index 8230ba45..00000000 --- a/conda/conda-build-anvil.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Load modules used to build packages with native MPI support (no CUDA). -module reset -module load gcc/11.2.0 openmpi/4.1.6 - -# Build package format 2 -export CONDARC=.condarc - -# Build on a node local SSD. -export CONDA_BLD_PATH=`mktemp --directory --tmpdir=/tmp conda-build-${USER}.XXXXXXXX` - -function cleanup { - rm -rf "$CONDA_BLD_PATH" -} - -trap cleanup EXIT - -conda build "$@" \ - --skip-existing \ - --variants "{'cluster': ['anvil'], 'device': ['cpu']}" \ - --output-folder /anvil/projects/x-dmr140129/software/conda diff --git a/conda/conda-build-delta.sh b/conda/conda-build-delta.sh deleted file mode 100755 index 483348ee..00000000 --- a/conda/conda-build-delta.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Please use a compute node to build packages. -# salloc --account=bbgw-delta-gpu --partition=gpuA40x4 --nodes=1 --tasks=1 --tasks-per-node=1 --cpus-per-task=16 --mem=48g --gpus=1 --time 4:00:00 - -# Load modules used to build packages with native MPI and CUDA support. -module reset -module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0 - -# Build package format 2 -export CONDARC=.condarc - -# Build on a node local SSD. -export CONDA_BLD_PATH=`mktemp --directory --tmpdir=/tmp conda-build-${USER}.XXXXXXXX` - -function cleanup { - rm -rf "$CONDA_BLD_PATH" -} - -trap cleanup EXIT - -conda build "$@" \ - --skip-existing \ - --variants "{'cluster': ['delta'], 'device': ['gpu']}" \ - --output-folder /projects/bbgw/software/conda diff --git a/conda/conda-build-greatlakes.sh b/conda/conda-build-greatlakes.sh deleted file mode 100755 index 8dafe4f2..00000000 --- a/conda/conda-build-greatlakes.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Please use a compute node to build packages. -# salloc --account=sglotzer9 --nodes=1 --ntasks-per-node=1 --cpus-per-task=36 --partition standard -t 8:00:00 --mem=144G - -# Load modules used to build packages with native MPI and CUDA support. -module reset -module load gcc/10.3.0 openmpi/4.1.6 cuda/12.3.0 - -# Build package format 2 -export CONDARC=.condarc - -# Build on a node local SSD. -export CONDA_BLD_PATH=`mktemp --directory --tmpdir=/tmpssd conda-build-${USER}.XXXXXXXX` - -function cleanup { - rm -rf "$CONDA_BLD_PATH" -} - -trap cleanup EXIT - -conda build "$@" \ - --skip-existing \ - --variants "{'cluster': ['greatlakes'], 'device': ['gpu']}" \ - --output-folder /nfs/turbo/glotzer/software/conda diff --git a/conda/job-build-all-anvil.sh b/conda/job-build-all-anvil.sh index f74508da..85f6bb2d 100644 --- a/conda/job-build-all-anvil.sh +++ b/conda/job-build-all-anvil.sh @@ -7,5 +7,12 @@ #SBATCH --cpus-per-task=32 #SBATCH --time=8:00:00 -./conda-build-anvil.sh hoomd mpi4py +# Load modules used to build packages with native MPI support (no CUDA). +module reset +module load gcc/11.2.0 openmpi/4.1.6 + +./build.sh "$@" \ + --skip-existing \ + --variants "{'cluster': ['anvil'], 'device': ['cpu']}" \ + --output-folder /anvil/projects/x-dmr140129/software/conda diff --git a/conda/job-build-all-delta.sh b/conda/job-build-all-delta.sh index f54e801f..3d447479 100644 --- a/conda/job-build-all-delta.sh +++ b/conda/job-build-all-delta.sh @@ -11,4 +11,13 @@ #SBATCH --time=8:00:00 # TODO: rewrite to use CPU account when it is renewed -./conda-build-delta.sh hoomd mpi4py + +# Load modules used to build packages with native MPI and CUDA support. +module reset +module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0 + +./build.sh "$@" \ + --skip-existing \ + --variants "{'cluster': ['delta'], 'device': ['gpu']}" \ + --output-folder /projects/bbgw/software/conda + diff --git a/conda/job-build-all-greatlakes.sh b/conda/job-build-all-greatlakes.sh index 15e5c38e..4ce6eb80 100644 --- a/conda/job-build-all-greatlakes.sh +++ b/conda/job-build-all-greatlakes.sh @@ -9,5 +9,13 @@ #SBATCH --mem=144g #SBATCH --time=8:00:00 -./conda-build-greatlakes.sh hoomd mpi4py +# Load modules used to build packages with native MPI and CUDA support. +module reset +module load gcc/10.3.0 openmpi/4.1.6 cuda/12.3.0 +export TMPDIR=/tmpssd + +./build.sh "$@" \ + --skip-existing \ + --variants "{'cluster': ['greatlakes'], 'device': ['gpu']}" \ + --output-folder /nfs/turbo/glotzer/software/conda From c2b6ee899fa7b6e8f7b8ecaa3e891cf6be8c3224 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Sat, 24 Feb 2024 11:24:14 -0500 Subject: [PATCH 13/24] Rearrange some files. --- conda/{job-build-all-anvil.sh => anvil.sh} | 4 +++- conda/{job-build-all-delta.sh => delta.sh} | 2 ++ conda/{job-build-all-greatlakes.sh => greatlakes.sh} | 0 doc/conda-build.rst | 4 ++-- 4 files changed, 7 insertions(+), 3 deletions(-) rename conda/{job-build-all-anvil.sh => anvil.sh} (86%) rename conda/{job-build-all-delta.sh => delta.sh} (96%) rename conda/{job-build-all-greatlakes.sh => greatlakes.sh} (100%) diff --git a/conda/job-build-all-anvil.sh b/conda/anvil.sh similarity index 86% rename from conda/job-build-all-anvil.sh rename to conda/anvil.sh index 85f6bb2d..42e454ae 100644 --- a/conda/job-build-all-anvil.sh +++ b/conda/anvil.sh @@ -11,8 +11,10 @@ module reset module load gcc/11.2.0 openmpi/4.1.6 +export TMPDIR=/tmp + ./build.sh "$@" \ --skip-existing \ --variants "{'cluster': ['anvil'], 'device': ['cpu']}" \ - --output-folder /anvil/projects/x-dmr140129/software/conda + --output-folder $PROJECT/software/conda diff --git a/conda/job-build-all-delta.sh b/conda/delta.sh similarity index 96% rename from conda/job-build-all-delta.sh rename to conda/delta.sh index 3d447479..f3d14ae9 100644 --- a/conda/job-build-all-delta.sh +++ b/conda/delta.sh @@ -16,6 +16,8 @@ module reset module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0 +export TMPDIR=/tmp + ./build.sh "$@" \ --skip-existing \ --variants "{'cluster': ['delta'], 'device': ['gpu']}" \ diff --git a/conda/job-build-all-greatlakes.sh b/conda/greatlakes.sh similarity index 100% rename from conda/job-build-all-greatlakes.sh rename to conda/greatlakes.sh diff --git a/doc/conda-build.rst b/doc/conda-build.rst index ad623654..f852754d 100644 --- a/doc/conda-build.rst +++ b/doc/conda-build.rst @@ -3,9 +3,9 @@ Building packages If you are not a Glotzer Lab member, or would like to build these packages on a new resource: -1. Install ``conda-build`` into your conda environment:: +1. Install ``conda-build`` and ``boa`` into your conda environment:: - mamba install conda-build + mamba install conda-build boa 2. Obtain the **glotzerlab-software** source code:: From fe1e253fcc1c023c335550f5d2fcb7cef9f965c4 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Sat, 24 Feb 2024 11:57:55 -0500 Subject: [PATCH 14/24] Fix tmpdir handling. --- conda/anvil.sh | 2 -- conda/build.sh | 2 +- conda/delta.sh | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/conda/anvil.sh b/conda/anvil.sh index 42e454ae..d2d2d188 100644 --- a/conda/anvil.sh +++ b/conda/anvil.sh @@ -11,8 +11,6 @@ module reset module load gcc/11.2.0 openmpi/4.1.6 -export TMPDIR=/tmp - ./build.sh "$@" \ --skip-existing \ --variants "{'cluster': ['anvil'], 'device': ['cpu']}" \ diff --git a/conda/build.sh b/conda/build.sh index fa272602..ff07e7c1 100755 --- a/conda/build.sh +++ b/conda/build.sh @@ -4,7 +4,7 @@ export CONDARC=.condarc # Build on a node local SSD. -export CONDA_BLD_PATH=`mktemp --directory conda-build-${USER}.XXXXXXXX` +export CONDA_BLD_PATH=`mktemp --directory conda-build-${USER}.XXXXXXXX --tmpdir` function cleanup { rm -rf "$CONDA_BLD_PATH" diff --git a/conda/delta.sh b/conda/delta.sh index f3d14ae9..3d447479 100644 --- a/conda/delta.sh +++ b/conda/delta.sh @@ -16,8 +16,6 @@ module reset module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0 -export TMPDIR=/tmp - ./build.sh "$@" \ --skip-existing \ --variants "{'cluster': ['delta'], 'device': ['gpu']}" \ From eca99f9f37a152eed14d15c6408e725cfed05659 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 10:25:22 -0500 Subject: [PATCH 15/24] Revise docs. --- doc/build.rst | 32 +++++++++++++++++++++++++ doc/conda-build.rst | 33 -------------------------- doc/conda.rst | 9 +++---- doc/glotzer.rst | 33 ++++++++++++++++++++++++++ doc/{conda-install.rst => install.rst} | 28 ++++------------------ 5 files changed, 75 insertions(+), 60 deletions(-) create mode 100644 doc/build.rst delete mode 100644 doc/conda-build.rst create mode 100644 doc/glotzer.rst rename doc/{conda-install.rst => install.rst} (64%) diff --git a/doc/build.rst b/doc/build.rst new file mode 100644 index 00000000..29ddecc6 --- /dev/null +++ b/doc/build.rst @@ -0,0 +1,32 @@ +Building packages +----------------- + +If you are not a Glotzer Lab member or would like to build these packages on a new resource: + +1. Install ``conda-build`` and ``boa`` into your conda environment:: + + mamba install conda-build boa + +2. Obtain the **glotzerlab-software** source code:: + + git clone https://github.com/glotzerlab/software + +3. Change to the ``conda`` directory:: + + cd software/conda + +4. Load any modules needed to provide compilers, MPI, and CUDA (optional). For example:: + + module load gcc openmpi cuda + +5. Build the packages:: + + ./build.sh hoomd mpi4py \ + --skip-existing \ + --variants "{'cluster': [''], 'device': ['gpu'], 'gpu_platform': ['CUDA']}" \ + --output-folder /path/to/conda/channel + +.. important:: + + The ``output-folder`` is the directory where ``conda build`` will write the packages. Set the + channel path ``file://path/to/conda/channel`` in ``.condarc`` to match the ``output-folder``. diff --git a/doc/conda-build.rst b/doc/conda-build.rst deleted file mode 100644 index f852754d..00000000 --- a/doc/conda-build.rst +++ /dev/null @@ -1,33 +0,0 @@ -Building packages ------------------ - -If you are not a Glotzer Lab member, or would like to build these packages on a new resource: - -1. Install ``conda-build`` and ``boa`` into your conda environment:: - - mamba install conda-build boa - -2. Obtain the **glotzerlab-software** source code:: - - git clone https://github.com/glotzerlab/software - -3. Change to the ``conda`` directory:: - - cd software/conda - -4. On Great Lakes, ANVIL, or Delta: Modify the ``output-folder`` in - ``conda-build-.sh`` and the account in ``job-build-all-.sh``. - Then submit ``job-build-all-.sh`` to build the packages:: - - sbatch job-build-all-.sh - -5. On a different HPC resource, use ``conda-build-greatlakes.sh`` as a starting point for your own - build script. Modify the ``module load`` line, the cluster name (and device, if needed) - ``--variants``, and the ``--output-folder``. Run the build script to build the packages:: - - ./conda-build-.sh hoomd mpi4py - -.. important:: - - The ``output-folder`` is the directory where ``conda build`` will write the packages. Set the - channel path ``file://path/to/conda/channel`` in ``.condarc`` to the ``output-folder``. diff --git a/doc/conda.rst b/doc/conda.rst index 2b916e87..32a95df9 100644 --- a/doc/conda.rst +++ b/doc/conda.rst @@ -2,8 +2,8 @@ Conda packages ============== **glotzerlab-software** provides conda packages built with cluster-specific *MPI* and *CUDA* -libraries. Use it to add a fully *MPI-* and *GPU-enabled* build of **HOOMD-blue** (and related -software) to your conda environment on a HPC resource. +libraries. Use it to add *MPI-* and *GPU-enabled* builds of **HOOMD-blue** and related +software to your conda environment on a HPC resource. **glotzerlab-software** provides the following packages: @@ -19,5 +19,6 @@ software) to your conda environment on a HPC resource. .. toctree:: :maxdepth: 1 - conda-install - conda-build + glotzer + build + install diff --git a/doc/glotzer.rst b/doc/glotzer.rst new file mode 100644 index 00000000..e7ba577c --- /dev/null +++ b/doc/glotzer.rst @@ -0,0 +1,33 @@ +Glotzer lab members +------------------- + +Members of the Glotzer lab: Follow the instructions in :doc:`install` to install the latest stable +release using the following channel paths and module versions: + +.. list-table:: + :header-rows: 1 + + * - HPC resource + - Channel location + - Module versions + * - UMich Great Lakes: + - ``file://nfs/turbo/glotzer/software/conda`` + - .. code:: + + module load gcc/10.3.0 openmpi/4.1.6 cuda/12.3.0 + * - Purdue ANVIL: + - ``file://anvil/projects/x-dmr140129/software/conda`` + - .. code:: + + module load gcc/11.2.0 openmpi/4.1.6 + * - NCSA Delta: + - ``file://projects/bbgw/software/conda`` + - .. code:: + + module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0 + * - OLCF Frontier: + - ``file://ccs/proj/mat110/software/frontier/conda`` + - .. code:: + + module load PrgEnv-gnu rocm/5.4.3 + module unload darshan-runtime diff --git a/doc/conda-install.rst b/doc/install.rst similarity index 64% rename from doc/conda-install.rst rename to doc/install.rst index 6af7d00f..bd4bfe11 100644 --- a/doc/conda-install.rst +++ b/doc/install.rst @@ -1,9 +1,9 @@ Installing packages ------------------- -1. `Install miniforge`_ or your preferred conda environment. +1. `Install miniforge`_ (or your preferred conda environment provider). -2. `Configure condarc`_ to use only the local HPC resource channel and **conda-forge**. +2. `Configure condarc`_ to use the local HPC resource channel and **conda-forge**. 3. `Install packages`_:: @@ -45,18 +45,6 @@ Replace ``$HOME/miniforge3/.condarc`` with: - mpich - cuda-cudart-dev -.. tip:: - - Glotzer Lab members should use the following channel paths: - - * UMich Great Lakes: ``file://nfs/turbo/glotzer/software/conda`` - * Purdue ANVIL: ``file://anvil/projects/x-dmr140129/software/conda`` - * NCSA Delta: ``file://projects/bbgw/software/conda`` - -.. seealso:: - - See :doc:`conda-build` if you are not a Glotzer Lab member. - .. note:: The ``disallow`` section prevents you from accidentally installing *MPI* and *GPU* packages @@ -66,7 +54,9 @@ Replace ``$HOME/miniforge3/.condarc`` with: .. rubric:: Install packages -You should also install any non-MPI and non-GPU packages you need. +You should also install any non-MPI and non-GPU packages you need. For example:: + + mamba install freud gsd .. _Load prerequisite modules: @@ -75,14 +65,6 @@ You should also install any non-MPI and non-GPU packages you need. Load the specific module versions that match those used to build the package. -.. tip:: - - Glotzer Lab members should load the following modules: - - * UMich Great Lakes: ``module load gcc/10.3.0 openmpi/4.1.6 cuda/12.3.0`` - * Purdue ANVIL: ``module load gcc/11.2.0 openmpi/4.1.6`` - * NCSA Delta: ``module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0`` - .. _Execute Python scripts: From 77effa298b18ec46bddf8004cd7b7c0b18b986d1 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 10:27:23 -0500 Subject: [PATCH 16/24] Add frontier support. --- conda/anvil.sh | 8 ++++++-- conda/delta.sh | 6 ++++-- conda/greatlakes.sh | 8 ++++++-- conda/hoomd/build_hoomd.sh | 1 + conda/hoomd/meta.yaml | 29 ++++++++++------------------- conda/mpi4py/meta.yaml | 6 ++++++ 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/conda/anvil.sh b/conda/anvil.sh index d2d2d188..fa597224 100644 --- a/conda/anvil.sh +++ b/conda/anvil.sh @@ -7,12 +7,16 @@ #SBATCH --cpus-per-task=32 #SBATCH --time=8:00:00 +export OUTPUT_FOLDER=$PROJECT/software/conda + # Load modules used to build packages with native MPI support (no CUDA). module reset module load gcc/11.2.0 openmpi/4.1.6 ./build.sh "$@" \ --skip-existing \ - --variants "{'cluster': ['anvil'], 'device': ['cpu']}" \ - --output-folder $PROJECT/software/conda + --variants "{'cluster': ['anvil'], 'device': ['cpu'], 'gpu_platform': ['none']}" \ + --output-folder $OUTPUT_FOLDER + +chmod g-w $OUTPUT_FOLDER -R diff --git a/conda/delta.sh b/conda/delta.sh index 3d447479..692fe0d0 100644 --- a/conda/delta.sh +++ b/conda/delta.sh @@ -11,6 +11,7 @@ #SBATCH --time=8:00:00 # TODO: rewrite to use CPU account when it is renewed +export OUTPUT_FOLDER=/projects/bbgw/software/conda # Load modules used to build packages with native MPI and CUDA support. module reset @@ -18,6 +19,7 @@ module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0 ./build.sh "$@" \ --skip-existing \ - --variants "{'cluster': ['delta'], 'device': ['gpu']}" \ - --output-folder /projects/bbgw/software/conda + --variants "{'cluster': ['delta'], 'device': ['gpu'], 'gpu_platform': ['CUDA']}" \ + --output-folder $OUTPUT_FOLDER +chmod g-w $OUTPUT_FOLDER -R diff --git a/conda/greatlakes.sh b/conda/greatlakes.sh index 4ce6eb80..6beb99b6 100644 --- a/conda/greatlakes.sh +++ b/conda/greatlakes.sh @@ -9,6 +9,8 @@ #SBATCH --mem=144g #SBATCH --time=8:00:00 +export OUTPUT_FOLDER=/nfs/turbo/glotzer/software/conda + # Load modules used to build packages with native MPI and CUDA support. module reset module load gcc/10.3.0 openmpi/4.1.6 cuda/12.3.0 @@ -17,5 +19,7 @@ export TMPDIR=/tmpssd ./build.sh "$@" \ --skip-existing \ - --variants "{'cluster': ['greatlakes'], 'device': ['gpu']}" \ - --output-folder /nfs/turbo/glotzer/software/conda + --variants "{'cluster': ['greatlakes'], 'device': ['gpu'], 'gpu_platform': ['CUDA']}" \ + --output-folder $OUTPUT_FOLDER + +chmod g-w $OUTPUT_FOLDER -R diff --git a/conda/hoomd/build_hoomd.sh b/conda/hoomd/build_hoomd.sh index b2a6c511..8bde180e 100644 --- a/conda/hoomd/build_hoomd.sh +++ b/conda/hoomd/build_hoomd.sh @@ -6,6 +6,7 @@ cmake -B build \ ${CMAKE_ARGS} \ -DENABLE_MPI=on \ -DENABLE_GPU=${ENABLE_GPU:-off} \ + -DHOOMD_GPU_PLATFORM=$2 \ -DBUILD_TESTING=off \ -DENABLE_TBB=off \ -DENABLE_LLVM=off \ diff --git a/conda/hoomd/meta.yaml b/conda/hoomd/meta.yaml index 9f9daa6b..2eda8815 100644 --- a/conda/hoomd/meta.yaml +++ b/conda/hoomd/meta.yaml @@ -14,16 +14,18 @@ source: sha256: {{ sha256 }} build: + script_env: + - LD_LIBRARY_PATH + - MPICC + - CC + - CXX + - CMAKE_PREFIX_PATH + - HIP_PLATFORM + - HIP_COMPILER + - HCC_AMDGPU_TARGET number: {{ build }} string: {{ cluster }}_{{ device }}_py{{ py }}h{{ PKG_HASH }}_{{ build }} - script: bash ${RECIPE_DIR}/build_hoomd.sh {{ device }} - -# rpaths: -# - lib/ -# - lib/python3.12/site-packages/{{ name }} # [py==312] -# - lib/python3.11/site-packages/{{ name }} # [py==311] -# - lib/python3.10/site-packages/{{ name }} # [py==310] -# - lib/python3.9/site-packages/{{ name }} # [py==39] + script: bash ${RECIPE_DIR}/build_hoomd.sh {{ device }} {{ gpu_platform }} requirements: build: @@ -43,14 +45,3 @@ requirements: - python - numpy -# Tests hang for unknown reasons on great lakes. -# test: -# requires: -# - pytest -# - rowan -# imports: -# - hoomd -# - hoomd.md -# - hoomd.hpmc -# commands: -# - pytest --pyargs hoomd -v diff --git a/conda/mpi4py/meta.yaml b/conda/mpi4py/meta.yaml index d080a396..c3615d84 100644 --- a/conda/mpi4py/meta.yaml +++ b/conda/mpi4py/meta.yaml @@ -15,6 +15,12 @@ source: build: number: {{ build }} string: {{ cluster }}_py{{ py }}h{{ PKG_HASH }}_{{ build }} + script_env: + - LD_LIBRARY_PATH + - MPICC + - CC + - CXX + - CMAKE_PREFIX_PATH script: - {{ PYTHON }} conf/cythonize.py # TODO: remove for mpi4py 4.0.0 - {{ PYTHON }} -m pip install --no-deps . -vv From 87dabee84a75908bdc6bee8a3bf6e4e684dbae51 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 10:37:12 -0500 Subject: [PATCH 17/24] Add missing frontier script. --- conda/frontier.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 conda/frontier.sh diff --git a/conda/frontier.sh b/conda/frontier.sh new file mode 100755 index 00000000..1b2e6011 --- /dev/null +++ b/conda/frontier.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Load modules used to build packages with native MPI and HIP support +module purge +module load PrgEnv-gnu rocm/5.4.3 +module unload darshan-runtime + +export OUTPUT_FOLDER=/ccs/proj/mat110/software/frontier/conda + +# The cray-mpich module does not provide this, it is needed to build mpi4py from source. +export MPICC=$CRAY_MPICH_DIR/bin/mpicc +export CC=$GCC_PATH/bin/gcc +export CXX=$GCC_PATH/bin/g++ +export HCC_AMDGPU_TARGET=gfx90a + +./build.sh "$@" \ + --skip-existing \ + --variants "{'cluster': ['frontier'], 'device': ['gpu'], 'gpu_platform': ['HIP']}" \ + --output-folder $OUTPUT_FOLDER + +chmod g-w $OUTPUT_FOLDER -R + From bc533d4c81707921512b39b57f45415e18d15d9e Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 10:51:48 -0500 Subject: [PATCH 18/24] Update docs. --- doc/glotzer.rst | 6 ++++++ doc/install.rst | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/doc/glotzer.rst b/doc/glotzer.rst index e7ba577c..fd38460e 100644 --- a/doc/glotzer.rst +++ b/doc/glotzer.rst @@ -31,3 +31,9 @@ release using the following channel paths and module versions: module load PrgEnv-gnu rocm/5.4.3 module unload darshan-runtime + + * - OLCF Andes: + - ``file://ccs/proj/mat110/software/andes/conda`` + - .. code:: + + module load gcc/10.3.0 openmpi/4.1.2 diff --git a/doc/install.rst b/doc/install.rst index bd4bfe11..17b89221 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -72,3 +72,8 @@ Load the specific module versions that match those used to build the package. Use your HPC resource scheduler to execute scripts on the appropriate compute nodes. Use ``srun``, ``mpirun``, or the appropriate launcher for MPI execution. + +.. tip:: + + After initial installation and setup is complete, you can update packages to their latest + versions with ``mamba update --all``. From 481d0c3d404030d40c96250e8e6a19a595cb4265 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 11:28:50 -0500 Subject: [PATCH 19/24] Use the correct gcc on anvil. --- conda/anvil.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda/anvil.sh b/conda/anvil.sh index fa597224..3a1c0081 100644 --- a/conda/anvil.sh +++ b/conda/anvil.sh @@ -13,6 +13,9 @@ export OUTPUT_FOLDER=$PROJECT/software/conda module reset module load gcc/11.2.0 openmpi/4.1.6 +export CC=$GCC_HOME/bin/gcc +export CXX=$GCC_HOME/bin/g++ + ./build.sh "$@" \ --skip-existing \ --variants "{'cluster': ['anvil'], 'device': ['cpu'], 'gpu_platform': ['none']}" \ From 96697afcc0f4a3bcb3d095f4a07a52db9e9fe118 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 13:23:06 -0600 Subject: [PATCH 20/24] Make packages group readable on delta. --- conda/delta.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/delta.sh b/conda/delta.sh index 692fe0d0..c3189b11 100644 --- a/conda/delta.sh +++ b/conda/delta.sh @@ -23,3 +23,4 @@ module load gcc/11.4.0 openmpi/4.1.6 cuda/12.3.0 --output-folder $OUTPUT_FOLDER chmod g-w $OUTPUT_FOLDER -R +chmod g+rX $OUTPUT_FOLDER -R From ca389e84f30f40ada23276b4f700244a106acefc Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 14:33:19 -0500 Subject: [PATCH 21/24] Add andes build script. Limit ninja to fewer tasks on andes. --- conda/andes.sh | 18 ++++++++++++++++++ conda/frontier.sh | 3 +++ conda/hoomd/build_hoomd.sh | 4 ++-- conda/hoomd/meta.yaml | 1 + 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100755 conda/andes.sh diff --git a/conda/andes.sh b/conda/andes.sh new file mode 100755 index 00000000..a2958f38 --- /dev/null +++ b/conda/andes.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Load modules used to build packages with native MPI support +module reset +module load gcc/10.3.0 openmpi/4.1.2 + +export OUTPUT_FOLDER=/ccs/proj/mat110/software/andes/conda + +# Control ninja parallelism in hoomd builds +export SLURM_CPUS_PER_TASK=8 + +./build.sh "$@" \ + --skip-existing \ + --variants "{'cluster': ['andes'], 'device': ['cpu'], 'gpu_platform': ['none']}" \ + --output-folder $OUTPUT_FOLDER + +chmod g-w $OUTPUT_FOLDER -R + diff --git a/conda/frontier.sh b/conda/frontier.sh index 1b2e6011..bed00a7a 100755 --- a/conda/frontier.sh +++ b/conda/frontier.sh @@ -7,6 +7,9 @@ module unload darshan-runtime export OUTPUT_FOLDER=/ccs/proj/mat110/software/frontier/conda +# Control ninja parallelism in hoomd builds +export SLURM_CPUS_PER_TASK=16 + # The cray-mpich module does not provide this, it is needed to build mpi4py from source. export MPICC=$CRAY_MPICH_DIR/bin/mpicc export CC=$GCC_PATH/bin/gcc diff --git a/conda/hoomd/build_hoomd.sh b/conda/hoomd/build_hoomd.sh index 8bde180e..de6a2185 100644 --- a/conda/hoomd/build_hoomd.sh +++ b/conda/hoomd/build_hoomd.sh @@ -14,5 +14,5 @@ cmake -B build \ -DCMAKE_CXX_FLAGS=-march=native -DCMAKE_C_FLAGS=-march=native \ -GNinja -ninja -C build -ninja -C build install +ninja -C build -j $SLURM_CPUS_PER_TASK +ninja -C build install -j $SLURM_CPUS_PER_TASK diff --git a/conda/hoomd/meta.yaml b/conda/hoomd/meta.yaml index 2eda8815..0c525cf6 100644 --- a/conda/hoomd/meta.yaml +++ b/conda/hoomd/meta.yaml @@ -23,6 +23,7 @@ build: - HIP_PLATFORM - HIP_COMPILER - HCC_AMDGPU_TARGET + - SLURM_CPUS_PER_TASK number: {{ build }} string: {{ cluster }}_{{ device }}_py{{ py }}h{{ PKG_HASH }}_{{ build }} script: bash ${RECIPE_DIR}/build_hoomd.sh {{ device }} {{ gpu_platform }} From b137158e6b1d874ee63a9fa36b3919fe1bed0203 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 14:38:37 -0500 Subject: [PATCH 22/24] Document tar file update on Frontier. --- doc/glotzer.rst | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/doc/glotzer.rst b/doc/glotzer.rst index fd38460e..ec31c5b8 100644 --- a/doc/glotzer.rst +++ b/doc/glotzer.rst @@ -37,3 +37,58 @@ release using the following channel paths and module versions: - .. code:: module load gcc/10.3.0 openmpi/4.1.2 + +.. rubric:: Frontier + +Individual users should install **miniforge** environments in their home directory (the default): +``$HOME/miniforge3`` on Frontier. + +Importing Python packages from this environment will be *very* slow with large node count jobs. +To improve performance, generate a **tar** file from the environment and store it on Orion. + +.. important:: + + Repeat this step after you install or update packages with ``mamba``. + +.. code-block:: + + $ tar --directory $HOME/miniforge3 -cf ${MEMBERWORK}/mat110/miniforge3.tar . + +.. tip:: + + Collaborative projects should aim to install and maintain a single copy of the software. + This should be installed to the project home:: + + bash Miniforge3-Linux-x86_64.sh -b -p /ccs/proj/{your-project}/software/frontier/{subproject-name} + chmod g+rwX /ccs/proj/{your-project}/software/frontier/{subproject-name} -R + + This allows environment changes to propagate between users, and cuts down on storage usage in + the project home directory. + + Collaborative projects can also work off of a single cached ``miniforge3.tar``:: + + $ tar --directory $HOME/miniforge3 -cf ${PROJWORK}/{your-project}/software/{subproject-name}/software.tar . + +Use the following lines in your job scripts (or interactively with ``salloc``) to load the cache +into NVME and execute software from there:: + + #SBATCH -C nvme + + module load PrgEnv-gnu rocm/5.4.3 + module unload darshan-runtime + + export MINIFORGE_ROOT=/mnt/bb/${USER}/miniforge + srun --ntasks-per-node 1 mkdir ${MINIFORGE_ROOT} + srun --ntasks-per-node 1 tar --directory ${MINIFORGE_ROOT} -xpf \ + ${MEMBERWORK}/mat110/miniforge3.tar + # ${PROJWORK}/{your-project}/software/{subproject-name}/miniforge3.tar # For use with shared projects. + + export PATH=${MINIFORGE_ROOT}/bin:$PATH + + srun {srun options} command arguments + +.. note:: + + The above script has been tested on environments with all packages installed into the base. + You may need to set additional environment variables or source activation scripts to activate + conda environments within this directory. From 22a5d2014733d6712f0f9ee3abe87d4d78ccb088 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Mon, 26 Feb 2024 18:43:49 -0500 Subject: [PATCH 23/24] Adjust frontier docs. --- doc/glotzer.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/glotzer.rst b/doc/glotzer.rst index ec31c5b8..159b2884 100644 --- a/doc/glotzer.rst +++ b/doc/glotzer.rst @@ -56,8 +56,9 @@ To improve performance, generate a **tar** file from the environment and store i .. tip:: - Collaborative projects should aim to install and maintain a single copy of the software. - This should be installed to the project home:: + Collaborative projects can maintain a single copy of the software in the shared project + directory. The miniforge installer accepts the installation prefix on the command line with + ``-p`` (adjust these steps as needed for a non-miniforge installer):: bash Miniforge3-Linux-x86_64.sh -b -p /ccs/proj/{your-project}/software/frontier/{subproject-name} chmod g+rwX /ccs/proj/{your-project}/software/frontier/{subproject-name} -R @@ -65,9 +66,9 @@ To improve performance, generate a **tar** file from the environment and store i This allows environment changes to propagate between users, and cuts down on storage usage in the project home directory. - Collaborative projects can also work off of a single cached ``miniforge3.tar``:: + Collaborative projects can also utilize a single cached ``miniforge3.tar``:: - $ tar --directory $HOME/miniforge3 -cf ${PROJWORK}/{your-project}/software/{subproject-name}/software.tar . + $ tar --directory $HOME/miniforge3 -cf ${PROJWORK}/{your-project}/software/{subproject-name}/miniforge3.tar . Use the following lines in your job scripts (or interactively with ``salloc``) to load the cache into NVME and execute software from there:: From 6e5a0cb6d1194fe692f816b3f0dc04f5ad701213 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Tue, 27 Feb 2024 09:49:04 -0500 Subject: [PATCH 24/24] Make conda docs reference generic package managers and environment names. --- doc/build.rst | 19 ++++++----- doc/conda.rst | 27 ++++++++++++---- doc/glotzer.rst | 51 +++++++++++++----------------- doc/install.rst | 84 ++++++++++++++----------------------------------- 4 files changed, 78 insertions(+), 103 deletions(-) diff --git a/doc/build.rst b/doc/build.rst index 29ddecc6..31e12a6e 100644 --- a/doc/build.rst +++ b/doc/build.rst @@ -1,11 +1,16 @@ Building packages ----------------- -If you are not a Glotzer Lab member or would like to build these packages on a new resource: +If you are not a Glotzer Lab member or you would like to build these packages on a new resource: -1. Install ``conda-build`` and ``boa`` into your conda environment:: +.. note:: - mamba install conda-build boa + Replace ``{{ package-manager }}`` with the name of your preferred conda-compatible package manager + executable. + +1. Install ``conda-build`` and ``boa`` into your environment:: + + {{ package-manager }} install conda-build boa 2. Obtain the **glotzerlab-software** source code:: @@ -15,7 +20,7 @@ If you are not a Glotzer Lab member or would like to build these packages on a n cd software/conda -4. Load any modules needed to provide compilers, MPI, and CUDA (optional). For example:: +4. Load any modules needed to provide compilers, *MPI*, and *CUDA* (optional). For example:: module load gcc openmpi cuda @@ -23,10 +28,10 @@ If you are not a Glotzer Lab member or would like to build these packages on a n ./build.sh hoomd mpi4py \ --skip-existing \ - --variants "{'cluster': [''], 'device': ['gpu'], 'gpu_platform': ['CUDA']}" \ - --output-folder /path/to/conda/channel + --variants "{'cluster': ['{{ cluster-name }}'], 'device': ['gpu'], 'gpu_platform': ['CUDA']}" \ + --output-folder {{ channel-path }} .. important:: The ``output-folder`` is the directory where ``conda build`` will write the packages. Set the - channel path ``file://path/to/conda/channel`` in ``.condarc`` to match the ``output-folder``. + channel path ``file:/{{ channel-path }}`` in ``.condarc`` to match (see :doc:`install`). diff --git a/doc/conda.rst b/doc/conda.rst index 32a95df9..7e92849f 100644 --- a/doc/conda.rst +++ b/doc/conda.rst @@ -1,18 +1,29 @@ Conda packages ============== -**glotzerlab-software** provides conda packages built with cluster-specific *MPI* and *CUDA* -libraries. Use it to add *MPI-* and *GPU-enabled* builds of **HOOMD-blue** and related -software to your conda environment on a HPC resource. - -**glotzerlab-software** provides the following packages: +**glotzerlab-software** provides `conda`_ formatted packages built with cluster-specific *MPI* and +*CUDA* libraries. Use it to add *MPI-* and *GPU-enabled* builds of the following software packages +to your conda-compatible environments on HPC resources: * hoomd * mpi4py +.. important:: + + These packages are built for ABI compatibility with packages on the **conda-forge** channel. + Ensure that you have **no** packages installed from the **default** channel before proceeding. + +.. tip:: + + `miniforge`_ provides a conda-compatible environment pre-configured to install packages only + from **conda-forge**. When you install `miniforge`_ with default options, replace: + + * ``{{ package-manager }}`` with ``mamba`` + * ``{{ environment-path }}`` with ``$HOME/miniforge3`` + .. danger:: - Do not install any MPI or GPU enabled packages from the **conda-forge** channel. The generic + Do not install any *MPI* or *GPU* enabled packages from the **conda-forge** channel. The generic *MPI* and *CUDA* libraries provided by **conda-forge** will take precedence and prevent the cluster-specific libraries from operating correctly. @@ -22,3 +33,7 @@ software to your conda environment on a HPC resource. glotzer build install + +.. _miniforge: https://github.com/conda-forge/miniforge + +.. _conda: https://docs.conda.io diff --git a/doc/glotzer.rst b/doc/glotzer.rst index 159b2884..e23ee246 100644 --- a/doc/glotzer.rst +++ b/doc/glotzer.rst @@ -1,8 +1,9 @@ Glotzer lab members ------------------- -Members of the Glotzer lab: Follow the instructions in :doc:`install` to install the latest stable -release using the following channel paths and module versions: +Members of the Glotzer lab can install precompiled packages. Follow the instructions in +:doc:`install` to install the latest stable release using the following channel paths and module +versions: .. list-table:: :header-rows: 1 @@ -40,56 +41,48 @@ release using the following channel paths and module versions: .. rubric:: Frontier -Individual users should install **miniforge** environments in their home directory (the default): -``$HOME/miniforge3`` on Frontier. - -Importing Python packages from this environment will be *very* slow with large node count jobs. -To improve performance, generate a **tar** file from the environment and store it on Orion. +Individual users should install conda-compatible environments in their **home directory** on +Frontier. Importing Python packages from this environment will be *very* slow with large node count +jobs. To improve performance, generate a **tar** file from the environment and store it on Orion. .. important:: - Repeat this step after you install or update packages with ``mamba``. + Repeat this step after you install or update packages with ``{{ package-manager }}``. .. code-block:: - $ tar --directory $HOME/miniforge3 -cf ${MEMBERWORK}/mat110/miniforge3.tar . + $ tar --directory {{ environment-path }} -cf ${MEMBERWORK}/mat110/conda-env.tar . .. tip:: - Collaborative projects can maintain a single copy of the software in the shared project - directory. The miniforge installer accepts the installation prefix on the command line with - ``-p`` (adjust these steps as needed for a non-miniforge installer):: - - bash Miniforge3-Linux-x86_64.sh -b -p /ccs/proj/{your-project}/software/frontier/{subproject-name} - chmod g+rwX /ccs/proj/{your-project}/software/frontier/{subproject-name} -R - - This allows environment changes to propagate between users, and cuts down on storage usage in - the project home directory. + Collaborative projects may maintain a single copy of the software in the shared project + directory: ``/ccs/proj/mat110/software/frontier/{{ subproject-name }}``. - Collaborative projects can also utilize a single cached ``miniforge3.tar``:: + Collaborative projects may also utilize a single cached ``conda-env.tar``:: - $ tar --directory $HOME/miniforge3 -cf ${PROJWORK}/{your-project}/software/{subproject-name}/miniforge3.tar . + $ tar --directory /ccs/proj/mat110/software/frontier/{{ subproject-name }} \ + -cf ${PROJWORK}/mat110/software/{{ subproject-name} }/conda-env.tar . -Use the following lines in your job scripts (or interactively with ``salloc``) to load the cache -into NVME and execute software from there:: +Use the following lines in your job scripts (or interactively with ``salloc``) to load the +environment into NVME and execute software from there:: #SBATCH -C nvme module load PrgEnv-gnu rocm/5.4.3 module unload darshan-runtime - export MINIFORGE_ROOT=/mnt/bb/${USER}/miniforge - srun --ntasks-per-node 1 mkdir ${MINIFORGE_ROOT} - srun --ntasks-per-node 1 tar --directory ${MINIFORGE_ROOT} -xpf \ - ${MEMBERWORK}/mat110/miniforge3.tar - # ${PROJWORK}/{your-project}/software/{subproject-name}/miniforge3.tar # For use with shared projects. + export CONDA_ENV_ROOT=/mnt/bb/${USER}/conda-env + srun --ntasks-per-node 1 mkdir ${CONDA_ENV_ROOT} + srun --ntasks-per-node 1 tar --directory ${CONDA_ENV_ROOT} -xpf \ + ${MEMBERWORK}/mat110/conda-env.tar + # ${PROJWORK}/mat110/software/{{ subproject-name }}/conda-env.tar # For use with shared projects. - export PATH=${MINIFORGE_ROOT}/bin:$PATH + export PATH=${CONDA_ENV_ROOT}/bin:$PATH srun {srun options} command arguments .. note:: - The above script has been tested on environments with all packages installed into the base. + The above script has been tested on environments with all packages installed into *base*. You may need to set additional environment variables or source activation scripts to activate conda environments within this directory. diff --git a/doc/install.rst b/doc/install.rst index 17b89221..c946ef8b 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -1,79 +1,41 @@ Installing packages ------------------- -1. `Install miniforge`_ (or your preferred conda environment provider). +1. **Configure condarc** to use the local HPC resource channel and **conda-forge**. -2. `Configure condarc`_ to use the local HPC resource channel and **conda-forge**. + Replace ``{{ environment-path }}/.condarc`` with: -3. `Install packages`_:: + .. code:: - mamba install hoomd mpi4py + channel_priority: strict + channels: + - file:/{{ channel-path }} + - conda-forge + disallow: + - openmpi + - mpich + - cuda-cudart-dev -4. `Load prerequisite modules`_. For example:: + and keep any custom configuration options you would like. - module load openmpi cuda + .. note:: + The ``disallow`` section prevents you from accidentally installing *MPI* and *GPU* packages + from conda-forge. -5. `Execute Python scripts`_. For example:: +2. **Install the packages**:: - srun -n 8 python3 script.py - -.. _Install miniforge: - -.. rubric:: Install miniforge. - -Follow the `miniforge`_ installation instructions for your platform. -This document assumes that you installed to the default path ``$HOME/miniforge3``. - -.. _miniforge: https://github.com/conda-forge/miniforge - - -.. _Configure condarc: - -.. rubric:: Configure condarc. - -Replace ``$HOME/miniforge3/.condarc`` with: - -.. code:: - - channel_priority: strict - channels: - - file://path/to/conda/channel - - conda-forge - - disallow: - - openmpi - - mpich - - cuda-cudart-dev + {{ package-manager }} install hoomd mpi4py -.. note:: +3. **Load the module versions** that match those used to build the package. For example:: - The ``disallow`` section prevents you from accidentally installing *MPI* and *GPU* packages - from conda-forge. + module load openmpi/X.Y.Z cuda/I.J.K -.. _Install packages: +4. **Execute Python scripts**. Use your HPC resource scheduler to request compute nodes and use the + resources's MPI launcher. For example:: -.. rubric:: Install packages - -You should also install any non-MPI and non-GPU packages you need. For example:: - - mamba install freud gsd - - -.. _Load prerequisite modules: - -.. rubric:: Load prerequisite modules - -Load the specific module versions that match those used to build the package. - - -.. _Execute Python scripts: - -.. rubric:: Execute Python scripts - -Use your HPC resource scheduler to execute scripts on the appropriate compute nodes. Use ``srun``, -``mpirun``, or the appropriate launcher for MPI execution. + srun -n 8 python3 script.py .. tip:: After initial installation and setup is complete, you can update packages to their latest - versions with ``mamba update --all``. + versions with ``{{package-manager}} update --all``.