Merge pull request #143 from SENSEI-insitu/develop

SENSEI v5.0.0

.githooks/post-checkout

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "updating submodules ..."
+git submodule update --init --recursive

.githooks/post-merge

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "updating submodules ..."
+git submodule update --init --recursive

.github/ci/configure_adios1.cmake

@@ -1 +1 @@
-set(ENABLE_ADIOS1 ON CACHE BOOL "")
+set(SENSEI_ENABLE_ADIOS1 ON CACHE BOOL "")

.github/ci/configure_adios2.cmake

@@ -1,2 +1,2 @@
-set(ENABLE_ADIOS2 ON CACHE BOOL "")
+set(SENSEI_ENABLE_ADIOS2 ON CACHE BOOL "")
 

.github/ci/configure_ascent.cmake

@@ -1 +1 @@
-set(ENABLE_ASCENT ON CACHE BOOL "")
+set(SENSEI_ENABLE_ASCENT ON CACHE BOOL "")

.github/ci/configure_catalyst.cmake

@@ -1,5 +1,5 @@
-set(ENABLE_CATALYST ON CACHE BOOL "")
+set(SENSEI_ENABLE_CATALYST ON CACHE BOOL "")
 
-if (ENABLE_PYTHON)
-  set(ENABLE_CATALYST_PYTHON ON CACHE BOOL "")
+if (SENSEI_ENABLE_PYTHON)
+  set(SENSEI_ENABLE_CATALYST_PYTHON ON CACHE BOOL "")
 endif ()

.github/ci/configure_common.cmake

@@ -1,5 +1,5 @@
 set(BUILD_TESTING ON CACHE BOOL "")
-set(ENABLE_SENSEI ON CACHE BOOL "")
+set(SENSEI_ENABLE_SENSEI ON CACHE BOOL "")
 set(MPIEXEC_PREFLAGS "--allow-run-as-root" CACHE STRING "")
 set(TEST_NP 2 CACHE STRING "")
 

.github/ci/configure_hdf5.cmake

@@ -1 +1 @@
-set(ENABLE_HDF5 ON CACHE BOOL "")
+set(SENSEI_ENABLE_HDF5 ON CACHE BOOL "")

.github/ci/configure_libsim.cmake

@@ -1 +1 @@
-set(ENABLE_LIBSIM ON CACHE BOOL "")
+set(SENSEI_ENABLE_LIBSIM ON CACHE BOOL "")

.github/ci/configure_options.cmake

@@ -8,10 +8,10 @@ function (configuration_flag variable configuration)
   endif ()
 endfunction ()
 
-if (ENABLE_VTK_IO)
-  configuration_flag(ENABLE_VTK_RENDERING "rendering")
+if (SENSEI_ENABLE_VTK_IO)
+  configuration_flag(SENSEI_ENABLE_VTK_RENDERING "rendering")
 endif ()
 
-if (ENABLE_VTKM)
-  configuration_flag(ENABLE_VTKM_RENDERING "rendering")
+if (SENSEI_ENABLE_VTKM)
+  configuration_flag(SENSEI_ENABLE_VTKM_RENDERING "rendering")
 endif ()

.github/ci/configure_python2.cmake

@@ -1,7 +1,7 @@
-set(ENABLE_PYTHON ON CACHE BOOL "")
+set(SENSEI_ENABLE_PYTHON ON CACHE BOOL "")
 set(SENSEI_PYTHON_VERSION 2 CACHE STRING "")
 
 # ParaView no longer supports python 2
-if (ENABLE_CATALYST)
-  set(ENABLE_CATALYST_PYTHON OFF CACHE BOOL "")
+if (SENSEI_ENABLE_CATALYST)
+  set(SENSEI_ENABLE_CATALYST_PYTHON OFF CACHE BOOL "")
 endif ()

.github/ci/configure_python3.cmake

@@ -9,9 +9,9 @@ set(Python3_EXECUTABLE "${PY_ROOT}/bin/python3" CACHE PATH "")
 set(Python3_FIND_STRATEGY "LOCATION" CACHE STRING "")
 set(Python3_FIND_FRAMEWORK "FIRST" CACHE STRING "")
 
-set(ENABLE_PYTHON ON CACHE BOOL "")
+set(SENSEI_ENABLE_PYTHON ON CACHE BOOL "")
 set(SENSEI_PYTHON_VERSION 3 CACHE STRING "")
 
 if (ENABLE_CATALYST)
-  set(ENABLE_CATALYST_PYTHON ON CACHE BOOL "")
+  set(SENSEI_ENABLE_CATALYST_PYTHON ON CACHE BOOL "")
 endif ()

.github/ci/configure_fedora35_ecp_catalyst.cmake → .github/ci/configure_ubuntu_ecp_catalyst.cmake

@@ -1,8 +1,9 @@
+set(SENSEI_USE_EXTERNAL_pugixml ON CACHE BOOL "")
 include(${CMAKE_CURRENT_LIST_DIR}/configure_adios2.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/configure_ascent.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/configure_catalyst.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/configure_hdf5.cmake)
-include(${CMAKE_CURRENT_LIST_DIR}/configure_python3.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/configure_vtkio.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/configure_python3.cmake)
 
-include(${CMAKE_CURRENT_LIST_DIR}/configure_fedora35.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/configure_common.cmake)

.github/ci/configure_vtkio.cmake

@@ -1,2 +1,2 @@
-set(ENABLE_VTK_IO ON CACHE BOOL "")
-set(ENABLE_VTK_FILTERS ON CACHE BOOL "")
+set(SENSEI_ENABLE_VTK_IO ON CACHE BOOL "")
+set(SENSEI_ENABLE_VTK_FILTERS ON CACHE BOOL "")

.github/ci/configure_vtkm.cmake

@@ -1,3 +1,3 @@
-set(ENABLE_VTKM ON CACHE BOOL "")
-set(ENABLE_VTK_MPI ON CACHE BOOL "")
-set(ENABLE_VTK_ACCELERATORS ON CACHE BOOL "")
+set(SENSEI_ENABLE_VTKM ON CACHE BOOL "")
+set(SENSEI_ENABLE_VTK_MPI ON CACHE BOOL "")
+set(SENSEI_ENABLE_VTK_ACCELERATORS ON CACHE BOOL "")

.github/ci/docker/.gitignore

@@ -0,0 +1,2 @@
+buildcache*
+*-log.txt

.github/ci/docker/build_image.sh

@@ -19,12 +19,30 @@ else
   exit 1
 fi
 
-base_name=$1
+# create image
 tag_date=$(date +%Y%m%d)
 tag=$1-$tag_date
 
-echo "Building container: senseiinsitu/ci:$tag"
+base_container=ghcr.io/sensei-insitu/ci-ecp
+buildcache_container=ghcr.io/sensei-insitu/sensei-buildcache
+
+echo
+echo "Building container: $base_container:$tag"
 echo "  Dockerfile: $wdir/Dockerfile"
-
-docker build -t senseiinsitu/ci:$tag $wdir |& tee $1-build-log.txt
-docker push senseiinsitu/ci:$tag
+
+docker buildx create --name extended_log --use --driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=100000000
+docker buildx build --load -t $base_container:$tag-init-spack $wdir --target init_spack |& tee $1-init-spack-log.txt
+
+BUILDCACHE_VERSION=$(docker run --rm -it $base_container:$tag-init-spack /sensei/bin/tag.sh | tr -d '\r')
+
+IMAGE_EXISTS=$(docker manifest inspect $buildcache_container:$BUILDCACHE_VERSION | grep "layers")
+if [[ -z ${IMAGE_EXISTS} ]]; then
+  echo "Buildcache version $BUILDCACHE_VERSION not found, using empty buildcache."
+  BUILDCACHE_VERSION="empty"
+fi
+
+echo "Buildcache version: $BUILDCACHE_VERSION"
+docker buildx build --load -t $base_container:$tag $wdir --build-arg BUILDCACHE_VERSION=$BUILDCACHE_VERSION --target build_sensei |& tee $1-build-sensei-log.txt
+docker push $base_container:$tag
+
+docker rmi $base_container:$tag-init-spack

.github/ci/docker/cache/Dockerfile

@@ -0,0 +1,4 @@
+FROM alpine
+
+# Install Pre-reqs and Spack
+COPY buildcache /buildcache

.github/ci/docker/cache/README.md

@@ -0,0 +1,36 @@
+# SENSEI Spack Buildcache
+
+This directory contains the spack binary buildcache for SENSEI's dependencies.
+
+Use of buildcaches speeds up the build process for iterating SENSEI CI containers.
+
+## Extracting a buildcache from a working CI image
+
+SENSEI uses CI docker images to perform automated testing. These CI images are built
+using the `build_image.sh` script. Occasionally development needs to be performed on
+the CI image themselves, which can be an arduous process given the need to rebuild
+SENSEI and all of its dependencies in Spack. To speed up this process, SENSEI uses
+buildcaches to speed up the build process.
+
+The `extract_buildcache.sh` script will extract the buildcache from a built SENSEI
+CI docker image and create a new minimal docker image containing the buildcache.
+
+To use the `extract_buildcache.sh` script, specify the source CI image you'd like to
+extract built binaries from. If no image is specified, the script will use the most
+recent `sensei-insitu/ci-ecp` image found locally. The script will then extract the
+buildcache to a local volume, and run a docker built to create a new buildcache image.
+This image is tagged with the operating system, compiler and architecture of the
+source CI image to ensure compatibility.
+
+## How buildcaches are used in SENSEI CI images
+
+SENSEI CI images are built with the `build_image.sh` script. This script builds a CI
+docker image based on a preexisting buildcache docker image. The script will identify
+the proper buildcache image tag for the CI environment, and attempt to use the
+prebuilt binaries if they are available. If the buildcache image is not in the
+container registry, it defaults to using the `empty` tagged image.
+
+`build_image.sh` invokes a docker build that uses the buildcache image as an
+intermediate layer to mount copy the buildcache into the CI image. The buildcache is
+then used speed up the to build the SENSEI CI image. The buildcache is not included
+in the final image.

.github/ci/docker/cache/extract_buildcache.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+cd $(dirname $0)
+
+IMAGE=$1
+if [[ -z $1 ]]; then
+    echo "No image specified. Using latest sensei-insitu/ci-ecp image."
+    IMAGE=$(docker image list | grep sensei-insitu/ci-ecp | awk '{ print $1 ":" $2 }' | head -n 1)
+fi
+echo "Using image: ${IMAGE}"
+echo
+
+tag=$(docker run -it --rm --volume "$(pwd):/sensei/cache" ${IMAGE} /sensei/bin/tag.sh | tr -d '\r')
+buildcache_container=ghcr.io/sensei-insitu/sensei-buildcache
+
+# append the buildcache to the prior version
+IMAGE_EXISTS=$(docker manifest inspect $buildcache_container:$tag | grep "layers")
+if [[ -n ${IMAGE_EXISTS} ]]; then
+    echo "Found existing buildcache version: $tag"
+    echo "Appending to image."
+
+    # clear the buildcache folder
+    docker run -it --rm --volume "$(pwd):/cache" alpine rm -rf /cache/buildcache
+
+    container=$(docker run --detach $buildcache_container:$tag)
+    docker cp $container:/buildcache ./buildcache
+    docker rm -f $container
+else
+    echo "Buildcache version $tag not found, using empty buildcache."
+fi
+
+# generate the buildcache from CI image
+docker run -it --rm --volume "$(pwd):/sensei/cache" ${IMAGE} \
+    bin/launch-env.sh \
+    spack buildcache create --force --allow-root --unsigned /sensei/cache/buildcache
+
+docker build -t $buildcache_container:$tag .
+docker push $buildcache_container:$tag
+
+# cleanup the buildcache directory, sometimes this has root permissions
+docker run -it --rm --volume "$(pwd):/cache" alpine rm -rf /cache/buildcache

.github/ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,26 @@
+ARG BUILDCACHE_VERSION=empty
+FROM ubuntu:20.04 AS init_spack
+
+ENV SPACK_PYTHON=/usr/bin/python3
+ENV SPACK_ROOT=/opt/spack
+ENV SENSEI_ENV=sensei
+
+# Install Pre-reqs and Spack
+COPY install-prereqs.sh /sensei/tmp/
+COPY spack.yaml /sensei/tmp/
+COPY modules.yaml $SPACK_ROOT/etc/spack/
+RUN /sensei/tmp/install-prereqs.sh
+COPY tag.sh /sensei/bin/
+
+FROM ghcr.io/sensei-insitu/sensei-buildcache:$BUILDCACHE_VERSION AS buildcache
+
+# Install SENSEI deps with spack
+FROM init_spack AS build_sensei
+COPY --from=buildcache /buildcache /sensei/buildcache
+COPY install-deps.sh /sensei/tmp/
+RUN /sensei/tmp/install-deps.sh
+
+WORKDIR /sensei/
+COPY configure-env.sh /sensei/bin/
+COPY launch-env.sh /sensei/bin/
+RUN echo ". /sensei/bin/configure-env.sh" >> ~/.bashrc

.github/ci/docker/ubuntu/configure-env.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+. ${SPACK_ROOT}/share/spack/setup-env.sh
+
+SPACK_LMOD_ROOT=$(spack find --format {prefix} lmod)
+. ${SPACK_LMOD_ROOT}/lmod/lmod/init/bash
+
+if [[ -z ${SENSEI_ENV} ]]; then
+  echo "No environment configured."
+  exit 1
+fi
+
+export CMAKE_CONFIGURATION=ubuntu_ecp_catalyst
+export CMAKE_CXX_COMPILER_LAUNCHER=ccache
+export OMPI_ALLOW_RUN_AS_ROOT=1
+export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+spack env activate ${SENSEI_ENV}
+
+module use ${SPACK_ROOT}/share/spack/lmod/linux-ubuntu20.04-x86_64/Core
+module load openmpi ninja swig
+
+. /sensei/loads