Create updated Ubuntu ECP image/CI job

.github/ci/configure_fedora35_ecp_catalyst.cmake → .github/ci/configure_ubuntu_ecp_catalyst.cmake

@@ -1,8 +1,9 @@
+set(SENSEI_USE_EXTERNAL_pugixml ON CACHE BOOL "")
 include(${CMAKE_CURRENT_LIST_DIR}/configure_adios2.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/configure_ascent.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/configure_catalyst.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/configure_hdf5.cmake)
-include(${CMAKE_CURRENT_LIST_DIR}/configure_python3.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/configure_vtkio.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/configure_python3.cmake)
 
-include(${CMAKE_CURRENT_LIST_DIR}/configure_fedora35.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/configure_common.cmake)

.github/ci/docker/.gitignore

@@ -0,0 +1,2 @@
+buildcache*
+*-log.txt

.github/ci/docker/build_image.sh

@@ -19,12 +19,30 @@ else
   exit 1
 fi
 
-base_name=$1
+# create image
 tag_date=$(date +%Y%m%d)
 tag=$1-$tag_date
 
-echo "Building container: senseiinsitu/ci:$tag"
+base_container=ghcr.io/sensei-insitu/ci-ecp
+buildcache_container=ghcr.io/sensei-insitu/sensei-buildcache
+
+echo
+echo "Building container: $base_container:$tag"
 echo "  Dockerfile: $wdir/Dockerfile"
-
-docker build -t senseiinsitu/ci:$tag $wdir |& tee $1-build-log.txt
-docker push senseiinsitu/ci:$tag
+
+docker buildx create --name extended_log --use --driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=100000000
+docker buildx build --load -t $base_container:$tag-init-spack $wdir --target init_spack |& tee $1-init-spack-log.txt
+
+BUILDCACHE_VERSION=$(docker run --rm -it $base_container:$tag-init-spack /sensei/bin/tag.sh | tr -d '\r')
+
+IMAGE_EXISTS=$(docker manifest inspect $buildcache_container:$BUILDCACHE_VERSION | grep "layers")
+if [[ -z ${IMAGE_EXISTS} ]]; then
+  echo "Buildcache version $BUILDCACHE_VERSION not found, using empty buildcache."
+  BUILDCACHE_VERSION="empty"
+fi
+
+echo "Buildcache version: $BUILDCACHE_VERSION"
+docker buildx build --load -t $base_container:$tag $wdir --build-arg BUILDCACHE_VERSION=$BUILDCACHE_VERSION --target build_sensei |& tee $1-build-sensei-log.txt
+docker push $base_container:$tag
+
+docker rmi $base_container:$tag-init-spack

.github/ci/docker/cache/Dockerfile

@@ -0,0 +1,4 @@
+FROM alpine
+
+# Install Pre-reqs and Spack
+COPY buildcache /buildcache

.github/ci/docker/cache/README.md

@@ -0,0 +1,36 @@
+# SENSEI Spack Buildcache
+
+This directory contains the spack binary buildcache for SENSEI's dependencies.
+
+Use of buildcaches speeds up the build process for iterating SENSEI CI containers.
+
+## Extracting a buildcache from a working CI image
+
+SENSEI uses CI docker images to perform automated testing. These CI images are built
+using the `build_image.sh` script. Occasionally development needs to be performed on
+the CI image themselves, which can be an arduous process given the need to rebuild
+SENSEI and all of its dependencies in Spack. To speed up this process, SENSEI uses
+buildcaches to speed up the build process.
+
+The `extract_buildcache.sh` script will extract the buildcache from a built SENSEI
+CI docker image and create a new minimal docker image containing the buildcache.
+
+To use the `extract_buildcache.sh` script, specify the source CI image you'd like to
+extract built binaries from. If no image is specified, the script will use the most
+recent `sensei-insitu/ci-ecp` image found locally. The script will then extract the
+buildcache to a local volume, and run a docker built to create a new buildcache image.
+This image is tagged with the operating system, compiler and architecture of the
+source CI image to ensure compatibility.
+
+## How buildcaches are used in SENSEI CI images
+
+SENSEI CI images are built with the `build_image.sh` script. This script builds a CI
+docker image based on a preexisting buildcache docker image. The script will identify
+the proper buildcache image tag for the CI environment, and attempt to use the
+prebuilt binaries if they are available. If the buildcache image is not in the
+container registry, it defaults to using the `empty` tagged image.
+
+`build_image.sh` invokes a docker build that uses the buildcache image as an
+intermediate layer to mount copy the buildcache into the CI image. The buildcache is
+then used speed up the to build the SENSEI CI image. The buildcache is not included
+in the final image.

.github/ci/docker/cache/extract_buildcache.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+cd $(dirname $0)
+
+IMAGE=$1
+if [[ -z $1 ]]; then
+    echo "No image specified. Using latest sensei-insitu/ci-ecp image."
+    IMAGE=$(docker image list | grep sensei-insitu/ci-ecp | awk '{ print $1 ":" $2 }' | head -n 1)
+fi
+echo "Using image: ${IMAGE}"
+echo
+
+tag=$(docker run -it --rm --volume "$(pwd):/sensei/cache" ${IMAGE} /sensei/bin/tag.sh | tr -d '\r')
+buildcache_container=ghcr.io/sensei-insitu/sensei-buildcache
+
+# append the buildcache to the prior version
+IMAGE_EXISTS=$(docker manifest inspect $buildcache_container:$tag | grep "layers")
+if [[ -n ${IMAGE_EXISTS} ]]; then
+    echo "Found existing buildcache version: $tag"
+    echo "Appending to image."
+
+    # clear the buildcache folder
+    docker run -it --rm --volume "$(pwd):/cache" alpine rm -rf /cache/buildcache
+
+    container=$(docker run --detach $buildcache_container:$tag)
+    docker cp $container:/buildcache ./buildcache
+    docker rm -f $container
+else
+    echo "Buildcache version $tag not found, using empty buildcache."
+fi
+
+# generate the buildcache from CI image
+docker run -it --rm --volume "$(pwd):/sensei/cache" ${IMAGE} \
+    bin/launch-env.sh \
+    spack buildcache create --force --allow-root --unsigned /sensei/cache/buildcache
+
+docker build -t $buildcache_container:$tag .
+docker push $buildcache_container:$tag
+
+# cleanup the buildcache directory, sometimes this has root permissions
+docker run -it --rm --volume "$(pwd):/cache" alpine rm -rf /cache/buildcache

.github/ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,26 @@
+ARG BUILDCACHE_VERSION=empty
+FROM ubuntu:20.04 AS init_spack
+
+ENV SPACK_PYTHON=/usr/bin/python3
+ENV SPACK_ROOT=/opt/spack
+ENV SENSEI_ENV=sensei
+
+# Install Pre-reqs and Spack
+COPY install-prereqs.sh /sensei/tmp/
+COPY spack.yaml /sensei/tmp/
+COPY modules.yaml $SPACK_ROOT/etc/spack/
+RUN /sensei/tmp/install-prereqs.sh
+COPY tag.sh /sensei/bin/
+
+FROM ghcr.io/sensei-insitu/sensei-buildcache:$BUILDCACHE_VERSION AS buildcache
+
+# Install SENSEI deps with spack
+FROM init_spack AS build_sensei
+COPY --from=buildcache /buildcache /sensei/buildcache
+COPY install-deps.sh /sensei/tmp/
+RUN /sensei/tmp/install-deps.sh
+
+WORKDIR /sensei/
+COPY configure-env.sh /sensei/bin/
+COPY launch-env.sh /sensei/bin/
+RUN echo ". /sensei/bin/configure-env.sh" >> ~/.bashrc

.github/ci/docker/ubuntu/configure-env.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+. ${SPACK_ROOT}/share/spack/setup-env.sh
+
+SPACK_LMOD_ROOT=$(spack find --paths lmod | grep ^lmod@ | awk '{ print $2 }')
+. ${SPACK_LMOD_ROOT}/lmod/lmod/init/bash
+
+if [[ -z ${SENSEI_ENV} ]]; then
+  echo "No environment configured."
+  exit 1
+fi
+
+export CMAKE_CONFIGURATION=ubuntu_ecp_catalyst
+export CMAKE_CXX_COMPILER_LAUNCHER=ccache
+export OMPI_ALLOW_RUN_AS_ROOT=1
+export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+spack env activate ${SENSEI_ENV}
+
+module use ${SPACK_ROOT}/share/spack/lmod/linux-ubuntu20.04-x86_64/Core
+module load openmpi ninja swig
+
+. ${SPACK_ROOT}/var/spack/environments/${SENSEI_ENV}/loads

.github/ci/docker/ubuntu/install-deps.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -xe
+
+source ${SPACK_ROOT}/share/spack/setup-env.sh
+spack env activate ${SENSEI_ENV}
+{
+    ls /sensei/buildcache/build_cache && \
+    spack mirror add sensei /sensei/buildcache && \
+    spack buildcache update-index sensei
+} || {
+    echo "Error: Failed to install buildcache."
+    echo "  Skipping buildcache installation..."
+}
+
+# install
+N_THREADS=$(grep -c '^processor' /proc/cpuinfo)
+spack install -v --use-cache --no-check-signature -j ${N_THREADS}
+spack install -v --use-cache --no-check-signature -j ${N_THREADS} sensei
+
+# cleanup
+spack clean -a
+rm -rf /root/.spack /sensei/tmp /sensei/buildcache
+
+# load modules
+spack module lmod refresh -y
+spack env loads -m lmod

.github/ci/docker/ubuntu/install-prereqs.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -xe
+
+export DEBIAN_FRONTEND="noninteractive"
+apt-get update -y
+apt-get install -y file git curl \
+  build-essential software-properties-common \
+  tar xz-utils bzip2 gzip unzip \
+  python3 python3-pip ccache \
+  patch patchelf libtool \
+  vim
+
+add-apt-repository ppa:ubuntu-toolchain-r/test -y
+apt-get update -y
+apt-get install -y gcc-11 g++-11 gfortran-11
+
+# Install spack
+mkdir -p ${SPACK_ROOT}
+cd ${SPACK_ROOT}
+git init
+git remote add origin https://github.com/spack/spack.git
+git fetch origin develop
+git checkout -f develop
+
+rm -rf .git
+
+source ${SPACK_ROOT}/share/spack/setup-env.sh
+spack compiler find --scope site
+
+# Extra packages required by SENSEI test scripts (Not used by spack)
+apt-get install -y bc
+apt-get clean -y
+
+# Setup sensei environment
+spack env create ${SENSEI_ENV}
+cp /sensei/tmp/spack.yaml ${SPACK_ROOT}/var/spack/environments/${SENSEI_ENV}
+spack env activate ${SENSEI_ENV}
+spack concretize -f

.github/ci/docker/ubuntu/launch-env.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+. /sensei/bin/configure-env.sh
+
+$@