CI: update to CentOS Stream 9 and start integrating universal target (algorand#6080)

Author: onetechnical

docker/build/cicd.centos.Dockerfile

@@ -1,18 +1,13 @@
 ARG ARCH="amd64"
 
-FROM quay.io/centos/centos:stream8
+FROM quay.io/centos/centos:stream9
 ARG GOLANG_VERSION
 ARG ARCH="amd64"
-RUN dnf update rpm -y && \
-    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+RUN dnf install -y epel-release epel-next-release && dnf config-manager --set-enabled crb && \
     dnf update -y && \
     dnf install -y autoconf wget awscli git gnupg2 nfs-utils python3-devel expect jq \
     libtool gcc-c++ libstdc++-devel rpmdevtools createrepo rpm-sign bzip2 which \
-    libffi-devel openssl-devel
-RUN dnf install -y epel-release && \
-    dnf update && \
-    dnf -y --enablerepo=powertools install libstdc++-static && \
-    dnf -y install make
+    libffi-devel openssl-devel libstdc++-static
 RUN echo "${BOLD}Downloading and installing binaries...${RESET}" && \
     curl -Of https://shellcheck.storage.googleapis.com/shellcheck-v0.7.0.linux.x86_64.tar.xz && \
     tar -C /usr/local/bin/ -xf shellcheck-v0.7.0.linux.x86_64.tar.xz --no-anchored 'shellcheck' --strip=1

docker/build/cicd.centos8.Dockerfile renamed to docker/build/cicd.centos9.Dockerfile

@@ -45,8 +45,8 @@ agents:
     workDir: $HOME/projects/go-algorand
 
   - name: rpm
-    dockerFilePath: docker/build/cicd.centos8.Dockerfile
-    image: algorand/go-algorand-ci-linux-centos8
+    dockerFilePath: docker/build/cicd.centos9.Dockerfile
+    image: algorand/go-algorand-ci-linux-centos9
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`

package-deploy.yaml

@@ -16,8 +16,8 @@ agents:
     workDir: $HOME/projects/go-algorand
 
   - name: rpm
-    dockerFilePath: docker/build/cicd.centos.Dockerfile
-    image: algorand/mule-linux-centos
+    dockerFilePath: docker/build/cicd.centos9.Dockerfile
+    image: algorand/mule-linux-centos9
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`

package-test.yaml

@@ -11,8 +11,8 @@ agents:
     workDir: $HOME/projects/go-algorand
 
   - name: rpm
-    dockerFilePath: docker/build/cicd.centos.Dockerfile
-    image: algorand/go-algorand-ci-linux-centos
+    dockerFilePath: docker/build/cicd.centos9.Dockerfile
+    image: algorand/go-algorand-ci-linux-centos9
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`

package.yaml

@@ -50,7 +50,7 @@ This section briefly describes the expected outcomes of the current build pipeli
 
 1. build
 
-    1. Build (compile) the binaries in a Centos 7 & 8 docker container that will then be used by both `deb` and `rpm` packaging.
+    1. Build (compile) the binaries in a Centos 9 docker container that will then be used by both `deb` and `rpm` packaging.
 
     1. Docker containers will package `deb` and `rpm` artifacts inside of Ubuntu 20.04 and Centos 7 & 8, respectively.
 
@@ -69,9 +69,9 @@ This section briefly describes the expected outcomes of the current build pipeli
     - The signatures are correct.
     - The packages are built from the correct branch and channel and are the correct version.  This done by running `algod -v`.
         + This is done for the following docker containers:
-            - centos:7
-            - quay.io/centos/centos:stream8
-            - fedora:38
+            - quay.io/centos/centos:stream9
+            - fedora:39
+            - fedora:40
             - ubuntu:20.04
             - ubuntu:22.04
 

scripts/release/README.md

@@ -30,8 +30,8 @@ else
     echo ${BUILD_NUMBER} > "${REPO_ROOT}"/buildnumber.dat
 fi
 
-# Run RPM build in Centos 7 & 8 Docker container
-sg docker "docker build -t algocentosbuild - < $HOME/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile"
+# Run RPM build in Centos 9 Docker container
+sg docker "docker build -t algocentosbuild - < $HOME/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos9.Dockerfile"
 sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME},dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/build/rpm/build.sh"
 
 echo

scripts/release/build/stage/build/task.sh

@@ -1,7 +1,8 @@
-FROM quay.io/centos/centos:stream8
+FROM quay.io/centos/centos:stream9
 
 WORKDIR /root
-RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+RUN dnf install -y epel-release epel-next-release && dnf config-manager --set-enabled crb && \
+    dnf update -y && \
     dnf install -y autoconf awscli curl git gnupg2 nfs-utils python36 expect jq libtool gcc-c++ libstdc++-devel rpmdevtools createrepo rpm-sign bzip2 which && \
     dnf -y --enablerepo=powertools install libstdc++-static
 

scripts/release/common/docker/centos.Dockerfile

@@ -104,8 +104,7 @@ else
 fi
 
 sudo usermod -a -G docker ubuntu
-sg docker "docker pull centos:7"
-sg docker "docker pull quay.io/centos/centos:stream8"
+sg docker "docker pull quay.io/centos/centos:stream9"
 sg docker "docker pull ubuntu:22.04"
 
 cat << EOF >> "${HOME}/.bashrc"

scripts/release/common/docker/centos8.Dockerfile renamed to scripts/release/common/docker/centos9.Dockerfile

@@ -2,9 +2,9 @@
 
 set -exo pipefail
 
-# Ensure the centos8 docker image is built and available
+# Ensure the centos docker image is built and available
 
-DOCKER_IMAGE="algorand/go-algorand-ci-linux-centos8:amd64-$(sha1sum scripts/configure_dev-deps.sh | cut -f1 -d' ')"
+DOCKER_IMAGE="algorand/go-algorand-ci-linux-centos9:amd64-$(sha1sum scripts/configure_dev-deps.sh | cut -f1 -d' ')"
 MATCH=${DOCKER_IMAGE/:*/}
 
 echo "Checking for RPM image"
@@ -13,5 +13,5 @@ if docker images $DOCKER_IMAGE | grep -qs $MATCH > /dev/null 2>&1; then
 else
   echo "RPM image doesn't exist, building"
   docker build --platform=linux/amd64 --build-arg ARCH=amd64 \
-    --build-arg GOLANG_VERSION=$(./scripts/get_golang_version.sh) -t $DOCKER_IMAGE -f docker/build/cicd.centos8.Dockerfile .
+    --build-arg GOLANG_VERSION=$(./scripts/get_golang_version.sh) -t $DOCKER_IMAGE -f docker/build/cicd.centos9.Dockerfile .
 fi

scripts/release/common/setup.sh

@@ -50,7 +50,7 @@ find tmp/node_pkgs -name "*${CHANNEL}*linux*${VERSION}*.tar.gz" | cut -d '/' -f3
             -e "s,@REQUIRED_ALGORAND_PKG@,$REQUIRED_ALGORAND_PACKAGE," \
         > "$TEMPDIR/$ALGORAND_PACKAGE_NAME.spec"
 
-    rpmbuild --buildroot "$HOME/foo" --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS xtrue" --define "LICENSE_FILE ./COPYING" -bb "$TEMPDIR/$ALGORAND_PACKAGE_NAME.spec" --target $ARCH_UNAME
+    rpmbuild --buildroot "$HOME/foo" --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS \"xtrue\"" --define "LICENSE_FILE ./COPYING" -bb "$TEMPDIR/$ALGORAND_PACKAGE_NAME.spec" --target $ARCH_UNAME
 
     cp -p "$RPMTMP"/*/*.rpm "./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"
     echo "${RPMTMP}"

scripts/release/mule/common/ensure_centos8_image.sh renamed to scripts/release/mule/common/ensure_centos9_image.sh

@@ -5,8 +5,8 @@ set -ex
 
 . "${HOME}"/build_env
 
-# Run RPM build in Centos 7 & 8 Docker container
-sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile"
+# Run RPM build in Centos 9 Docker container
+sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos9.Dockerfile"
 
 sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=/run/user/1000/gnupg/S.gpg-agent,dst=/root/S.gpg-agent --mount type=bind,src=${HOME}/prodrepo,dst=/root/prodrepo --mount type=bind,src=${HOME}/keys,dst=/root/keys --mount type=bind,src=${HOME},dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/prod/rpm/snapshot.sh"
 

scripts/release/mule/package/rpm/package.sh

@@ -14,8 +14,8 @@ if [ "$CHANNEL" = beta ]; then
     exit 0
 fi
 
-# Run RPM build in Centos 7 & 8 Docker container
-sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile"
+# Run RPM build in Centos 9 Docker container
+sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos9.Dockerfile"
 
 cat <<EOF>"${HOME}"/dummyrepo/algodummy.repo
 [algodummy]

scripts/release/prod/rpm/run_centos.sh

@@ -8,9 +8,9 @@ set -ex
 . "${HOME}"/build_env
 
 OS_LIST=(
-    centos:7
-    quay.io/centos/centos:stream8
-    fedora:38
+    quay.io/centos/centos:stream9
+    fedora:39
+    fedora:40
     ubuntu:20.04
     ubuntu:22.04
 )

scripts/release/test/rpm/run_centos.sh

@@ -15,23 +15,9 @@ agents:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`
       - ARCH=amd64
       - GOARCH=amd64
-  - name: cicd.centos.amd64
-    dockerFilePath: docker/build/cicd.centos.Dockerfile
-    image: algorand/go-algorand-ci-linux-centos
-    version: scripts/configure_dev-deps.sh
-    arch: amd64
-    env:
-      - TRAVIS_BRANCH=${GIT_BRANCH}
-      - NETWORK=$NETWORK
-      - VERSION=$VERSION
-      - BUILD_NUMBER=$BUILD_NUMBER
-      - GOHOSTARCH=amd64
-    buildArgs:
-      - GOLANG_VERSION=`./scripts/get_golang_version.sh`
-      - ARCH=amd64
-  - name: cicd.centos8.amd64
-    dockerFilePath: docker/build/cicd.centos8.Dockerfile
-    image: algorand/go-algorand-ci-linux-centos8
+  - name: cicd.centos9.amd64
+    dockerFilePath: docker/build/cicd.centos9.Dockerfile
+    image: algorand/go-algorand-ci-linux-centos9
     version: scripts/configure_dev-deps.sh
     arch: amd64
     env:
@@ -106,12 +92,12 @@ tasks:
 
   - task: docker.Make
     name: archive
-    agent: cicd.centos8.amd64
+    agent: cicd.centos9.amd64
     target: archive
 
   - task: docker.Make
     name: rpm.amd64
-    agent: cicd.centos.amd64
+    agent: cicd.centos9.amd64
     target: mule-package-rpm
 
   - task: docker.Make

scripts/release/test/util/test_package.sh

@@ -7,9 +7,9 @@ BLUE_FG=$(tput setaf 4 2>/dev/null)
 END_FG_COLOR=$(tput sgr0 2>/dev/null)
 
 OS_LIST=(
-    centos:7
-    quay.io/centos/centos:stream8
-    fedora:38
+    quay.io/centos/centos:stream9
+    fedora:39
+    fedora:40
     ubuntu:20.04
     ubuntu:22.04
 )