diff --git a/.cicd/build.sh b/.cicd/build.sh index ef90eea1f2..7c0fc2234a 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -58,9 +58,10 @@ else # Linux fi . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile" COMMANDS="$PRE_COMMANDS && $COMMANDS" - DOCKER_RUN="docker run $ARGS $(buildkite-intrinsics) --env CMAKE_EXTRAS='$CMAKE_EXTRAS' '$FULL_TAG' bash -c '$COMMANDS'" - echo "$ $DOCKER_RUN" - eval $DOCKER_RUN + DOCKER_RUN_ARGS="$ARGS $(buildkite-intrinsics) --env CMAKE_EXTRAS='$CMAKE_EXTRAS' '$FULL_TAG' bash -c '$COMMANDS'" + echo "$ docker run $DOCKER_RUN_ARGS" + [[ -z "${PROXY_DOCKER_RUN_ARGS:-}" ]] || echo "Appending proxy args: '${PROXY_DOCKER_RUN_ARGS}'" + eval "docker run ${PROXY_DOCKER_RUN_ARGS:-}${DOCKER_RUN_ARGS}" fi if [[ "$BUILDKITE" == 'true' && "$ENABLE_INSTALL" != 'true' ]]; then echo '--- :arrow_up: Uploading Artifacts' diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index 5c3c3a234d..b952dbcd40 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -9,18 +9,13 @@ SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" [[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." # docker build echo "+++ :docker: Build Docker Container" -DOCKERHUB_REGISTRY='docker.io/eosio/eosio' -IMAGE="${DOCKERHUB_REGISTRY}:${BUILDKITE_COMMIT:-latest}" -DOCKER_BUILD="docker build -t '$IMAGE' -f ./docker/dockerfile ." -echo "$ $DOCKER_BUILD" -eval $DOCKER_BUILD +IMAGE="${DOCKER_REGISTRY:-$REGISTRY_BINARY}:${BUILDKITE_COMMIT:-latest}" +DOCKER_BUILD_ARGS="-t '$IMAGE' -f ./docker/dockerfile ." +echo "$ docker build $DOCKER_BUILD_ARGS" +[[ -z "${PROXY_DOCKER_BUILD_ARGS:-}" ]] || echo "Appending proxy args: '${PROXY_DOCKER_BUILD_ARGS}'" +eval "docker build ${PROXY_DOCKER_BUILD_ARGS:-}${DOCKER_BUILD_ARGS}" # docker tag echo '--- :label: Tag Container' -if [[ "$BUILDKITE_PIPELINE_SLUG" =~ "-sec" ]] ; then - REGISTRIES=("$EOSIO_REGISTRY") -else - REGISTRIES=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") -fi for REG in ${REGISTRIES[@]}; do DOCKER_TAG_BRANCH="docker tag '$IMAGE' '$REG:$SANITIZED_BRANCH'" echo "$ $DOCKER_TAG_BRANCH" diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 3c4ddf35c9..7b0c98a15c 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eo pipefail +set -euo pipefail echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh PREFIX='base-ubuntu-18.04' @@ -11,7 +11,7 @@ echo '$ echo ${#CONTRACT_REGISTRIES[@]} # array length' echo ${#CONTRACT_REGISTRIES[@]} echo '$ echo ${CONTRACT_REGISTRIES[@]} # array' echo ${CONTRACT_REGISTRIES[@]} -export IMAGE="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" +export IMAGE="${REGISTRY_SOURCE:-$DOCKER_CONTRACTS_REGISTRY}:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" # pull echo '+++ :arrow_down: Pulling Container(s)' DOCKER_PULL_COMMAND="docker pull '$IMAGE'" diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh index b6731a51f6..3d703e7514 100755 --- a/.cicd/generate-base-images.sh +++ b/.cicd/generate-base-images.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eo pipefail +set -euo pipefail . ./.cicd/helpers/general.sh . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile" # search for base image in docker registries @@ -7,7 +7,7 @@ echo '--- :docker: Build or Pull Base Image :minidisc:' echo "Looking for '$HASHED_IMAGE_TAG' container in our registries." export EXISTS_DOCKER_HUB='false' export EXISTS_MIRROR='false' -MANIFEST_COMMAND="docker manifest inspect '${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG'" +MANIFEST_COMMAND="docker manifest inspect '${REGISTRY_BASE:-$DOCKER_CI_REGISTRY}:$HASHED_IMAGE_TAG'" echo "$ $MANIFEST_COMMAND" set +e eval $MANIFEST_COMMAND @@ -21,34 +21,34 @@ if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then fi fi # pull and copy as-necessary -if [[ "$EXISTS_MIRROR" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then - DOCKER_PULL_COMMAND="docker pull '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" +if [[ "$EXISTS_MIRROR" == 'true' && ! -z "$REGISTRY_BASE" ]]; then + DOCKER_PULL_COMMAND="docker pull '$REGISTRY_BASE:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PULL_COMMAND" eval $DOCKER_PULL_COMMAND # copy, if necessary - if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then + if [[ "$EXISTS_DOCKER_HUB" == 'false' && "$(echo "$BUILDKITE_PIPELINE_SLUG" | grep -icP '^(eosio|eosio-build-unpinned|eosio-base-images.*)$')" != '0' ]]; then # tag - DOCKER_TAG_COMMAND="docker tag '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG' '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_TAG_COMMAND="docker tag '$REGISTRY_BASE:$HASHED_IMAGE_TAG' '$DOCKER_CI_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND # push - DOCKER_PUSH_COMMAND="docker push '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_PUSH_COMMAND="docker push '$DOCKER_CI_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND export EXISTS_DOCKER_HUB='true' fi elif [[ "$EXISTS_DOCKER_HUB" == 'true' ]]; then - DOCKER_PULL_COMMAND="docker pull '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_PULL_COMMAND="docker pull '$DOCKER_CI_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PULL_COMMAND" eval $DOCKER_PULL_COMMAND # copy, if necessary - if [[ "$EXISTS_MIRROR" == 'false' && ! -z "$MIRROR_REGISTRY" ]]; then + if [[ "$EXISTS_MIRROR" == 'false' && ! -z "$REGISTRY_BASE" ]]; then # tag - DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_TAG_COMMAND="docker tag '$DOCKER_CI_REGISTRY:$HASHED_IMAGE_TAG' '$REGISTRY_BASE:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND # push - DOCKER_PUSH_COMMAND="docker push '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_PUSH_COMMAND="docker push '$REGISTRY_BASE:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND export EXISTS_MIRROR='true' @@ -64,9 +64,10 @@ elif [[ "$FORCE_BASE_IMAGE" == 'true' ]]; then fi # build, if neccessary if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false') || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first - export DOCKER_BUILD_COMMAND="docker build --no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ." - echo "$ $DOCKER_BUILD_COMMAND" - eval $DOCKER_BUILD_COMMAND + export DOCKER_BUILD_ARGS="--no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ." + echo "$ docker build $DOCKER_BUILD_ARGS" + [[ -z "${PROXY_DOCKER_BUILD_ARGS:-}" ]] || echo "Appending proxy args: '${PROXY_DOCKER_BUILD_ARGS}'" + eval "docker build ${PROXY_DOCKER_BUILD_ARGS:-}${DOCKER_BUILD_ARGS}" if [[ "$FORCE_BASE_IMAGE" != 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then for REGISTRY in ${CI_REGISTRIES[*]}; do if [[ ! -z "$REGISTRY" ]]; then @@ -80,13 +81,13 @@ if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false') || "$FORC eval $DOCKER_PUSH_COMMAND # clean up if [[ "$FULL_TAG" != "$REGISTRY:$HASHED_IMAGE_TAG" ]]; then - DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$HASHED_IMAGE_TAG' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND fi fi done - DOCKER_RMI_COMMAND="docker rmi 'ci:$HASHED_IMAGE_TAG'" + DOCKER_RMI_COMMAND="docker rmi 'ci:$HASHED_IMAGE_TAG' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND else diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 3e6c08bf3b..e48293f7df 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -2,11 +2,13 @@ set -eo pipefail # environment . ./.cicd/helpers/general.sh +[[ -z "$ANKA_REMOTE" ]] && export ANKA_REMOTE="${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}" +[[ -z "$BUILDKITE_BASIC_AGENT_QUEUE" ]] && BUILDKITE_BASIC_AGENT_QUEUE='automation-basic-builder-fleet' +[[ -z "$BUILDKITE_BUILD_AGENT_QUEUE" ]] && BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' +[[ -z "$BUILDKITE_TEST_AGENT_QUEUE" ]] && BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' export PLATFORMS_JSON_ARRAY='[]' [[ -z "$ROUNDS" ]] && export ROUNDS='1' [[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='1' -BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' -BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd" export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" @@ -93,7 +95,6 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do "VERSION_MINOR": env.VERSION_MINOR, "VERSION_FULL": env.VERSION_FULL, "PLATFORM_NAME_FULL": env.PLATFORM_NAME_FULL, - "DOCKERHUB_FULL_TAG": env.FULL_TAG, "HASHED_IMAGE_TAG": env.HASHED_IMAGE_TAG, "ICON": env.ICON, "ANKA_TAG_BASE": env.ANKA_TAG_BASE, @@ -152,7 +153,7 @@ EOF cat < ~/.bashrc && \ echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> ~/.bashrc diff --git a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile index 51b10d1276..037402b049 100644 --- a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile +++ b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile @@ -12,15 +12,19 @@ RUN yum update -y && \ libuuid-devel libtasn1-devel expect socat libseccomp-devel iproute && \ yum clean all && rm -rf /var/cache/yum # install erlang and rabbitmq -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | bash && \ +RUN curl -fsSLO https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh && \ + bash script.rpm.sh && \ + rm script.rpm.sh && \ yum install -y erlang -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | bash && \ +RUN curl -fsSLO https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh && \ + bash script.rpm.sh && \ + rm script.rpm.sh && \ yum install -y rabbitmq-server # upgrade pip installation RUN source /opt/rh/rh-python36/enable && \ pip install --upgrade pip && pip install requests requests_unixsocket # build cmake -RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ +RUN curl -fsSLO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ cd cmake-3.16.2 && \ source /opt/rh/devtoolset-8/enable && \ @@ -61,7 +65,7 @@ RUN curl -fsSLO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 # TPM support; this is a little tricky because we'd like nodeos static linked with it, but the tpm2-tools needed # for unit testing will need to be dynamic linked -RUN curl -LO https://github.com/tpm2-software/tpm2-tss/releases/download/3.0.1/tpm2-tss-3.0.1.tar.gz +RUN curl -fsSLO https://github.com/tpm2-software/tpm2-tss/releases/download/3.0.1/tpm2-tss-3.0.1.tar.gz # build static tpm2-tss; this needs some "patching" by way of removing some duplicate symbols at end of tcti impls RUN tar xf tpm2-tss-3.0.1.tar.gz && \ cd tpm2-tss-3.0.1 && \ @@ -85,13 +89,14 @@ RUN tar xf tpm2-tss-3.0.1.tar.gz && \ cd .. && \ rm -rf tpm2-tss-3.0.1* # build TPM components used in unitests; tpm2-tools first -RUN curl -L https://github.com/tpm2-software/tpm2-tools/releases/download/4.3.0/tpm2-tools-4.3.0.tar.gz | tar zx && \ +RUN curl -fsSLO https://github.com/tpm2-software/tpm2-tools/releases/download/4.3.0/tpm2-tools-4.3.0.tar.gz && \ + tar zxf tpm2-tools-4.3.0.tar.gz && \ cd tpm2-tools-4.3.0 && \ . /opt/rh/devtoolset-8/enable && \ PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./configure && \ make -j$(nproc) install && \ cd .. && \ - rm -rf tpm2-tools-4.3.0 + rm -rf tpm2-tools-4.3.0* # build libtpms RUN git clone -b v0.7.3 https://github.com/stefanberger/libtpms && \ cd libtpms && \ @@ -114,7 +119,9 @@ RUN git clone -b v0.5.0 https://github.com/stefanberger/swtpm && \ rm -rf swtpm RUN ldconfig # install nvm -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +RUN curl -fsSLO https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh && \ + bash install.sh && \ + rm install.sh # load nvm in non-interactive shells RUN cp ~/.bashrc ~/.bashrc.bak && \ cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ diff --git a/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile index dc32d18591..bb22e358c7 100644 --- a/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile @@ -15,7 +15,7 @@ RUN apt-get update && \ # install request and requests_unixsocket module RUN pip3 install requests requests_unixsocket # build cmake -RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ +RUN curl -fsSLO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ @@ -56,7 +56,7 @@ RUN curl -fsSLO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source # TPM support; this is a little tricky because we'd like nodeos static linked with it, but the tpm2-tools needed # for unit testing will need to be dynamic linked -RUN curl -LO https://github.com/tpm2-software/tpm2-tss/releases/download/3.0.1/tpm2-tss-3.0.1.tar.gz +RUN curl -fsSLO https://github.com/tpm2-software/tpm2-tss/releases/download/3.0.1/tpm2-tss-3.0.1.tar.gz # build static tpm2-tss; this needs some "patching" by way of removing some duplicate symbols at end of tcti impls RUN tar xf tpm2-tss-3.0.1.tar.gz && \ @@ -80,12 +80,13 @@ RUN tar xf tpm2-tss-3.0.1.tar.gz && \ rm -rf tpm2-tss-3.0.1* # build TPM components used in unitests; tpm2-tools first -RUN curl -L https://github.com/tpm2-software/tpm2-tools/releases/download/4.3.0/tpm2-tools-4.3.0.tar.gz | tar zx && \ +RUN curl -fsSLO https://github.com/tpm2-software/tpm2-tools/releases/download/4.3.0/tpm2-tools-4.3.0.tar.gz && \ + tar zxf tpm2-tools-4.3.0.tar.gz && \ cd tpm2-tools-4.3.0 && \ ./configure && \ make -j$(nproc) install && \ cd .. && \ - rm -rf tpm2-tools-4.3.0 + rm -rf tpm2-tools-4.3.0* # build libtpms RUN git clone -b v0.7.3 https://github.com/stefanberger/libtpms && \ cd libtpms && \ @@ -104,7 +105,9 @@ RUN git clone -b v0.5.0 https://github.com/stefanberger/swtpm && \ rm -rf swtpm RUN ldconfig # install nvm -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +RUN curl -fsSLO https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh && \ + bash install.sh && \ + rm install.sh # load nvm in non-interactive shells RUN cp ~/.bashrc ~/.bashrc.bak && \ cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ @@ -113,7 +116,10 @@ RUN cp ~/.bashrc ~/.bashrc.bak && \ # install node 10 RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node -RUN curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash - -RUN apt-get update && apt-get install -y nodejs && \ +RUN curl -fsSLO https://deb.nodesource.com/setup_13.x && \ + bash setup_13.x && \ + rm setup_13.x +RUN apt-get update && \ + apt-get install -y nodejs && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/.cicd/platforms/pinned/ubuntu-20.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-20.04-pinned.dockerfile index bb082d17fe..8d29cf7f81 100644 --- a/.cicd/platforms/pinned/ubuntu-20.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-20.04-pinned.dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && \ # install request and requests_unixsocket module RUN pip3 install requests requests_unixsocket # build cmake -RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ +RUN curl -fsSLO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ diff --git a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile index 01a73c5012..0db56e1010 100644 --- a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile @@ -8,15 +8,19 @@ RUN yum update -y && \ graphviz clang patch llvm-devel llvm-static vim-common jq && \ yum clean all && rm -rf /var/cache/yum # install erlang and rabbitmq -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | bash && \ +RUN curl -fsSLO https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh && \ + bash script.rpm.sh && \ + rm script.rpm.sh && \ yum install -y erlang -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | bash && \ +RUN curl -fsSLO https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh && \ + bash script.rpm.sh && \ + rm script.rpm.sh && \ yum install -y rabbitmq-server # upgrade pip installation. request and requests_unixsocket module RUN pip3 install --upgrade pip && \ pip3 install requests requests_unixsocket # build cmake -RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ +RUN curl -fsSLO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ @@ -35,7 +39,9 @@ RUN curl -fsSLO "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERS cd / && \ rm -rf "boost_${BOOST_VERSION}.tar.bz2" "/boost_${BOOST_VERSION}" # install nvm -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +RUN curl -fsSLO https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh && \ + bash install.sh && \ + rm install.sh # load nvm in non-interactive shells RUN echo 'export NVM_DIR="$HOME/.nvm"' > ~/.bashrc && \ echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> ~/.bashrc diff --git a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile index dd9e33085e..ee7a787091 100644 --- a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile @@ -11,14 +11,18 @@ RUN yum update -y && \ libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static && \ yum clean all && rm -rf /var/cache/yum # install erlang and rabbitmq -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | bash && \ +RUN curl -fsSLO https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh && \ + bash script.rpm.sh && \ + rm script.rpm.sh && \ yum install -y erlang -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | bash && \ +RUN curl -fsSLO https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh && \ + bash script.rpm.sh && \ + rm script.rpm.sh && \ yum install -y rabbitmq-server RUN source /opt/rh/rh-python36/enable && \ pip install --upgrade pip && pip install requests requests_unixsocket # build cmake -RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ +RUN curl -fsSLO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ cd cmake-3.16.2 && \ source /opt/rh/devtoolset-8/enable && \ @@ -40,7 +44,9 @@ RUN curl -fsSLO "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERS cd / && \ rm -rf "boost_${BOOST_VERSION}.tar.bz2" "/boost_${BOOST_VERSION}" # install nvm -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +RUN curl -fsSLO https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh && \ + bash install.sh && \ + rm install.sh # load nvm in non-interactive shells RUN cp ~/.bashrc ~/.bashrc.bak && \ cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ diff --git a/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile b/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile index 2c39a1dabe..9e2ac85ec3 100644 --- a/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && \ # install request and requests_unixsocket module RUN pip3 install requests requests_unixsocket # build cmake -RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ +RUN curl -fsSLO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ @@ -33,7 +33,9 @@ RUN curl -fsSLO "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERS cd / && \ rm -rf "boost_${BOOST_VERSION}.tar.bz2" "/boost_${BOOST_VERSION}" # install nvm -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +RUN curl -fsSLO https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh && \ + bash install.sh && \ + rm install.sh # load nvm in non-interactive shells RUN cp ~/.bashrc ~/.bashrc.bak && \ cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ @@ -42,7 +44,10 @@ RUN cp ~/.bashrc ~/.bashrc.bak && \ # install node 10 RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node -RUN curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash - -RUN sudo apt-get install -y nodejs && \ +RUN curl -fsSLO https://deb.nodesource.com/setup_13.x && \ + bash setup_13.x && \ + rm setup_13.x +RUN apt-get update && \ + apt-get install -y nodejs && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/.cicd/platforms/unpinned/ubuntu-20.04-unpinned.dockerfile b/.cicd/platforms/unpinned/ubuntu-20.04-unpinned.dockerfile index 00685393ff..23bca1d4a3 100644 --- a/.cicd/platforms/unpinned/ubuntu-20.04-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/ubuntu-20.04-unpinned.dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && \ # install request and requests_unixsocket module RUN pip3 install requests requests_unixsocket # build cmake -RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ +RUN curl -fsSLO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ diff --git a/.cicd/test-package.docker.sh b/.cicd/test-package.docker.sh index b96f0e9271..a9409e544f 100755 --- a/.cicd/test-package.docker.sh +++ b/.cicd/test-package.docker.sh @@ -6,4 +6,7 @@ set -euo pipefail echo '--- :docker: Pretest Setup' perform "docker pull $IMAGE" -perform "docker run --rm -v \"\$(pwd):/eos\" -w '/eos' -it $IMAGE ./.cicd/test-package.run.sh" +DOCKER_RUN_ARGS="--rm -v \"\$(pwd):/eos\" -w '/eos' -it $IMAGE ./.cicd/test-package.run.sh" +echo "$ docker run $DOCKER_RUN_ARGS" +[[ -z "${PROXY_DOCKER_RUN_ARGS:-}" ]] || echo "Appending proxy args: '${PROXY_DOCKER_RUN_ARGS}'" +eval "docker run ${PROXY_DOCKER_RUN_ARGS:-}${DOCKER_RUN_ARGS}" diff --git a/.cicd/test.sh b/.cicd/test.sh index e76881c20c..02f8fa5b54 100755 --- a/.cicd/test.sh +++ b/.cicd/test.sh @@ -15,10 +15,10 @@ else # Linux TEST_COMMAND="'\"'$MOUNTED_DIR/$1'\"' ${@: 2}" COMMANDS="echo \"$ $TEST_COMMAND\" && eval $TEST_COMMAND" . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile" - DOCKER_RUN_COMMAND="docker run --rm --init -v \"\$(pwd):$MOUNTED_DIR\" $(buildkite-intrinsics) -e JOBS -e BUILDKITE_API_KEY '$FULL_TAG' bash -c '$COMMANDS'" + DOCKER_RUN_COMMAND="--rm --init -v \"\$(pwd):$MOUNTED_DIR\" $(buildkite-intrinsics) -e JOBS -e BUILDKITE_API_KEY '$FULL_TAG' bash -c '$COMMANDS'" set +e # defer error handling to end - echo "$ $DOCKER_RUN_COMMAND" - eval $DOCKER_RUN_COMMAND + echo "$ docker run $DOCKER_RUN_COMMAND" + eval "docker run ${DOCKER_RUN_COMMAND}" EXIT_STATUS=$? fi # buildkite diff --git a/.cicd/base-images.yml b/.cicd/unpinned-base-images.yml similarity index 50% rename from .cicd/base-images.yml rename to .cicd/unpinned-base-images.yml index dc3280ecc4..fa0539a4d8 100644 --- a/.cicd/base-images.yml +++ b/.cicd/unpinned-base-images.yml @@ -1,78 +1,6 @@ steps: - wait - - label: ":aws: Amazon_Linux 2 - Base Image Pinned" - command: - - "./.cicd/generate-base-images.sh" - env: - FORCE_BASE_IMAGE: true - IMAGE_TAG: amazon_linux-2-pinned - PLATFORM_TYPE: pinned - agents: - queue: "automation-eks-eos-builder-fleet" - timeout: 180 - skip: ${SKIP_AMAZON_LINUX_2}${SKIP_LINUX} - - - label: ":centos: CentOS 7.7 - Base Image Pinned" - command: - - "./.cicd/generate-base-images.sh" - env: - FORCE_BASE_IMAGE: true - IMAGE_TAG: centos-7.7-pinned - PLATFORM_TYPE: pinned - agents: - queue: "automation-eks-eos-builder-fleet" - timeout: 180 - skip: ${SKIP_CENTOS_7_7}${SKIP_LINUX} - - - label: ":darwin: macOS 10.15 - Base Image Pinned" - command: - - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" - - "cd eos && ./.cicd/platforms/pinned/macos-10.15-pinned.sh" - plugins: - - EOSIO/anka#v0.6.1: - debug: true - vm-name: "10.15.5_6C_14G_80G" - no-volume: true - always-pull: true - wait-network: true - pre-execute-sleep: 5 - pre-execute-ping-sleep: github.com - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - failover-registries: - - "registry_1" - - "registry_2" - inherit-environment-vars: true - - EOSIO/skip-checkout#v0.1.1: - cd: ~ - agents: "queue=mac-anka-node-fleet" - timeout: 180 - skip: ${SKIP_MACOS_10_15}${SKIP_MAC} - - - label: ":ubuntu: Ubuntu 18.04 - Base Image Pinned" - command: - - "./.cicd/generate-base-images.sh" - env: - FORCE_BASE_IMAGE: true - IMAGE_TAG: ubuntu-18.04-pinned - PLATFORM_TYPE: pinned - agents: - queue: "automation-eks-eos-builder-fleet" - timeout: 180 - skip: ${SKIP_UBUNTU_18_04}${SKIP_LINUX} - - - label: ":ubuntu: Ubuntu 20.04 - Base Image Pinned" - command: - - "./.cicd/generate-base-images.sh" - env: - FORCE_BASE_IMAGE: true - IMAGE_TAG: ubuntu-20.04-pinned - PLATFORM_TYPE: pinned - agents: - queue: "automation-eks-eos-builder-fleet" - timeout: 180 - skip: ${SKIP_UBUNTU_20_04}${SKIP_LINUX} - - label: ":aws: Amazon_Linux 2 - Base Image Unpinned" command: - "./.cicd/generate-base-images.sh" diff --git a/scripts/serial-test.sh b/scripts/serial-test.sh index 969f5e7f9b..3f87b03f6d 100755 --- a/scripts/serial-test.sh +++ b/scripts/serial-test.sh @@ -12,8 +12,10 @@ if [[ "$(uname)" == 'Linux' ]]; then fi cd "$GIT_ROOT" fi -echo "$ npm install" -npm install +if [[ "$NPM_INSTALL" == 'true' ]]; then + echo "$ npm install" + npm install +fi cd "$GIT_ROOT/build" # tests if [[ -z "$TEST" ]]; then # run all serial tests