Merge pull request #11057 from EOSIO/registry-work

CI Improvements + HTTP(S) Proxy Support

.cicd/build.sh

@@ -58,9 +58,10 @@ else # Linux
     fi
     . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile"
     COMMANDS="$PRE_COMMANDS && $COMMANDS"
-    DOCKER_RUN="docker run $ARGS $(buildkite-intrinsics) --env CMAKE_EXTRAS='$CMAKE_EXTRAS' '$FULL_TAG' bash -c '$COMMANDS'"
-    echo "$ $DOCKER_RUN"
-    eval $DOCKER_RUN
+    DOCKER_RUN_ARGS="$ARGS $(buildkite-intrinsics) --env CMAKE_EXTRAS='$CMAKE_EXTRAS' '$FULL_TAG' bash -c '$COMMANDS'"
+    echo "$ docker run $DOCKER_RUN_ARGS"
+    [[ -z "${PROXY_DOCKER_RUN_ARGS:-}" ]] || echo "Appending proxy args: '${PROXY_DOCKER_RUN_ARGS}'"
+    eval "docker run ${PROXY_DOCKER_RUN_ARGS:-}${DOCKER_RUN_ARGS}"
 fi
 if [[ "$BUILDKITE" == 'true' && "$ENABLE_INSTALL" != 'true' ]]; then
     echo '--- :arrow_up: Uploading Artifacts'

.cicd/create-docker-from-binary.sh

@@ -9,18 +9,13 @@ SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")"
 [[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'."
 # docker build
 echo "+++ :docker: Build Docker Container"
-DOCKERHUB_REGISTRY='docker.io/eosio/eosio'
-IMAGE="${DOCKERHUB_REGISTRY}:${BUILDKITE_COMMIT:-latest}"
-DOCKER_BUILD="docker build -t '$IMAGE' -f ./docker/dockerfile ."
-echo "$ $DOCKER_BUILD"
-eval $DOCKER_BUILD
+IMAGE="${DOCKER_REGISTRY:-$REGISTRY_BINARY}:${BUILDKITE_COMMIT:-latest}"
+DOCKER_BUILD_ARGS="-t '$IMAGE' -f ./docker/dockerfile ."
+echo "$ docker build $DOCKER_BUILD_ARGS"
+[[ -z "${PROXY_DOCKER_BUILD_ARGS:-}" ]] || echo "Appending proxy args: '${PROXY_DOCKER_BUILD_ARGS}'"
+eval "docker build ${PROXY_DOCKER_BUILD_ARGS:-}${DOCKER_BUILD_ARGS}"
 # docker tag
 echo '--- :label: Tag Container'
-if [[ "$BUILDKITE_PIPELINE_SLUG" =~ "-sec" ]] ; then
-    REGISTRIES=("$EOSIO_REGISTRY")
-else
-    REGISTRIES=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY")
-fi
 for REG in ${REGISTRIES[@]}; do
     DOCKER_TAG_BRANCH="docker tag '$IMAGE' '$REG:$SANITIZED_BRANCH'"
     echo "$ $DOCKER_TAG_BRANCH"

.cicd/docker-tag.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -eo pipefail
+set -euo pipefail
 echo '--- :evergreen_tree: Configuring Environment'
 . ./.cicd/helpers/general.sh
 PREFIX='base-ubuntu-18.04'
@@ -11,7 +11,7 @@ echo '$ echo ${#CONTRACT_REGISTRIES[@]} # array length'
 echo ${#CONTRACT_REGISTRIES[@]}
 echo '$ echo ${CONTRACT_REGISTRIES[@]} # array'
 echo ${CONTRACT_REGISTRIES[@]}
-export IMAGE="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE"
+export IMAGE="${REGISTRY_SOURCE:-$DOCKER_CONTRACTS_REGISTRY}:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE"
 # pull
 echo '+++ :arrow_down: Pulling Container(s)'
 DOCKER_PULL_COMMAND="docker pull '$IMAGE'"

.cicd/generate-base-images.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
-set -eo pipefail
+set -euo pipefail
 . ./.cicd/helpers/general.sh
 . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile"
 # search for base image in docker registries
 echo '--- :docker: Build or Pull Base Image :minidisc:'
 echo "Looking for '$HASHED_IMAGE_TAG' container in our registries."
 export EXISTS_DOCKER_HUB='false'
 export EXISTS_MIRROR='false'
-MANIFEST_COMMAND="docker manifest inspect '${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG'"
+MANIFEST_COMMAND="docker manifest inspect '${REGISTRY_BASE:-$DOCKER_CI_REGISTRY}:$HASHED_IMAGE_TAG'"
 echo "$ $MANIFEST_COMMAND"
 set +e
 eval $MANIFEST_COMMAND
@@ -21,34 +21,34 @@ if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then
     fi
 fi
 # pull and copy as-necessary
-if [[ "$EXISTS_MIRROR" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then
-    DOCKER_PULL_COMMAND="docker pull '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
+if [[ "$EXISTS_MIRROR" == 'true' && ! -z "$REGISTRY_BASE" ]]; then
+    DOCKER_PULL_COMMAND="docker pull '$REGISTRY_BASE:$HASHED_IMAGE_TAG'"
     echo "$ $DOCKER_PULL_COMMAND"
     eval $DOCKER_PULL_COMMAND
     # copy, if necessary
-    if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then
+    if [[ "$EXISTS_DOCKER_HUB" == 'false' && "$(echo "$BUILDKITE_PIPELINE_SLUG" | grep -icP '^(eosio|eosio-build-unpinned|eosio-base-images.*)$')" != '0' ]]; then
         # tag
-        DOCKER_TAG_COMMAND="docker tag '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG' '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
+        DOCKER_TAG_COMMAND="docker tag '$REGISTRY_BASE:$HASHED_IMAGE_TAG' '$DOCKER_CI_REGISTRY:$HASHED_IMAGE_TAG'"
         echo "$ $DOCKER_TAG_COMMAND"
         eval $DOCKER_TAG_COMMAND
         # push
-        DOCKER_PUSH_COMMAND="docker push '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
+        DOCKER_PUSH_COMMAND="docker push '$DOCKER_CI_REGISTRY:$HASHED_IMAGE_TAG'"
         echo "$ $DOCKER_PUSH_COMMAND"
         eval $DOCKER_PUSH_COMMAND
         export EXISTS_DOCKER_HUB='true'
     fi
 elif [[ "$EXISTS_DOCKER_HUB" == 'true' ]]; then
-    DOCKER_PULL_COMMAND="docker pull '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
+    DOCKER_PULL_COMMAND="docker pull '$DOCKER_CI_REGISTRY:$HASHED_IMAGE_TAG'"
     echo "$ $DOCKER_PULL_COMMAND"
     eval $DOCKER_PULL_COMMAND
     # copy, if necessary
-    if [[ "$EXISTS_MIRROR" == 'false' && ! -z "$MIRROR_REGISTRY" ]]; then
+    if [[ "$EXISTS_MIRROR" == 'false' && ! -z "$REGISTRY_BASE" ]]; then
         # tag
-        DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
+        DOCKER_TAG_COMMAND="docker tag '$DOCKER_CI_REGISTRY:$HASHED_IMAGE_TAG' '$REGISTRY_BASE:$HASHED_IMAGE_TAG'"
         echo "$ $DOCKER_TAG_COMMAND"
         eval $DOCKER_TAG_COMMAND
         # push
-        DOCKER_PUSH_COMMAND="docker push '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
+        DOCKER_PUSH_COMMAND="docker push '$REGISTRY_BASE:$HASHED_IMAGE_TAG'"
         echo "$ $DOCKER_PUSH_COMMAND"
         eval $DOCKER_PUSH_COMMAND
         export EXISTS_MIRROR='true'
@@ -64,9 +64,10 @@ elif [[ "$FORCE_BASE_IMAGE" == 'true' ]]; then
 fi
 # build, if neccessary
 if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false') || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first
-    export DOCKER_BUILD_COMMAND="docker build --no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ."
-    echo "$ $DOCKER_BUILD_COMMAND"
-    eval $DOCKER_BUILD_COMMAND
+    export DOCKER_BUILD_ARGS="--no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ."
+    echo "$ docker build $DOCKER_BUILD_ARGS"
+    [[ -z "${PROXY_DOCKER_BUILD_ARGS:-}" ]] || echo "Appending proxy args: '${PROXY_DOCKER_BUILD_ARGS}'"
+    eval "docker build ${PROXY_DOCKER_BUILD_ARGS:-}${DOCKER_BUILD_ARGS}"
     if [[ "$FORCE_BASE_IMAGE" != 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then
         for REGISTRY in ${CI_REGISTRIES[*]}; do
             if [[ ! -z "$REGISTRY" ]]; then
@@ -80,13 +81,13 @@ if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false') || "$FORC
                 eval $DOCKER_PUSH_COMMAND
                 # clean up
                 if  [[ "$FULL_TAG" != "$REGISTRY:$HASHED_IMAGE_TAG" ]]; then
-                    DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$HASHED_IMAGE_TAG'"
+                    DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$HASHED_IMAGE_TAG' || :"
                     echo "$ $DOCKER_RMI_COMMAND"
                     eval $DOCKER_RMI_COMMAND
                 fi
             fi
         done
-        DOCKER_RMI_COMMAND="docker rmi 'ci:$HASHED_IMAGE_TAG'"
+        DOCKER_RMI_COMMAND="docker rmi 'ci:$HASHED_IMAGE_TAG' || :"
         echo "$ $DOCKER_RMI_COMMAND"
         eval $DOCKER_RMI_COMMAND
     else

.cicd/generate-pipeline.sh

@@ -2,11 +2,13 @@
 set -eo pipefail
 # environment
 . ./.cicd/helpers/general.sh
+[[ -z "$ANKA_REMOTE" ]] && export ANKA_REMOTE="${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}"
+[[ -z "$BUILDKITE_BASIC_AGENT_QUEUE" ]] && BUILDKITE_BASIC_AGENT_QUEUE='automation-basic-builder-fleet'
+[[ -z "$BUILDKITE_BUILD_AGENT_QUEUE" ]] && BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet'
+[[ -z "$BUILDKITE_TEST_AGENT_QUEUE" ]] && BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet'
 export PLATFORMS_JSON_ARRAY='[]'
 [[ -z "$ROUNDS" ]] && export ROUNDS='1'
 [[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='1'
-BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet'
-BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet'
 # attach pipeline documentation
 export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd"
 export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")"
@@ -93,7 +95,6 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do
         "VERSION_MINOR": env.VERSION_MINOR,
         "VERSION_FULL": env.VERSION_FULL,
         "PLATFORM_NAME_FULL": env.PLATFORM_NAME_FULL,
-        "DOCKERHUB_FULL_TAG": env.FULL_TAG,
         "HASHED_IMAGE_TAG": env.HASHED_IMAGE_TAG,
         "ICON": env.ICON,
         "ANKA_TAG_BASE": env.ANKA_TAG_BASE,
@@ -152,7 +153,7 @@ EOF
         cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build"
     command:
-      - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
+      - "git clone \$ANKA_REMOTE eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
       - "cd eos && ./.cicd/build.sh"
     plugins:
       - EOSIO/anka#v0.6.1:
@@ -176,13 +177,13 @@ EOF
           cd: ~
     env:
       DCMAKE_BUILD_TYPE: $DCMAKE_BUILD_TYPE
-      REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}
+      REPO: $ANKA_REMOTE
       REPO_COMMIT: $BUILDKITE_COMMIT
       TEMPLATE: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TEMPLATE_NAME)
       TEMPLATE_TAG: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TAG_BASE)
       IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
       PLATFORM_TYPE: $PLATFORM_TYPE
-      TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive && export IMAGE_TAG=$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) && export PLATFORM_TYPE=$PLATFORM_TYPE && . ./.cicd/platforms/$PLATFORM_TYPE/$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME).sh && cd ~/eos && cd .. && rm -rf eos"
+      TAG_COMMANDS: "git clone $ANKA_REMOTE eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive && export IMAGE_TAG=$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) && export PLATFORM_TYPE=$PLATFORM_TYPE && . ./.cicd/platforms/$PLATFORM_TYPE/$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME).sh && cd ~/eos && cd .. && rm -rf eos"
       PROJECT_TAG: $(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG)
     timeout: ${TIMEOUT:-180}
     agents: "queue=mac-anka-large-node-fleet"
@@ -238,7 +239,7 @@ EOF
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Unit Tests"
     command:
-      - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
+      - "git clone \$ANKA_REMOTE eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
       - "cd eos && buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz"
       - "cd eos && ./.cicd/test.sh scripts/parallel-test.sh"
     plugins:
@@ -295,7 +296,7 @@ EOF
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - WASM Spec Tests"
     command:
-      - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
+      - "git clone \$ANKA_REMOTE eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
       - "cd eos && buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz"
       - "cd eos && ./.cicd/test.sh scripts/wasm-spec-test.sh"
     plugins:
@@ -334,6 +335,7 @@ EOF
                 SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq | grep -P "^$TEST$" | awk "{while(i++<$ROUND_SIZE)print;i=0}")"
             fi
             for TEST_NAME in $SERIAL_TESTS; do
+                [[ "$(echo "$TEST_NAME" | grep -cP '^ship')" == '1' ]] && export NPM_INSTALL='true' || export NPM_INSTALL='false'
                 if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then
                     cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
@@ -342,6 +344,7 @@ EOF
       ./.cicd/test.sh scripts/serial-test.sh $TEST_NAME
     env:
       IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)
+      NPM_INSTALL: ${NPM_INSTALL:-false}
       PLATFORM_TYPE: $PLATFORM_TYPE
     agents:
       queue: "$BUILDKITE_TEST_AGENT_QUEUE"
@@ -375,7 +378,7 @@ EOF
                 cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
-      - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
+      - "git clone \$ANKA_REMOTE eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
       - "cd eos && buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz"
       - "cd eos && ./.cicd/test.sh scripts/serial-test.sh $TEST_NAME"
     plugins:
@@ -394,6 +397,8 @@ EOF
             - 'registry_2'
       - EOSIO/skip-checkout#v0.1.1:
           cd: ~
+    env:
+      NPM_INSTALL: ${NPM_INSTALL:-false}
     agents: "queue=mac-anka-node-fleet"
     retry:
       manual:
@@ -434,7 +439,7 @@ EOF
                     cat <<EOF
   - label: "$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - $TEST_NAME"
     command:
-      - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
+      - "git clone \$ANKA_REMOTE eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive"
       - "cd eos && buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' ${BUILD_SOURCE} && tar -xzf build.tar.gz"
       - "cd eos && ./.cicd/test.sh scripts/long-running-test.sh $TEST_NAME"
     plugins:
@@ -621,7 +626,7 @@ EOF
 
   - label: ":darwin: macOS 10.15 - Package Builder"
     command:
-      - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
+      - "git clone \$ANKA_REMOTE eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
       - "cd eos && buildkite-agent artifact download build.tar.gz . --step ':darwin: macOS 10.15 - Build' && tar -xzf build.tar.gz"
       - "cd eos && ./.cicd/package.sh"
     plugins:
@@ -693,7 +698,7 @@ EOF
 
   - label: ":darwin: macOS 10.15 - Install Package"
     command:
-      - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
+      - "git clone \$ANKA_REMOTE eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT"
       - "cd eos && buildkite-agent artifact download '*' . --step ':darwin: macOS 10.15 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN"
       - "cd eos && ./.cicd/test-package.anka.sh"
     plugins:
@@ -732,15 +737,15 @@ EOF
   - label: ":git: Git Submodule Regression Check"
     command: "./.cicd/submodule-regression-check.sh"
     agents:
-      queue: "automation-basic-builder-fleet"
+      queue: "$BUILDKITE_BASIC_AGENT_QUEUE"
     timeout: ${TIMEOUT:-5}
 
   - label: ":beer: Brew Updater"
     command: |
       buildkite-agent artifact download eosio.rb . --step ':darwin: macOS 10.15 - Package Builder'
       buildkite-agent artifact upload eosio.rb
     agents:
-      queue: "automation-basic-builder-fleet"
+      queue: "$BUILDKITE_BASIC_AGENT_QUEUE"
     timeout: ${TIMEOUT:-5}
     skip: "See BLU-30502"
 

.cicd/helpers/file-hash.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
-set -eo pipefail
+set -euo pipefail
 [[ -z "$1" ]] && echo 'Please provide the file to be hashed as first argument.' && exit 1
 FILE_NAME="$(basename "$1" | awk '{split($0,a,/\.(d|s)/); print a[1] }')"
 export DETERMINED_HASH=$(sha1sum "$1" | awk '{ print $1 }')
 export HASHED_IMAGE_TAG="eos-${FILE_NAME}-${DETERMINED_HASH}"
-export FULL_TAG="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG"
+if [[ -n "${REGISTRY_BASE:-''}" ]] ; then
+    export FULL_TAG="$REGISTRY_BASE:$HASHED_IMAGE_TAG"
+fi
+set +u

.cicd/helpers/general.sh

@@ -5,14 +5,20 @@ export HELPERS_DIR="$CICD_DIR/helpers"
 export JOBS=${JOBS:-"$(getconf _NPROCESSORS_ONLN)"}
 export MOUNTED_DIR='/eos'
 export DOCKER_CLI_EXPERIMENTAL='enabled'
-export DOCKERHUB_CI_REGISTRY="docker.io/eosio/ci"
-export DOCKERHUB_CONTRACTS_REGISTRY="docker.io/eosio/ci-contracts-builder"
-if [[ "$BUILDKITE_PIPELINE_SLUG" =~ "-sec" ]] ; then
-    export CI_REGISTRIES=("$MIRROR_REGISTRY")
-    export CONTRACT_REGISTRIES=("$MIRROR_REGISTRY")
+export DOCKER_REGISTRY='docker.io/eosio/eosio'
+export DOCKER_CI_REGISTRY="docker.io/eosio/ci"
+export DOCKER_CONTRACTS_REGISTRY="docker.io/eosio/ci-contracts-builder"
+export REGISTRY_BASE=${REGISTRY_BASE:-$MIRROR_REGISTRY}
+export REGISTRY_BINARY=${REGISTRY_BINARY:-$EOSIO_REGISTRY}
+export REGISTRY_SOURCE=${REGISTRY_SOURCE:-$MIRROR_REGISTRY}
+if [[ "$(echo "$BUILDKITE_PIPELINE_SLUG" | grep -icP '^(eosio|eosio-build-unpinned|eosio-base-images.*)$')" == '0' ]] ; then
+    export REGISTRIES=("$REGISTRY_BINARY")
+    export CI_REGISTRIES=("$REGISTRY_BASE")
+    export CONTRACT_REGISTRIES=("$REGISTRY_SOURCE")
 else
-    export CI_REGISTRIES=("$DOCKERHUB_CI_REGISTRY" "$MIRROR_REGISTRY")
-    export CONTRACT_REGISTRIES=("$DOCKERHUB_CONTRACTS_REGISTRY" "$MIRROR_REGISTRY")
+    export REGISTRIES=("$DOCKER_REGISTRY" "$REGISTRY_BINARY")
+    export CI_REGISTRIES=("$DOCKER_CI_REGISTRY" "$REGISTRY_BASE")
+    export CONTRACT_REGISTRIES=("$DOCKER_CONTRACTS_REGISTRY" "$REGISTRY_SOURCE")
 fi
 
 # capitalize each word in a string

.cicd/installation-build.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
-set -eo pipefail
+set -euo pipefail
 echo '--- :evergreen_tree: Configuring Environment'
 . ./.cicd/helpers/general.sh
 export ENABLE_INSTALL='true'
 export SANITIZED_BRANCH=$(sanitize "$BUILDKITE_BRANCH")
 echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'."
-export CONTRACTS_BUILDER_TAG="eosio/ci-contracts-builder:base-ubuntu-18.04"
 export ARGS="--name ci-contracts-builder-$BUILDKITE_PIPELINE_SLUG-$BUILDKITE_BUILD_NUMBER --init -v \"\$(pwd):$MOUNTED_DIR\""
 BUILD_COMMAND="'$CICD_DIR/build.sh'"
 echo "$ $BUILD_COMMAND"

.cicd/package.sh

@@ -24,9 +24,10 @@ else # Linux
         PACKAGE_COMMANDS="mkdir -p ~/rpmbuild/BUILD && mkdir -p ~/rpmbuild/BUILDROOT && mkdir -p ~/rpmbuild/RPMS && mkdir -p ~/rpmbuild/SOURCES && mkdir -p ~/rpmbuild/SPECS && mkdir -p ~/rpmbuild/SRPMS && yum install -y rpm-build && ./generate_package.sh \"$PACKAGE_TYPE\""
     fi
     COMMANDS="echo \"+++ :package: Packaging EOSIO\" && $PRE_COMMANDS && $PACKAGE_COMMANDS"
-    DOCKER_RUN_COMMAND="docker run $ARGS $(buildkite-intrinsics) '$FULL_TAG' bash -c '$COMMANDS'"
-    echo "$ $DOCKER_RUN_COMMAND"
-    eval $DOCKER_RUN_COMMAND
+    DOCKER_RUN_ARGS="$ARGS $(buildkite-intrinsics) '$FULL_TAG' bash -c '$COMMANDS'"
+    echo "$ docker run $DOCKER_RUN_ARGS"
+    [[ -z "${PROXY_DOCKER_RUN_ARGS:-}" ]] || echo "Appending proxy args: '${PROXY_DOCKER_RUN_ARGS}'"
+    eval "docker run ${PROXY_DOCKER_RUN_ARGS:-}${DOCKER_RUN_ARGS}"
 fi
 cd build/packages
 [[ -d x86_64 ]] && cd 'x86_64' # backwards-compatibility with release/1.6.x