Skip to content

Merge pull request #1710 from chris-j-h/fix-release-workflow #266

Merge pull request #1710 from chris-j-h/fix-release-workflow

Merge pull request #1710 from chris-j-h/fix-release-workflow #266

Workflow file for this run

name: Build Check
on:
push:
paths-ignore:
- 'docs/**'
- '**.md'
- 'mkdocs.yml'
- '.gitignore'
branches:
- master
env:
GO_VERSION: 1.21.3
GOPATH: ${{ github.workspace }}/go
WORKING_DIR: ${{ github.workspace }}/go/src/github.com/ethereum/go-ethereum
jobs:
build:
name: 'Run tests and build on ${{ matrix.os }}'
strategy:
fail-fast: false
matrix:
# Not enable for macos as there's a consistent failure:
# --- FAIL: TestUPNP_DDWRT (2.20s)
# ###[error] natupnp_test.go:165: not discovered
# must be sommething with Github Actions VM networking setup.
# Event Ubuntu requires a workaround
os: [ "ubuntu-20.04" ]
env:
QUORUM_IGNORE_TEST_PACKAGES: github.com/ethereum/go-ethereum/les,github.com/ethereum/go-ethereum/les/flowcontrol,github.com/ethereum/go-ethereum/mobile
runs-on: ${{ matrix.os }}
steps:
- name: 'Setup Go ${{ env.GO_VERSION }}'
uses: actions/setup-go@v1
with:
go-version: ${{ env.GO_VERSION }}
- name: 'Check out project files'
uses: actions/checkout@v2
with:
submodules: recursive
path: ${{ env.WORKING_DIR }}
- name: 'Apply workaround to fix networking in Linux'
if: runner.os == 'Linux'
run: |
# https://github.com/actions/virtual-environments/issues/798
sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
- name: 'Prepare environment'
run: |
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
- name: 'Run tests and build all'
working-directory: ${{ env.WORKING_DIR }}
run: |
make test all
docker-build:
name: 'Build Docker image'
runs-on: ubuntu-20.04
steps:
- name: 'Check out project files'
uses: actions/checkout@v2
- name: 'Build docker image'
id: build
run: |
output_dir=${{ runner.temp }}/docker
mkdir -p $output_dir
docker build -t quorumengineering/quorum:pr .
docker save quorumengineering/quorum:pr > quorum-pr.tar
tar cfvz $output_dir/quorum-pr.tar.gz quorum-pr.tar
echo "::set-output name=output_dir::$output_dir"
- name: 'Upload workflow artifact - Docker image'
uses: actions/upload-artifact@v1
with:
name: docker-image
path: ${{ steps.build.outputs.output_dir }}
acceptance-tests-basic:
name: Acceptance tests (${{ matrix.tag }})
needs:
- docker-build
if: success()
strategy:
fail-fast: false
matrix:
# list of tag expression being executed in parallel
# for PR, only selective tests are run.
# More comprehensive suites are scheduled to run in master
tag:
- 'basic || basic-raft || (advanced && raft) || networks/typical::raft'
- 'basic || basic-istanbul || (advanced && istanbul) || empty-block-period || networks/typical::istanbul'
- 'basic || basic-istanbul || (advanced && istanbul) || empty-block-period || block-reward || networks/typical::qbft'
- 'gcmode && block-sync && networks/template::raft-3plus1'
- 'gcmode && block-sync && networks/template::istanbul-3plus1'
- 'gcmode && block-sync && networks/template::qbft-3plus1'
- 'learner-peer-management || raftdnsenable && networks/template::raft-3plus1'
- 'validator-management && networks/template::qbft-3plus1'
- 'validator-management && networks/template::istanbul-3plus1'
- 'hybrid-validator-management-manage-besu && networks/typical-hybrid::hybrid-template-q2b1'
- 'hybrid-validator-management-manage-quorum && networks/typical-hybrid::hybrid-template-q1b2'
- 'qbft-transition-network && networks/template::qbft-4nodes-transition'
- 'basic || basic-raft || (advanced && raft) || networks/plugins::raft'
- 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::qbft'
- 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::istanbul'
- 'basic || basic-raft || (advanced && raft) || networks/plugins::raft-account-plugin-hashicorp-vault'
- 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::qbft-account-plugin-hashicorp-vault'
- 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::istanbul-account-plugin-hashicorp-vault'
- 'basic-rpc-security || networks/plugins::raft-rpc-security'
- 'basic-rpc-security || networks/plugins::qbft-rpc-security'
- 'basic-rpc-security || networks/plugins::istanbul-rpc-security'
- 'migration && networks/template::raft-4nodes'
- 'migration && networks/template::istanbul-4nodes'
- 'migration && networks/template::raft-4nodes-ancientdb'
- 'migration && networks/template::istanbul-4nodes-ancientdb'
- 'permissions-v1 && networks/template::raft-3plus1'
- 'permissions-v2 && networks/template::raft-3plus1'
- 'privacy-enhancements-upgrade || networks/template::raft-4nodes-pe'
- 'privacy-enhancements-upgrade || networks/template::istanbul-4nodes-pe'
- 'multitenancy && networks/plugins::raft-multitenancy'
- 'basic || basic-raft || (advanced && raft) || networks/typical::raft-simple-mps'
- 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::qbft-simple-mps'
- 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul-simple-mps'
- 'basic || networks/typical::raftmps'
- 'basic || networks/typical::qbftmps'
- 'basic || networks/typical::istanbulmps'
- 'mps-upgrade-txtrace || networks/template::raft-4nodes-mps'
- 'mps-upgrade-txtrace || networks/template::istanbul-4nodes-mps'
- 'mps-mixed-network-psr-check || networks/template::raft-4nodes-mps-mixed'
- 'mps-mixed-network-psr-check || networks/template::istanbul-4nodes-mps-mixed'
- '(basic && !nosupport && !mps && !(spam && !raw) && !eth-api-signed && !privacy-enhancements-disabled && !graphql && !async && !extension && !storage-root && !personal-api-signed) || networks/typical-hybrid::hybrid'
runs-on: ubuntu-20.04
steps:
- name: 'Download workflow artifact - Docker image'
uses: actions/download-artifact@v1
with:
name: docker-image
- name: 'Load Docker image'
id: setup
run: |
tar xfvz docker-image/quorum-pr.tar.gz
docker load --input quorum-pr.tar
docker_env_file="${{ runner.temp }}/env.list"
echo "TF_VAR_quorum_docker_image={ name = \"quorumengineering/quorum:pr\", local = true }" >> $docker_env_file
echo "::set-output name=outputDir::${{ runner.temp }}"
echo "::set-output name=dockerEnvFile::$docker_env_file"
- name: 'Run acceptance tests'
run: |
cat ${{ steps.setup.outputs.dockerEnvFile }}
docker run --rm \
--network host \
-v /var/run/docker.sock:/var/run/docker.sock \
-v ${{ steps.setup.outputs.outputDir }}:${{ steps.setup.outputs.outputDir }} \
--env-file ${{ steps.setup.outputs.dockerEnvFile }} \
quorumengineering/acctests:latest test \
-Pauto \
-Dauto.outputDir=${{ steps.setup.outputs.outputDir }} \
-Dtags="${{ matrix.tag }}"
- name: 'Debug'
run: |
docker images
docker ps -a
acceptance-tests-extra:
name: Acceptance tests (${{ matrix.tag }})
needs:
- docker-build
if: success()
strategy:
fail-fast: false
matrix:
# list of tag expression being executed in parallel
include:
# privacy enhancements tests
- tag: '(basic && !privacy-enhancements-disabled) || privacy-enhancements || mandatory-recipients || basic-raft || (advanced && raft) || networks/typical::raft'
privacy-enhancements: true
privacy-precompile: false
privacy-marker-transactions: false
- tag: '(basic && !privacy-enhancements-disabled) || privacy-enhancements || mandatory-recipients || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul'
privacy-enhancements: true
privacy-precompile: false
privacy-marker-transactions: false
# privacy precompile/privacy marker transaction tests
- tag: 'basic || basic-raft || (advanced && raft) || networks/typical::raft'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: false
- tag: 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: false
- tag: 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::qbft'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: false
- tag: '(multitenancy || privacy-precompile-enabled) && networks/plugins::raft-multitenancy'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: true
- tag: '(basic && !privacy-precompile-disabled) || basic-raft || (advanced && raft) || networks/typical::raft-simple-mps'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: true
- tag: '(basic && !privacy-precompile-disabled) || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul-simple-mps'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: true
- tag: '(basic && !privacy-precompile-disabled) || basic-istanbul || (advanced && istanbul) || networks/typical::qbft-simple-mps'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: true
- tag: '(basic && !privacy-precompile-disabled) || networks/typical::raftmps'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: true
- tag: '(basic && !privacy-precompile-disabled) || networks/typical::istanbulmps'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: true
- tag: '(basic && !privacy-precompile-disabled) || networks/typical::qbftmps'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: true
# privacy enhancements + privacy precompile/privacy marker transaction tests
- tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-raft || (advanced && raft) || networks/typical::raft'
privacy-enhancements: true
privacy-precompile: true
privacy-marker-transactions: true
- tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul'
privacy-enhancements: true
privacy-precompile: true
privacy-marker-transactions: true
- tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-istanbul || (advanced && istanbul) || networks/typical::qbft'
privacy-enhancements: true
privacy-precompile: true
privacy-marker-transactions: true
- tag: 'privacy-precompile-compatibility && networks/template::raft-4nodes'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: false # do not enable pmts as the test will do this on the necessary nodes
- tag: 'privacy-precompile-compatibility && networks/template::istanbul-4nodes'
privacy-enhancements: false
privacy-precompile: true
privacy-marker-transactions: false # do not enable pmts as the test will do this on the necessary nodes
runs-on: ubuntu-20.04
steps:
- name: 'Download workflow artifact - Docker image'
uses: actions/download-artifact@v1
with:
name: docker-image
- name: 'Load Docker image'
id: setup
run: |
tar xfvz docker-image/quorum-pr.tar.gz
docker load --input quorum-pr.tar
docker_env_file="${{ runner.temp }}/env.list"
echo "TF_VAR_quorum_docker_image={ name = \"quorumengineering/quorum:pr\", local = true }" >> $docker_env_file
echo "TF_VAR_privacy_enhancements={block=0, enabled=${{ matrix.privacy-enhancements}}}" >> $docker_env_file
echo "TF_VAR_privacy_precompile={block=0, enabled=${{ matrix.privacy-precompile}}}" >> $docker_env_file
echo "TF_VAR_privacy_marker_transactions=${{ matrix.privacy-marker-transactions}}" >> $docker_env_file
echo "::set-output name=outputDir::${{ runner.temp }}"
echo "::set-output name=dockerEnvFile::$docker_env_file"
- name: 'Run extra acceptance tests'
run: |
cat ${{ steps.setup.outputs.dockerEnvFile }}
docker run --rm \
--network host \
-v /var/run/docker.sock:/var/run/docker.sock \
-v ${{ steps.setup.outputs.outputDir }}:${{ steps.setup.outputs.outputDir }} \
--env-file ${{ steps.setup.outputs.dockerEnvFile }} \
quorumengineering/acctests:latest test \
-Pauto \
-Dauto.outputDir=${{ steps.setup.outputs.outputDir }} \
-Dtags="${{ matrix.tag }}"
- name: 'Debug'
run: |
docker images
docker ps -a
peeps-tests:
name: Run PEEPS tests
needs:
- docker-build
runs-on: ubuntu-20.04
steps:
- name: 'Checkout'
uses: actions/checkout@v2
- name: 'Download workflow artifact - Docker image'
uses: actions/download-artifact@v1
with:
name: docker-image
- name: 'Load Docker image'
id: setup
run: |
tar xfvz docker-image/quorum-pr.tar.gz
docker load --input quorum-pr.tar
docker image tag quorumengineering/quorum:pr quorumengineering/quorum:develop
docker image ls
- name: Set up Java
uses: actions/setup-java@v2
with:
distribution: 'adopt'
java-version: 11
check-latest: true
- name: PEEPS
run: |
cd build
./run-peeps.sh
- name: PEEPS Test Report
uses: mikepenz/action-junit-report@v2
if: always()
with:
report_paths: '**/build/test-results/*/TEST-*.xml'
check_name: PEEPS test report
publish-docker:
name: Publish Docker Image
needs:
- build
- acceptance-tests-basic
- acceptance-tests-extra
runs-on: ubuntu-20.04
steps:
- name: 'Checkout'
uses: actions/checkout@v2
- name: 'Build and publish to Docker Hub'
uses: docker/build-push-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
repository: ${{ secrets.DOCKER_REPO }}
tags: develop
add_git_labels: true
notify:
if: always()
name: Notify
needs:
- build
- publish-docker
runs-on: ubuntu-20.04
steps:
- name: 'Setup metadata'
id: setup
run: |
gitref_path="${{ github.ref }}"
gitref_path=${gitref_path/refs\/heads/tree} # for refs/heads/my-branch
gitref_path=${gitref_path/refs\/tags/tree} # for refs/tags/v1.0.0
gitref_path=${gitref_path#refs\/} # for refs/pull/123/merge
gitref_path=${gitref_path%/merge} # for refs/pull/123/merge
echo "::set-output name=gitref-path::$gitref_path"
- name: 'Prepare Slack message with full info'
id: status
uses: actions/github-script@0.8.0
with:
script: |
var gitref_path = "${{ steps.setup.outputs.gitref-path }}"
////////////////////////////////////
// retrieve workflow run data
////////////////////////////////////
console.log("get workflow run")
const wf_run = await github.actions.getWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{ github.run_id }}
})
console.log(wf_run.data)
console.log("get jobs for workflow run:", wf_run.data.jobs_url)
const jobs_response = await github.request(wf_run.data.jobs_url)
////////////////////////////////////
// build slack notification message
////////////////////////////////////
// some utility functions
var date_diff_func = function(start, end) {
var duration = end - start
// format the duration
var delta = duration / 1000
var days = Math.floor(delta / 86400)
delta -= days * 86400
var hours = Math.floor(delta / 3600) % 24
delta -= hours * 3600
var minutes = Math.floor(delta / 60) % 60
delta -= minutes * 60
var seconds = Math.floor(delta % 60)
var format_func = function(v, text, check) {
if (v <= 0 && check) {
return ""
} else {
return v + text
}
}
return format_func(days, "d", true) + format_func(hours, "h", true) + format_func(minutes, "m", true) + format_func(seconds, "s", false)
}
var status_icon_func = function(s) {
switch (s) {
case "w_success":
return ":white_check_mark:"
case "w_failure":
return ":no_entry:"
case "w_cancelled":
return ":warning:"
case "success":
return "\u2713"
case "failure":
return "\u2717"
default:
return "\u20e0"
}
}
// build the message
var job_blocks = []
var is_wf_success = true
var is_wf_failure = false
for (j of jobs_response.data.jobs) {
console.log(j.name, ":", j.status, j.conclusion, j.started_at, j.completed_at)
// ignore the current job running this script
if (j.status != "completed") {
continue
}
if (j.conclusion != "success") {
is_wf_success = false
}
if (j.conclusion == "failure") {
is_wf_failure = true
}
job_blocks.push({
type: "section",
text: {
type: "mrkdwn",
text: `${status_icon_func(j.conclusion)} <${j.html_url}|${j.name}> took ${date_diff_func(new Date(j.started_at), new Date(j.completed_at))}`
}
})
}
var workflow_status = "w_cancelled"
if (is_wf_success) {
workflow_status = "w_success"
} else if (is_wf_failure) {
workflow_status = "w_failure"
}
var context_elements = [
{
"type": "mrkdwn",
"text": "*Repo:* <https://github.com/${{ github.repository }}|${{ github.repository }}>"
},
{
"type": "mrkdwn",
"text": `*Branch:* <https://github.com/${{ github.repository }}/${gitref_path}|${{ github.ref }}>`
},
{
"type": "mrkdwn",
"text": `*Event:* ${wf_run.data.event}`
},
{
"type": "mrkdwn",
"text": `*Commit:* <https://github.com/${{ github.repository }}/commit/${wf_run.data.head_commit.id}|${wf_run.data.head_commit.id.substr(0, 8)}>`
},
{
"type": "mrkdwn",
"text": `*Author:* ${wf_run.data.head_commit.author.name}`
}
]
var header_blocks = [
{
type: "section",
text: {
type: "mrkdwn",
text: `${status_icon_func(workflow_status)} *${{ github.workflow }}* <${wf_run.data.html_url}|#${{ github.run_number }}> took ${date_diff_func(new Date(wf_run.data.created_at), new Date(wf_run.data.updated_at))}`
}
},
{
type: "context",
elements: context_elements,
},
{
type: "divider"
}
]
var slack_msg = {
blocks: [].concat(header_blocks, job_blocks)
}
return slack_msg
- name: 'Prepare Slack message with partial info'
id: short_status
if: failure()
uses: actions/github-script@0.8.0
with:
script: |
////////////////////////////////////
// retrieve workflow run data
////////////////////////////////////
const wf_run = await github.actions.getWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{ github.run_id }}
})
var date_diff_func = function(start, end) {
var duration = end - start
// format the duration
var delta = duration / 1000
var days = Math.floor(delta / 86400)
delta -= days * 86400
var hours = Math.floor(delta / 3600) % 24
delta -= hours * 3600
var minutes = Math.floor(delta / 60) % 60
delta -= minutes * 60
var seconds = Math.floor(delta % 60)
var format_func = function(v, text, check) {
if (v <= 0 && check) {
return ""
} else {
return v + text
}
}
return format_func(days, "d", true) + format_func(hours, "h", true) + format_func(minutes, "m", true) + format_func(seconds, "s", false)
}
var slack_msg = {
blocks: [
{
type: "section",
text: {
type: "mrkdwn",
text: `:skull_and_crossbones: *${{ github.workflow }}* <${wf_run.data.html_url}|#${{ github.run_number }}> (took ${date_diff_func(new Date(wf_run.data.created_at), new Date(wf_run.data.updated_at))})`
}
}
]
}
return slack_msg
- name: 'Send to Slack'
if: always()
run: |
cat <<JSON > long_message.json
${{ steps.status.outputs.result }}
JSON
cat <<JSON > short_message.json
${{ steps.short_status.outputs.result }}
JSON
_post() {
curl -X POST ${{ secrets.SLACK_WEBHOOK_URL }} -H "Content-type: application/json" --data "@${1}"
}
_post "long_message.json" || _post "short_message.json"