Skip to content

Commit 449aa90

Browse files
liamcervanteliamcervante
authored
Execute equivalence tests on every pull request (#35976)
1 parent c816275 commit 449aa90

File tree

7 files changed

+253
-96
lines changed

7 files changed

+253
-96
lines changed

‎.github/actions/equivalence-test/action.yml

Lines changed: 39 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,8 @@
22
# SPDX-License-Identifier: BUSL-1.1
33

44
name: equivalence-test
5-
description: "Execute the suite of Terraform equivalence tests in testing/equivalence-tests"
5+
description: "Execute the suite of Terraform equivalence tests in testing/equivalence-tests and update the golden files."
66
inputs:
7-
target-terraform-version:
8-
description: "The version of Terraform to use in execution."
9-
required: true
10-
target-terraform-branch:
11-
description: "The branch within this repository to update and compare."
12-
required: true
137
target-equivalence-test-version:
148
description: "The version of the Terraform equivalence tests to use."
159
default: "0.3.0"
@@ -19,9 +13,22 @@ inputs:
1913
target-arch:
2014
description: "Current architecture"
2115
default: "amd64"
16+
current-branch:
17+
description: "What branch are we currently on?"
18+
required: true
19+
new-branch:
20+
description: "Name of new branch to be created for the review."
21+
required: true
22+
reviewers:
23+
description: "Comma-separated list of GitHub usernames to request review from."
24+
required: true
25+
message:
26+
description: "Message to include in the commit."
27+
required: true
2228
runs:
2329
using: "composite"
2430
steps:
31+
2532
- name: "download equivalence test binary"
2633
shell: bash
2734
run: |
@@ -30,15 +37,13 @@ runs:
3037
./bin/equivalence-tests \
3138
${{ inputs.target-os }} \
3239
${{ inputs.target-arch }}
33-
- name: "download terraform binary"
40+
41+
- name: Build terraform
3442
shell: bash
35-
run: |
36-
./.github/scripts/equivalence-test.sh download_terraform_binary \
37-
${{ inputs.target-terraform-version }} \
38-
./bin/terraform \
39-
${{ inputs.target-os }} \
40-
${{ inputs.target-arch }}
43+
run: ./.github/scripts/equivalence-test.sh build_terraform_binary ./bin/terraform
44+
4145
- name: "run and update equivalence tests"
46+
id: execute
4247
shell: bash
4348
run: |
4449
./bin/equivalence-tests update \
@@ -47,15 +52,24 @@ runs:
4752
--binary=$(pwd)/bin/terraform
4853
4954
changed=$(git diff --quiet -- testing/equivalence-tests/outputs || echo true)
50-
if [[ $changed == "true" ]]; then
51-
echo "found changes, and pushing new golden files into branch ${{ inputs.target-terraform-branch }}."
52-
53-
git config user.email "52939924+teamterraform@users.noreply.github.com"
54-
git config user.name "The Terraform Team"
55+
echo "changed=$changed" >> "${GITHUB_OUTPUT}"
5556
56-
git add ./testing/equivalence-tests/outputs
57-
git commit -m "Automated equivalence test golden file update for release ${{ inputs.target-terraform-version }}."
58-
git push
59-
else
60-
echo "found no changes, so not pushing any updates."
61-
fi
57+
- name: "branch, commit, and push changes"
58+
if: steps.execute.outputs.changed == 'true'
59+
shell: bash
60+
run: |
61+
git checkout -b ${{ inputs.new-branch }}
62+
git add testing/equivalence-tests/outputs
63+
git commit -m "Update equivalence test golden files."
64+
git push --set-upstream origin ${{ inputs.new-branch }}
65+
66+
- name: "create pull request"
67+
if: steps.execute.outputs.changed == 'true'
68+
shell: bash
69+
run: |
70+
gh pr create \
71+
--base ${{ inputs.current-branch }} \
72+
--head ${{ inputs.new-branch }} \
73+
--title "Update equivalence test golden files" \
74+
--body "This PR updates the equivalence test golden files." \
75+
--reviewer ${{ inputs.reviewers }}

‎.github/scripts/equivalence-test.sh

Lines changed: 12 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,11 @@ Commands:
2626
2727
./equivalence-test.sh download_equivalence_test_binary 0.3.0 ./bin/terraform-equivalence-testing linux amd64
2828
29-
download_terraform_binary <version> <target> <os> <arch>
30-
download_terraform_binary downloads the terraform release binary for a given
31-
version and places it at the target path.
29+
build_terraform_binary <target>
30+
download_terraform_binary builds the Terraform binary and places it at the
31+
target path.
3232
33-
./equivalence-test.sh download_terraform_binary 1.4.3 ./bin/terraform linux amd64
33+
./equivalence-test.sh build_terraform_binary ./bin/terraform
3434
EOF
3535
}
3636

@@ -65,25 +65,17 @@ function download_equivalence_test_binary {
6565
rm releases.json
6666
}
6767

68-
function download_terraform_binary {
69-
VERSION="${1:-}"
70-
TARGET="${2:-}"
71-
OS="${3:-}"
72-
ARCH="${4:-}"
68+
function build_terraform_binary {
69+
TARGET="${1:-}"
7370

74-
if [[ -z "$VERSION" || -z "$TARGET" || -z "$OS" || -z "$ARCH" ]]; then
75-
echo "missing at least one of [<version>, <target>, <os>, <arch>] arguments"
71+
if [[ -z "$TARGET" ]]; then
72+
echo "target argument"
7673
usage
7774
exit 1
7875
fi
7976

80-
mkdir -p zip
81-
curl "https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_${OS}_${ARCH}.zip" > "zip/terraform.zip"
82-
83-
mkdir -p bin
84-
unzip -p "zip/terraform.zip" terraform > "$TARGET"
77+
go build -o "$TARGET" ./
8578
chmod u+x "$TARGET"
86-
rm -r zip
8779
}
8880

8981
function get_target_branch {
@@ -142,14 +134,14 @@ function main {
142134
download_equivalence_test_binary "$2" "$3" "$4" "$5"
143135

144136
;;
145-
download_terraform_binary)
146-
if [ "${#@}" != 5 ]; then
137+
build_terraform_binary)
138+
if [ "${#@}" != 2 ]; then
147139
echo "invalid number of arguments"
148140
usage
149141
exit 1
150142
fi
151143

152-
download_terraform_binary "$2" "$3" "$4" "$5"
144+
build_terraform_binary "$2"
153145

154146
;;
155147
*)

‎.github/workflows/crt-hook-equivalence-tests.yml

Lines changed: 0 additions & 45 deletions
This file was deleted.

‎.github/workflows/equivalence-test-diff.yml

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: equivalence-test-diff
2+
3+
on:
4+
pull_request:
5+
types:
6+
- synchronize
7+
- ready_for_review
8+
- reopened
9+
- synchronize
10+
11+
permissions:
12+
contents: read
13+
pull-requests: write
14+
15+
jobs:
16+
equivalence-test-diff:
17+
name: "Equivalence Test Diff"
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Fetch source code
22+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
23+
24+
- name: Determine Go version
25+
id: go
26+
uses: ./.github/actions/go-version
27+
28+
- name: Install Go toolchain
29+
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
30+
with:
31+
go-version: ${{ steps.go.outputs.version }}
32+
cache-dependency-path: go.sum
33+
34+
- name: Download testing framework
35+
shell: bash
36+
run: |
37+
./.github/scripts/equivalence-test.sh download_equivalence_test_binary \
38+
0.3.0 \
39+
./bin/equivalence-tests \
40+
linux \
41+
amd64
42+
43+
- name: Build terraform
44+
shell: bash
45+
run: ./.github/scripts/equivalence-test.sh build_terraform_binary ./bin/terraform
46+
47+
- name: Run equivalence tests
48+
id: equivalence-tests
49+
shell: bash {0} # we want to capture the exit code
50+
run: |
51+
./bin/equivalence-tests diff \
52+
--tests=testing/equivalence-tests/tests \
53+
--goldens=testing/equivalence-tests/outputs \
54+
--binary=$(pwd)/bin/terraform
55+
echo "exit-code=$?" >> "${GITHUB_OUTPUT}"
56+
57+
- name: Equivalence tests failed
58+
if: steps.equivalence-tests.outputs.exit-code == 1 # 1 is the exit code for failure
59+
shell: bash
60+
run: |
61+
gh pr comment ${{ github.event.pull_request.number }} \
62+
--body "The equivalence tests failed. Please investigate [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ github.job }})."
63+
exit 1 # fail the job
64+
65+
- name: Equivalence tests changed
66+
if: steps.equivalence-tests.outputs.exit-code == 2 # 2 is the exit code for changed
67+
shell: bash
68+
run: |
69+
gh pr comment ${{ github.event.pull_request.number }} \
70+
--body "The equivalence tests will be updated. Please verify the changes [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ github.job }})."

‎.github/workflows/manual-equivalence-tests.yml renamed to ‎.github/workflows/equivalence-test-manual-update.yml

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: manual-equivalence-tests
1+
name: equivalence-tests-manual
22

33
on:
44
workflow_dispatch:
@@ -7,9 +7,9 @@ on:
77
type: string
88
description: "Which branch should be updated?"
99
required: true
10-
terraform-version:
10+
new-branch:
1111
type: string
12-
description: "Terraform version to run against (no v prefix, eg. 1.4.4)."
12+
description: "Name of new branch to be created for the review."
1313
required: true
1414
equivalence-test-version:
1515
type: string
@@ -18,7 +18,8 @@ on:
1818
required: true
1919

2020
permissions:
21-
contents: write # We push updates to the equivalence tests back into the repository.
21+
contents: write
22+
pull-requests: write
2223

2324
jobs:
2425
run-equivalence-tests:
@@ -28,10 +29,23 @@ jobs:
2829
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
2930
with:
3031
ref: ${{ inputs.target-branch }}
32+
33+
- name: Determine Go version
34+
id: go
35+
uses: ./.github/actions/go-version
36+
37+
- name: Install Go toolchain
38+
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
39+
with:
40+
go-version: ${{ steps.go.outputs.version }}
41+
cache-dependency-path: go.sum
42+
3143
- uses: ./.github/actions/equivalence-test
3244
with:
33-
target-terraform-version: ${{ inputs.terraform-version }}
34-
target-terraform-branch: ${{ inputs.target-branch }}
3545
target-equivalence-test-version: ${{ inputs.equivalence-test-version }}
3646
target-os: linux
3747
target-arch: amd64
48+
current-branch: ${{ inputs.target-branch }}
49+
new-branch: ${{ inputs.new-branch }}
50+
reviewers: ${{ github.actor }}
51+
message: "Update equivalence test golden files."

‎.github/workflows/equivalence-test-update.yml

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: equivalence-test-update
2+
3+
on:
4+
pull_request_target:
5+
types: [ closed ]
6+
7+
permissions:
8+
contents: write
9+
pull-requests: write
10+
11+
jobs:
12+
check:
13+
name: "Should run equivalence tests?"
14+
runs-on: ubuntu-latest
15+
outputs:
16+
should_run: ${{ steps.target_branch.outputs.should_run }}
17+
steps:
18+
- name: target_branch
19+
id: target_branch
20+
run: |
21+
merged = ${{ github.event.pull_request.merged }}
22+
target_branch = ${{ github.event.pull_request.base.ref }}
23+
24+
targets_release_branch = false
25+
if [ "$target_branch" == "main" ]; then
26+
targets_release_branch = true
27+
elif [ "$target_branch" =~ ^v[0-9]+\.[0-9]+$ ]; then
28+
targets_release_branch = true
29+
fi
30+
31+
should_run=false
32+
if [ "$merged" == "true" ] && [ "$targets_release_branch" == "true" ]; then
33+
should_run=true
34+
fi
35+
36+
echo "should_run=$should_run" >> ${GITHUB_OUTPUT}
37+
run-equivalence-tests:
38+
name: "Run equivalence tests"
39+
needs:
40+
- check
41+
if: needs.check.outputs.should_run == 'true'
42+
runs-on: ubuntu-latest
43+
steps:
44+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
45+
with:
46+
ref: ${{ inputs.target-branch }}
47+
48+
- name: Determine Go version
49+
id: go
50+
uses: ./.github/actions/go-version
51+
52+
- name: Install Go toolchain
53+
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
54+
with:
55+
go-version: ${{ steps.go.outputs.version }}
56+
cache-dependency-path: go.sum
57+
58+
- uses: ./.github/actions/equivalence-test
59+
with:
60+
target-equivalence-test-version: ${{ inputs.equivalence-test-version }}
61+
target-os: linux
62+
target-arch: amd64
63+
current-branch: ${{ github.event.pull_request.base.ref }}
64+
new-branch: equivalence-testing/${{ github.event.pull_request.head.ref }}
65+
reviewers: ${{ github.event.pull_request.merged_by.login }}
66+
message: "Update equivalence test golden files after ${{ github.event.pull_request.url }}."

0 commit comments

Comments
 (0)