From f40e5ae4ac9884affe9d4a4cbe2dc56023035f5b Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 20 Jul 2023 16:29:09 -0500 Subject: [PATCH 1/2] build: Update to latest action versions. This updates to the following Github Actions: - actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 #v4.0.1 - actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 --- .github/workflows/go.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 94cba0e000..cca462091c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -12,15 +12,15 @@ jobs: go: ["1.19", "1.20"] steps: - name: Set up Go - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 #v3.5.0 + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: go-version: ${{ matrix.go }} - name: Check out source - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #v3.5.2 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Install Linters run: "curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.1" - name: Use test and module cache - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 with: path: | ~/.cache/go-build From a40b79393bdaedd2d9eb9f759b1ecec66bb8a7bf Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 20 Jul 2023 19:00:29 -0500 Subject: [PATCH 2/2] build: Use golangci action and make parallel. This updates the GitHub workflow to use the official golangci-lint action which has additional caching logic and also creates GitHub annotations for any issues which makes it easier to identify issues versus needing to dig through build logs. It also arranges for the linters to run as a separate job which executes in parallel to the go build and tests. Since the repository has multiple modules and golangci-lint does not support running all modules in the repository directly, this uses a separate job to resolve the modules dynamically and then feeds those into the separate linter job as a matrix of modules (work directories) so the linters run on all modules in the repo. In order to avoid running the linters as a part of the normal test job while still allowing developers to invoke all of the tests and linters via the "run_tests.sh" script, this separates the linter logic into a separate "lint.sh" script and conditionally invokes that script only when not running as a GitHub action. Finally, it pins the dependencies for both the action itself as well as the golangci-lint version by using their respective hashes. --- .github/workflows/go.yml | 38 ++++++++++++++++++++++++++++++++++---- lint.sh | 32 ++++++++++++++++++++++++++++++++ run_tests.sh | 38 ++++++-------------------------------- 3 files changed, 72 insertions(+), 36 deletions(-) create mode 100644 lint.sh diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index cca462091c..870b452343 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -4,6 +4,19 @@ permissions: contents: read jobs: + resolve-modules: + name: Resolve Go modules + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Check out source + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - id: set-matrix + run: | + echo "Resolving modules in $(pwd)" && \ + MODPATHS=$(find . -mindepth 2 -type f -name go.mod -printf '{"workdir":"%h"},') && \ + echo "matrix={\"include\":[${MODPATHS%,}]}" >> $GITHUB_OUTPUT build: name: Go CI runs-on: ubuntu-latest @@ -11,14 +24,12 @@ jobs: matrix: go: ["1.19", "1.20"] steps: + - name: Check out source + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Set up Go uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: go-version: ${{ matrix.go }} - - name: Check out source - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - name: Install Linters - run: "curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.1" - name: Use test and module cache uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 with: @@ -35,3 +46,22 @@ jobs: - name: Test run: | sh ./run_tests.sh + lint: + name: Lint + needs: resolve-modules + runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJson(needs.resolve-modules.outputs.matrix ) }} + steps: + - name: Check out source + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - name: Set up Go + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version: "1.20" + - name: golangci-lint + uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299 #v3.6.0 + with: + install-mode: "goinstall" + version: 2dcd82f331c9e834f283075b23ef289435be9354 # v1.53.3 + working-directory: ${{ matrix.workdir }} diff --git a/lint.sh b/lint.sh new file mode 100644 index 0000000000..ed1a1b0154 --- /dev/null +++ b/lint.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -ex + +# The script uses golangci-lint (github.com/golangci/golangci-lint) to run all +# linters defined by the configuration in .golangci.yml on every module in the +# repository. + +go version + +# loop all modules +ROOTPKG=$(go list) +ROOTPKGPATTERN=$(echo $ROOTPKG | sed 's,\\,\\\\,g' | sed 's,/,\\/,g') +MODPATHS=$(go list -m all | grep "^$ROOTPKGPATTERN" | cut -d' ' -f1) +for module in $MODPATHS; do + echo "==> lint ${module}" + + # determine module directory + MODNAME=$(echo $module | sed -E -e "s/^$ROOTPKGPATTERN//" \ + -e 's,^/,,' -e 's,/v[0-9]+$,,') + if [ -z "$MODNAME" ]; then + MODNAME=. + fi + + # run commands in the module directory as a subshell + ( + cd $MODNAME + + # run linters + golangci-lint run + ) +done diff --git a/run_tests.sh b/run_tests.sh index 93ad77c2b9..6e3f3e8b9d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -2,17 +2,11 @@ set -ex -# The script does automatic checking on a Go package and its sub-packages, -# including: -# 1. gofmt (https://golang.org/cmd/gofmt/) -# 2. gosimple (https://github.com/dominikh/go-simple) -# 3. unconvert (https://github.com/mdempsky/unconvert) -# 4. ineffassign (https://github.com/gordonklaus/ineffassign) -# 5. go vet (https://golang.org/cmd/vet) -# 6. misspell (https://github.com/client9/misspell) - -# golangci-lint (github.com/golangci/golangci-lint) is used to run each -# static checker. +# This script runs the tests for all packages in all Go modules in the +# repository. +# +# It will also run the linters for all Go modules in the repository when not +# running as a GitHub action. go version @@ -21,27 +15,7 @@ echo "==> test all modules" ROOTPKG=$(go list) go test -short -tags rpctest $ROOTPKG/... -# loop all modules -ROOTPKGPATTERN=$(echo $ROOTPKG | sed 's,\\,\\\\,g' | sed 's,/,\\/,g') -MODPATHS=$(go list -m all | grep "^$ROOTPKGPATTERN" | cut -d' ' -f1) -for module in $MODPATHS; do - echo "==> lint ${module}" - - # determine module directory - MODNAME=$(echo $module | sed -E -e "s/^$ROOTPKGPATTERN//" \ - -e 's,^/,,' -e 's,/v[0-9]+$,,') - if [ -z "$MODNAME" ]; then - MODNAME=. - fi - - # run commands in the module directory as a subshell - ( - cd $MODNAME - - # run linters - golangci-lint run - ) -done +[ -z "$GITHUB_ACTIONS" ] && ./lint.sh echo "------------------------------------------" echo "Tests completed successfully!"