From 68b7b6f9cb9eb52af0ed558b2952a3239926b5b7 Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Thu, 27 Jul 2023 11:16:09 -0700 Subject: [PATCH] chore(ci): re-use cache key from build workflow (#11299) This makes build cache key generation reusable across different workflows. The test and release workflows now use the same "recipe" for cache key generation--no more copy/paste. --- .github/actions/build-cache-key/action.yml | 45 ++++++++++++++++++++++ .github/workflows/build.yml | 16 ++++++-- .github/workflows/build_and_test.yml | 22 ++++------- .github/workflows/perf.yml | 21 ++++++---- .github/workflows/release.yml | 12 ++++-- 5 files changed, 88 insertions(+), 28 deletions(-) create mode 100644 .github/actions/build-cache-key/action.yml diff --git a/.github/actions/build-cache-key/action.yml b/.github/actions/build-cache-key/action.yml new file mode 100644 index 000000000000..7b91acb1210e --- /dev/null +++ b/.github/actions/build-cache-key/action.yml @@ -0,0 +1,45 @@ +name: Build Cache Key + +description: > + Generates a cache key suitable for save/restore of Kong builds. + +inputs: + prefix: + description: 'String prefix applied to the build cache key' + required: false + default: 'build' + +outputs: + cache-key: + description: 'The generated cache key' + value: ${{ steps.cache-key.outputs.CACHE_KEY }} + +runs: + using: composite + steps: + - name: Generate cache key + id: cache-key + shell: bash + run: | + # please keep these sorted + FILE_HASHES=( + ${{ hashFiles('.bazelignore') }} + ${{ hashFiles('.bazelrc') }} + ${{ hashFiles('.bazelversion') }} + ${{ hashFiles('.github/actions/build-cache-key/**') }} + ${{ hashFiles('.github/workflows/build.yml') }} + ${{ hashFiles('.requirements') }} + ${{ hashFiles('BUILD.bazel') }} + ${{ hashFiles('WORKSPACE') }} + ${{ hashFiles('bin/kong') }} + ${{ hashFiles('bin/kong-health') }} + ${{ hashFiles('build/**') }} + ${{ hashFiles('kong-*.rockspec') }} + ${{ hashFiles('kong.conf.default') }} + ${{ hashFiles('kong/**') }} + ) + + HASH=$(sha256sum - <<< "${FILE_HASHES[*]}" | awk '{print $1}' ) + CACHE_KEY=${{ inputs.prefix }}::${HASH} + echo "cache-key: ${CACHE_KEY}" + echo "CACHE_KEY=${CACHE_KEY}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2ead81f4871..54ed720cd54a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,6 +5,10 @@ on: relative-build-root: required: true type: string + outputs: + cache-key: + description: 'Computed cache key, used for restoring cache in other workflows' + value: ${{ jobs.build.outputs.cache-key }} env: BUILD_ROOT: ${{ github.workspace }}/${{ inputs.relative-build-root }} @@ -14,6 +18,9 @@ jobs: name: Build dependencies runs-on: ubuntu-22.04 + outputs: + cache-key: ${{ steps.cache-key.outputs.cache-key }} + steps: - name: Checkout Kong source code uses: actions/checkout@v3 @@ -24,13 +31,16 @@ jobs: - name: Build WASM Test Filters uses: ./.github/actions/build-wasm-test-filters + - name: Generate cache key + id: cache-key + uses: ./.github/actions/build-cache-key + - name: Lookup build cache id: cache-deps uses: actions/cache@v3 with: - path: | - ${{ env.BUILD_ROOT }} - key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }} + path: ${{ env.BUILD_ROOT }} + key: ${{ steps.cache-key.outputs.cache-key }} - name: Install packages if: steps.cache-deps.outputs.cache-hit != 'true' diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d020ef7779bf..97805dc62d30 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -62,9 +62,8 @@ jobs: id: cache-deps uses: actions/cache@v3 with: - path: | - ${{ env.BUILD_ROOT }} - key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }} + path: ${{ env.BUILD_ROOT }} + key: ${{ needs.build.outputs.cache-key }} - name: Check test-helpers doc generation run: | @@ -162,10 +161,8 @@ jobs: id: cache-deps uses: actions/cache@v3 with: - path: | - ${{ env.BUILD_ROOT }} - - key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }} + path: ${{ env.BUILD_ROOT }} + key: ${{ needs.build.outputs.cache-key }} - name: Build WASM Test Filters uses: ./.github/actions/build-wasm-test-filters @@ -267,10 +264,8 @@ jobs: id: cache-deps uses: actions/cache@v3 with: - path: | - ${{ env.BUILD_ROOT }} - - key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }} + path: ${{ env.BUILD_ROOT }} + key: ${{ needs.build.outputs.cache-key }} - name: Build WASM Test Filters uses: ./.github/actions/build-wasm-test-filters @@ -330,9 +325,8 @@ jobs: id: cache-deps uses: actions/cache@v3 with: - path: | - ${{ env.BUILD_ROOT }} - key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }} + path: ${{ env.BUILD_ROOT }} + key: ${{ needs.build.outputs.cache-key }} - name: Install Test::Nginx run: | diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 9eecbef8d155..f9ac14d4ab54 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -29,17 +29,25 @@ jobs: (startsWith(github.event.comment.body, '/perf') || startsWith(github.event.comment.body, '/flamegraph')) ) + outputs: + cache-key: ${{ steps.cache-key.outputs.cache-key }} + steps: - name: Checkout Kong source code uses: actions/checkout@v3 + - name: Generate cache key + id: cache-key + uses: ./.github/actions/build-cache-key + with: + prefix: perf + - name: Lookup build cache id: cache-deps uses: actions/cache@v3 with: - path: | - ${{ env.BUILD_ROOT }} - key: perf-${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }} + path: ${{ env.BUILD_ROOT }} + key: ${{ steps.cache-key.outputs.cache-key }} - name: Install packages if: steps.cache-deps.outputs.cache-hit != 'true' @@ -72,8 +80,6 @@ jobs: run: | make install-dev-rocks - # the above should be same as build_and_test.yml expect that perf.yml is used in cache_key - perf: name: RPS, latency and flamegraphs runs-on: ubuntu-22.04 @@ -116,9 +122,8 @@ jobs: if: env.GHA_CACHE == 'true' uses: actions/cache@v3 with: - path: | - ${{ env.BUILD_ROOT }} - key: perf-${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }} + path: ${{ env.BUILD_ROOT }} + key: ${{ needs.build-packages.outputs.cache-key }} - name: Install performance test Dependencies run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e02596f9207c..f269657a3fb3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -167,14 +167,20 @@ jobs: - name: Swap git with https run: git config --global url."https://github".insteadOf git://github + - name: Generate build cache key + id: cache-key + if: env.GHA_CACHE == 'true' + uses: ./.github/actions/build-cache-key + with: + prefix: ${{ matrix.label }}-build + - name: Cache Packages id: cache-deps if: env.GHA_CACHE == 'true' uses: actions/cache@v3 with: - path: | - bazel-bin/pkg - key: ${{ matrix.label }}-build-${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', 'kong/**/*.lua', '**/*.bzl', '**/*.bazel') }} + path: bazel-bin/pkg + key: ${{ steps.cache-key.outputs.cache-key }} - name: Set .requirements into environment variables run: |