chore(ci): re-use cache key from build workflow (#11299)

This makes build cache key generation reusable across different workflows. The test and release workflows now use the same "recipe" for cache key generation--no more copy/paste.
Kong · Jul 27, 2023 · 68b7b6f · 68b7b6f · khcp-gha-bot · Jul 27, 2023
1 parent 91dcd61
commit 68b7b6f
Show file tree

Hide file tree

Showing 5 changed files with 88 additions and 28 deletions.
diff --git a/.github/actions/build-cache-key/action.yml b/.github/actions/build-cache-key/action.yml
@@ -0,0 +1,45 @@
+name: Build Cache Key
+
+description: >
+  Generates a cache key suitable for save/restore of Kong builds.
+
+inputs:
+  prefix:
+    description: 'String prefix applied to the build cache key'
+    required: false
+    default: 'build'
+
+outputs:
+  cache-key:
+    description: 'The generated cache key'
+    value: ${{ steps.cache-key.outputs.CACHE_KEY }}
+
+runs:
+  using: composite
+  steps:
+    - name: Generate cache key
+      id: cache-key
+      shell: bash
+      run: |
+        # please keep these sorted
+        FILE_HASHES=(
+          ${{ hashFiles('.bazelignore') }}
+          ${{ hashFiles('.bazelrc') }}
+          ${{ hashFiles('.bazelversion') }}
+          ${{ hashFiles('.github/actions/build-cache-key/**') }}
+          ${{ hashFiles('.github/workflows/build.yml') }}
+          ${{ hashFiles('.requirements') }}
+          ${{ hashFiles('BUILD.bazel') }}
+          ${{ hashFiles('WORKSPACE') }}
+          ${{ hashFiles('bin/kong') }}
+          ${{ hashFiles('bin/kong-health') }}
+          ${{ hashFiles('build/**') }}
+          ${{ hashFiles('kong-*.rockspec') }}
+          ${{ hashFiles('kong.conf.default') }}
+          ${{ hashFiles('kong/**') }}
+        )
+
+        HASH=$(sha256sum - <<< "${FILE_HASHES[*]}" | awk '{print $1}' )
+        CACHE_KEY=${{ inputs.prefix }}::${HASH}
+        echo "cache-key: ${CACHE_KEY}"
+        echo "CACHE_KEY=${CACHE_KEY}" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -5,6 +5,10 @@ on:
       relative-build-root:
         required: true
         type: string
+    outputs:
+      cache-key:
+        description: 'Computed cache key, used for restoring cache in other workflows'
+        value: ${{ jobs.build.outputs.cache-key }}
 
 env:
   BUILD_ROOT: ${{ github.workspace }}/${{ inputs.relative-build-root }}
@@ -14,6 +18,9 @@ jobs:
     name: Build dependencies
     runs-on: ubuntu-22.04
 
+    outputs:
+      cache-key: ${{ steps.cache-key.outputs.cache-key }}
+
     steps:
     - name: Checkout Kong source code
       uses: actions/checkout@v3
@@ -24,13 +31,16 @@ jobs:
     - name: Build WASM Test Filters
       uses: ./.github/actions/build-wasm-test-filters
 
+    - name: Generate cache key
+      id: cache-key
+      uses: ./.github/actions/build-cache-key
+
     - name: Lookup build cache
       id: cache-deps
       uses: actions/cache@v3
       with:
-        path: |
-          ${{ env.BUILD_ROOT }}
-        key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }}
+        path: ${{ env.BUILD_ROOT }}
+        key: ${{ steps.cache-key.outputs.cache-key }}
 
     - name: Install packages
       if: steps.cache-deps.outputs.cache-hit != 'true'

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -62,9 +62,8 @@ jobs:
       id: cache-deps
       uses: actions/cache@v3
       with:
-        path: |
-          ${{ env.BUILD_ROOT }}
-        key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }}
+        path: ${{ env.BUILD_ROOT }}
+        key: ${{ needs.build.outputs.cache-key }}
 
     - name: Check test-helpers doc generation
       run: |
@@ -162,10 +161,8 @@ jobs:
       id: cache-deps
       uses: actions/cache@v3
       with:
-        path: |
-          ${{ env.BUILD_ROOT }}
-
-        key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }}
+        path: ${{ env.BUILD_ROOT }}
+        key: ${{ needs.build.outputs.cache-key }}
 
     - name: Build WASM Test Filters
       uses: ./.github/actions/build-wasm-test-filters
@@ -267,10 +264,8 @@ jobs:
       id: cache-deps
       uses: actions/cache@v3
       with:
-        path: |
-          ${{ env.BUILD_ROOT }}
-
-        key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }}
+        path: ${{ env.BUILD_ROOT }}
+        key: ${{ needs.build.outputs.cache-key }}
 
     - name: Build WASM Test Filters
       uses: ./.github/actions/build-wasm-test-filters
@@ -330,9 +325,8 @@ jobs:
       id: cache-deps
       uses: actions/cache@v3
       with:
-        path: |
-          ${{ env.BUILD_ROOT }}
-        key: ${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }}
+        path: ${{ env.BUILD_ROOT }}
+        key: ${{ needs.build.outputs.cache-key }}
 
     - name: Install Test::Nginx
       run: |

diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml
@@ -29,17 +29,25 @@ jobs:
         (startsWith(github.event.comment.body, '/perf') || startsWith(github.event.comment.body, '/flamegraph'))
       )
 
+    outputs:
+      cache-key: ${{ steps.cache-key.outputs.cache-key }}
+
     steps:
     - name: Checkout Kong source code
       uses: actions/checkout@v3
 
+    - name: Generate cache key
+      id: cache-key
+      uses: ./.github/actions/build-cache-key
+      with:
+        prefix: perf
+
     - name: Lookup build cache
       id: cache-deps
       uses: actions/cache@v3
       with:
-        path: |
-          ${{ env.BUILD_ROOT }}
-        key: perf-${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }}
+        path: ${{ env.BUILD_ROOT }}
+        key: ${{ steps.cache-key.outputs.cache-key }}
 
     - name: Install packages
       if: steps.cache-deps.outputs.cache-hit != 'true'
@@ -72,8 +80,6 @@ jobs:
       run: |
         make install-dev-rocks
 
-  # the above should be same as build_and_test.yml expect that perf.yml is used in cache_key
-
   perf:
     name: RPS, latency and flamegraphs
     runs-on: ubuntu-22.04
@@ -116,9 +122,8 @@ jobs:
       if: env.GHA_CACHE == 'true'
       uses: actions/cache@v3
       with:
-        path: |
-          ${{ env.BUILD_ROOT }}
-        key: perf-${{ hashFiles('.requirements', 'kong-*.rockspec', '.bazelversion', '.bazelrc', 'build/**', 'BUILD.bazel', 'WORKSPACE', '.github/workflows/build_and_test.yml') }}
+        path: ${{ env.BUILD_ROOT }}
+        key: ${{ needs.build-packages.outputs.cache-key }}
 
     - name: Install performance test Dependencies
       run: |

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -167,14 +167,20 @@ jobs:
     - name: Swap git with https
       run: git config --global url."https://github".insteadOf git://github
 
+    - name: Generate build cache key
+      id: cache-key
+      if: env.GHA_CACHE == 'true'
+      uses: ./.github/actions/build-cache-key
+      with:
+        prefix: ${{ matrix.label }}-build
+
     - name: Cache Packages
       id: cache-deps
       if: env.GHA_CACHE == 'true'
       uses: actions/cache@v3
       with:
-        path: |
-          bazel-bin/pkg
-        key: ${{ matrix.label }}-build-${{ hashFiles('bin/kong', '.requirements', 'kong-*.rockspec', 'kong/**/*.lua', '**/*.bzl', '**/*.bazel') }}
+        path: bazel-bin/pkg
+        key: ${{ steps.cache-key.outputs.cache-key }}
 
     - name: Set .requirements into environment variables
       run: |