From 1e569c856d174328ac28b0b91926508b2e83aecb Mon Sep 17 00:00:00 2001
From: Enrico Minack <github@enrico.minack.dev>
Date: Mon, 4 Nov 2024 15:07:36 +0100
Subject: [PATCH] Only save caches on master (and not schedule event)

---
 .github/actions/build-whl/action.yml    | 28 ++++++++++-----
 .github/actions/build/action.yml        | 46 ++++++++++++++++++++++---
 .github/actions/check-compat/action.yml | 19 ++++++----
 .github/actions/test-jvm/action.yml     | 18 ++++++----
 .github/actions/test-python/action.yml  | 39 +++++++++++++++------
 5 files changed, 114 insertions(+), 36 deletions(-)

diff --git a/.github/actions/build-whl/action.yml b/.github/actions/build-whl/action.yml
index c0a7f309..acd97866 100644
--- a/.github/actions/build-whl/action.yml
+++ b/.github/actions/build-whl/action.yml
@@ -34,20 +34,25 @@ runs:
       name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
       path: .
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
-  - name: Cache Pip packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Pip packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.cache/pip
-      key: ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ inputs.spark-version }}
+      key: ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
+      restore-keys:
+        ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
+        ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-
 
   - name: Setup Python
     uses: actions/setup-python@v5
@@ -72,6 +77,13 @@ runs:
       python test-release.py
     shell: bash
 
+  - name: Save Pip packages cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/.cache/pip
+      key: ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}-${{ github.run_id }}}
+
   - name: Upload whl
     uses: actions/upload-artifact@v4
     with:
diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
index 187d018d..50c4006a 100644
--- a/.github/actions/build/action.yml
+++ b/.github/actions/build/action.yml
@@ -28,13 +28,15 @@ runs:
       git diff
     shell: bash
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
   - name: Setup JDK ${{ inputs.java-compat-version }}
     uses: actions/setup-java@v4
@@ -51,6 +53,42 @@ runs:
       mvn --batch-mode install -Dspotless.check.skip -DskipTests -Dmaven.test.skip=true -Dgpg.skip
     shell: bash
 
+  - name: Save Maven packages cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/.m2/repository
+      key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}-${{ github.run_id }}}
+
+  # merely populate the cache for test-jvm and test-python actions
+  - name: Restore Spark Binaries cache
+    if: github.event_name != 'schedule' && ! contains(inputs.spark-version, '-SNAPSHOT')
+    uses: actions/cache/restore@v4
+    with:
+      path: ~/spark
+      key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+      restore-keys:
+        ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+
+  - name: Setup Spark Binaries
+    if: ( ! contains(inputs.spark-version, '-SNAPSHOT') )
+    env:
+      SPARK_PACKAGE: spark-${{ inputs.spark-version }}/spark-${{ inputs.spark-version }}-bin-hadoop${{ inputs.hadoop-version }}${{ inputs.scala-compat-version == '2.13' && '-scala2.13' || '' }}.tgz
+    run: |
+      if [[ ! -e ~/spark ]]
+      then
+        wget --progress=dot:giga "https://www.apache.org/dyn/closer.lua/spark/${SPARK_PACKAGE}?action=download" -O - | tar -xzC "${{ runner.temp }}"
+        archive=$(basename "${SPARK_PACKAGE}") bash -c "mv -v "${{ runner.temp }}/\${archive/%.tgz/}" ~/spark"
+      fi
+    shell: bash
+
+  - name: Save Spark Binaries cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/spark
+      key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-${{ github.run_id }}}
+
   - name: Upload Binaries
     uses: actions/upload-artifact@v4
     with:
diff --git a/.github/actions/check-compat/action.yml b/.github/actions/check-compat/action.yml
index 75d778b4..3e8bf1bd 100644
--- a/.github/actions/check-compat/action.yml
+++ b/.github/actions/check-compat/action.yml
@@ -34,15 +34,15 @@ runs:
       name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
       path: .
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: |
-        ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}
-        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
   - name: Setup JDK 1.8
     uses: actions/setup-java@v4
@@ -77,6 +77,13 @@ runs:
       japi-compliance-checker ~/.m2/repository/uk/co/gresearch/spark/spark-extension_${{ inputs.scala-compat-version }}/${{ inputs.package-version }}-${{ inputs.spark-compat-version }}/spark-extension_${{ inputs.scala-compat-version }}-${{ inputs.package-version }}-${{ inputs.spark-compat-version }}.jar target/spark-extension*.jar
     shell: bash
 
+  - name: Save Maven packages cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/.m2/repository
+      key: ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}-${{ github.run_id }}}
+
   - name: Upload Report
     uses: actions/upload-artifact@v4
     if: always() && steps.exists.outcome == 'success'
diff --git a/.github/actions/test-jvm/action.yml b/.github/actions/test-jvm/action.yml
index 66466e7c..b0a6240b 100644
--- a/.github/actions/test-jvm/action.yml
+++ b/.github/actions/test-jvm/action.yml
@@ -37,12 +37,14 @@ runs:
       name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
       path: .
 
-  - name: Cache Spark Binaries
-    uses: actions/cache@v4
-    if: ( ! contains(inputs.spark-version, '-SNAPSHOT') )
+  - name: Restore Spark Binaries cache
+    if: github.event_name != 'schedule' && ! contains(inputs.spark-version, '-SNAPSHOT')
+    uses: actions/cache/restore@v4
     with:
       path: ~/spark
       key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+      restore-keys:
+        ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
 
   - name: Setup Spark Binaries
     if: ( ! contains(inputs.spark-version, '-SNAPSHOT') )
@@ -57,13 +59,15 @@ runs:
       echo "SPARK_HOME=$(cd ~/spark; pwd)" >> $GITHUB_ENV
     shell: bash
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
   - name: Setup JDK ${{ inputs.java-compat-version }}
     uses: actions/setup-java@v4
diff --git a/.github/actions/test-python/action.yml b/.github/actions/test-python/action.yml
index 1a1a4c36..2701044d 100644
--- a/.github/actions/test-python/action.yml
+++ b/.github/actions/test-python/action.yml
@@ -27,6 +27,8 @@ inputs:
 
 runs:
   using: 'composite'
+  env:
+    DEFAULT_BRANCH_REF: ${{ format('refs/heads/{0}', github.event.repository.default_branch) }}
   steps:
   - name: Set versions in pom.xml
     run: |
@@ -43,12 +45,14 @@ runs:
       name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
       path: .
 
-  - name: Cache Spark Binaries
-    uses: actions/cache@v4
-    if: inputs.scala-compat-version == '2.12' && ! contains(inputs.spark-version, '-SNAPSHOT')
+  - name: Restore Spark Binaries cache
+    if: github.event_name != 'schedule' && inputs.scala-compat-version == '2.12' && ! contains(inputs.spark-version, '-SNAPSHOT')
+    uses: actions/cache/restore@v4
     with:
       path: ~/spark
       key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+      restore-keys:
+        ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
 
   - name: Setup Spark Binaries
     if: inputs.scala-compat-version == '2.12' && ! contains(inputs.spark-version, '-SNAPSHOT')
@@ -63,13 +67,15 @@ runs:
       echo "SPARK_BIN_HOME=$(cd ~/spark; pwd)" >> $GITHUB_ENV
     shell: bash
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
   - name: Setup JDK 1.8
     uses: actions/setup-java@v4
@@ -77,13 +83,17 @@ runs:
       java-version: '8'
       distribution: 'zulu'
 
-  - name: Cache Pip packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Pip packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.cache/pip
       key: ${{ runner.os }}-pip-test-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
-      restore-keys: ${{ runner.os }}-pip-test-${{ inputs.python-version }}-
+      restore-keys:
+        ${{ runner.os }}-pip-test-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
+        ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
+        ${{ runner.os }}-pip-test-${{ inputs.python-version }}-
+        ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-
 
   - name: Setup Python
     uses: actions/setup-python@v5
@@ -181,6 +191,13 @@ runs:
       $SPARK_HOME/bin/spark-shell --packages uk.co.gresearch.spark:spark-extension_${{ inputs.scala-compat-version }}:$SPARK_EXTENSION_VERSION < test-release.scala
     shell: bash
 
+  - name: Save Pip packages cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/.cache/pip
+      key: ${{ runner.os }}-pip-test-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}-${{ github.run_id }}}
+
   - name: Upload Test Results
     if: always()
     uses: actions/upload-artifact@v4