Use dependencies and spark binaries from artifact rather than cache

G-Research · Dec 14, 2024 · 6c37a56 · 6c37a56
1 parent b841496
commit 6c37a56
Show file tree

Hide file tree

Showing 5 changed files with 65 additions and 33 deletions.
diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
@@ -43,6 +43,7 @@ runs:
 
     - name: Build
       run: |
+        mvn --batch-mode -Dspotless.check.skip --update-snapshots dependency:go-offline
         mvn --batch-mode -Dspotless.check.skip --update-snapshots clean compile test-compile
         mvn --batch-mode -Dspotless.check.skip -DskipTests -Dmaven.test.skip=true package
         mvn --batch-mode -Dspotless.check.skip -DskipTests -Dmaven.test.skip=true -Dgpg.skip install
@@ -58,6 +59,12 @@ runs:
           !target/*-javadoc.jar
           !target/site
 
+    - name: Upload Dependencies
+      uses: actions/upload-artifact@v4
+      with:
+        name: Dependencies-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
+        path: ~/.m2/repository
+
 branding:
   icon: 'check-circle'
   color: 'green'
diff --git a/.github/actions/test-integrate/action.yml b/.github/actions/test-integrate/action.yml
@@ -40,38 +40,24 @@ runs:
         name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
         path: .
 
-    - name: Cache Maven packages
-      uses: actions/cache@v4
+    - name: Fetch Dependencies Artifact
+      uses: actions/download-artifact@v4
       with:
+        name: Dependencies-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
         path: ~/.m2/repository
-        key: ${{ runner.os }}-mvn-integrate-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-${{ hashFiles('pom.xml') }}
-        restore-keys: |
-          ${{ runner.os }}-mvn-integrate-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-
-          ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-
 
-    - name: Cache Spark Binaries
-      uses: actions/cache@v4
+    - name: Fetch Spark Binaries Artifact
+      uses: actions/download-artifact@v4
       with:
+        name: Spark-Binaries-${{ inputs.spark-version }}-${{ inputs.hadoop-version }}
         path: ~/spark
-        key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
 
     - name: Setup JDK
       uses: actions/setup-java@v4
       with:
         java-version: ${{ inputs.java-version }}
         distribution: 'zulu'
 
-    - name: Setup Spark Binaries
-      env:
-        SPARK_PACKAGE: spark-${{ inputs.spark-version }}/spark-${{ inputs.spark-version }}-bin-hadoop${{ inputs.hadoop-version }}.tgz
-      run: |
-        if [[ ! -e ~/spark ]]
-        then
-          wget --progress=dot:giga "https://www.apache.org/dyn/closer.lua/spark/${SPARK_PACKAGE}?action=download" -O - | tar -xzC "${{ runner.temp }}"
-          archive=$(basename "${SPARK_PACKAGE}") bash -c "mv -v "${{ runner.temp }}/\${archive/%.tgz/}" ~/spark"
-        fi
-      shell: bash
-
     - name: Parametrize
       id: params
       run: |

diff --git a/.github/actions/test-python/action.yml b/.github/actions/test-python/action.yml
@@ -37,15 +37,11 @@ runs:
         name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
         path: .
 
-    - name: Cache Maven packages
-      uses: actions/cache@v4
+    - name: Fetch Dependencies Artifact
+      uses: actions/download-artifact@v4
       with:
+        name: Dependencies-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
         path: ~/.m2/repository
-        key: ${{ runner.os }}-mvn-python-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-${{ hashFiles('pom.xml') }}
-        restore-keys: |
-          ${{ runner.os }}-mvn-python-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-
-          ${{ runner.os }}-mvn-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-
-          ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-
 
     - name: Setup JDK 11
       uses: actions/setup-java@v4

diff --git a/.github/actions/test-scala/action.yml b/.github/actions/test-scala/action.yml
@@ -37,14 +37,11 @@ runs:
         name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
         path: .
 
-    - name: Cache Maven packages
-      uses: actions/cache@v4
+    - name: Fetch Dependencies Artifact
+      uses: actions/download-artifact@v4
       with:
+        name: Dependencies-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
         path: ~/.m2/repository
-        key: ${{ runner.os }}-mvn-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-${{ hashFiles('pom.xml') }}
-        restore-keys: |
-          ${{ runner.os }}-mvn-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-
-          ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-
 
     - name: Setup JDK
       uses: actions/setup-java@v4

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -54,6 +54,52 @@ jobs:
           git diff
         shell: bash
 
+  download:
+    name: "Spark"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # use spark versions from test-integration.yaml workflow
+        include:
+          - spark-version: '3.0.3'
+            hadoop-version: '2.7'
+          - spark-version: '3.1.3'
+            hadoop-version: '2.7'
+          - spark-version: '3.2.4'
+            hadoop-version: '2.7'
+          - spark-version: '3.3.4'
+            hadoop-version: '3'
+          - spark-version: '3.4.3'
+            hadoop-version: '3'
+          - spark-version: '3.5.3'
+            hadoop-version: '3'
+          - spark-version: '4.0.0-preview2'
+            hadoop-version: '3'
+    steps:
+      - name: Cache Spark Binaries
+        uses: actions/cache@v4
+        with:
+          path: ~/spark
+          key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+
+      - name: Setup Spark Binaries
+        env:
+          SPARK_PACKAGE: spark-${{ inputs.spark-version }}/spark-${{ inputs.spark-version }}-bin-hadoop${{ inputs.hadoop-version }}.tgz
+        run: |
+          if [[ ! -e ~/spark ]]
+          then
+            wget --progress=dot:giga "https://www.apache.org/dyn/closer.lua/spark/${SPARK_PACKAGE}?action=download" -O - | tar -xzC "${{ runner.temp }}"
+            archive=$(basename "${SPARK_PACKAGE}") bash -c "mv -v "${{ runner.temp }}/\${archive/%.tgz/}" ~/spark"
+          fi
+        shell: bash
+
+    - name: Upload Spark Binaries
+      uses: actions/upload-artifact@v4
+      with:
+        name: Spark-Binaries-${{ inputs.spark-version }}-${{ inputs.hadoop-version }}
+        path: ~/spark
+
   build:
     name: "Build"
     uses: "./.github/workflows/build.yml"