diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 8203e8a7..94f80b41 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -43,6 +43,7 @@ runs: - name: Build run: | + mvn --batch-mode -Dspotless.check.skip --update-snapshots dependency:go-offline mvn --batch-mode -Dspotless.check.skip --update-snapshots clean compile test-compile mvn --batch-mode -Dspotless.check.skip -DskipTests -Dmaven.test.skip=true package mvn --batch-mode -Dspotless.check.skip -DskipTests -Dmaven.test.skip=true -Dgpg.skip install @@ -58,6 +59,12 @@ runs: !target/*-javadoc.jar !target/site + - name: Upload Dependencies + uses: actions/upload-artifact@v4 + with: + name: Dependencies-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }} + path: ~/.m2/repository + branding: icon: 'check-circle' color: 'green' diff --git a/.github/actions/test-integrate/action.yml b/.github/actions/test-integrate/action.yml index e96130dc..b0ab8d4d 100644 --- a/.github/actions/test-integrate/action.yml +++ b/.github/actions/test-integrate/action.yml @@ -40,20 +40,17 @@ runs: name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }} path: . - - name: Cache Maven packages - uses: actions/cache@v4 + - name: Fetch Dependencies Artifact + uses: actions/download-artifact@v4 with: + name: Dependencies-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }} path: ~/.m2/repository - key: ${{ runner.os }}-mvn-integrate-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-${{ hashFiles('pom.xml') }} - restore-keys: | - ${{ runner.os }}-mvn-integrate-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}- - ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}- - - name: Cache Spark Binaries - uses: actions/cache@v4 + - name: Fetch Spark Binaries Artifact + uses: actions/download-artifact@v4 with: + name: Spark-Binaries-${{ inputs.spark-version }}-${{ inputs.hadoop-version }} path: ~/spark - key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }} - name: Setup JDK uses: actions/setup-java@v4 @@ -61,17 +58,6 @@ runs: java-version: ${{ inputs.java-version }} distribution: 'zulu' - - name: Setup Spark Binaries - env: - SPARK_PACKAGE: spark-${{ inputs.spark-version }}/spark-${{ inputs.spark-version }}-bin-hadoop${{ inputs.hadoop-version }}.tgz - run: | - if [[ ! -e ~/spark ]] - then - wget --progress=dot:giga "https://www.apache.org/dyn/closer.lua/spark/${SPARK_PACKAGE}?action=download" -O - | tar -xzC "${{ runner.temp }}" - archive=$(basename "${SPARK_PACKAGE}") bash -c "mv -v "${{ runner.temp }}/\${archive/%.tgz/}" ~/spark" - fi - shell: bash - - name: Parametrize id: params run: | diff --git a/.github/actions/test-python/action.yml b/.github/actions/test-python/action.yml index 753e8e90..e6f8220c 100644 --- a/.github/actions/test-python/action.yml +++ b/.github/actions/test-python/action.yml @@ -37,15 +37,11 @@ runs: name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }} path: . - - name: Cache Maven packages - uses: actions/cache@v4 + - name: Fetch Dependencies Artifact + uses: actions/download-artifact@v4 with: + name: Dependencies-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }} path: ~/.m2/repository - key: ${{ runner.os }}-mvn-python-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-${{ hashFiles('pom.xml') }} - restore-keys: | - ${{ runner.os }}-mvn-python-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}- - ${{ runner.os }}-mvn-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}- - ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}- - name: Setup JDK 11 uses: actions/setup-java@v4 diff --git a/.github/actions/test-scala/action.yml b/.github/actions/test-scala/action.yml index 7e61a216..1d6eea32 100644 --- a/.github/actions/test-scala/action.yml +++ b/.github/actions/test-scala/action.yml @@ -37,14 +37,11 @@ runs: name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }} path: . - - name: Cache Maven packages - uses: actions/cache@v4 + - name: Fetch Dependencies Artifact + uses: actions/download-artifact@v4 with: + name: Dependencies-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }} path: ~/.m2/repository - key: ${{ runner.os }}-mvn-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-${{ hashFiles('pom.xml') }} - restore-keys: | - ${{ runner.os }}-mvn-test-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}- - ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}- - name: Setup JDK uses: actions/setup-java@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58638722..a300a167 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,52 @@ jobs: git diff shell: bash + download: + name: "Spark" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # use spark versions from test-integration.yaml workflow + include: + - spark-version: '3.0.3' + hadoop-version: '2.7' + - spark-version: '3.1.3' + hadoop-version: '2.7' + - spark-version: '3.2.4' + hadoop-version: '2.7' + - spark-version: '3.3.4' + hadoop-version: '3' + - spark-version: '3.4.3' + hadoop-version: '3' + - spark-version: '3.5.3' + hadoop-version: '3' + - spark-version: '4.0.0-preview2' + hadoop-version: '3' + steps: + - name: Cache Spark Binaries + uses: actions/cache@v4 + with: + path: ~/spark + key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }} + + - name: Setup Spark Binaries + env: + SPARK_PACKAGE: spark-${{ inputs.spark-version }}/spark-${{ inputs.spark-version }}-bin-hadoop${{ inputs.hadoop-version }}.tgz + run: | + if [[ ! -e ~/spark ]] + then + wget --progress=dot:giga "https://www.apache.org/dyn/closer.lua/spark/${SPARK_PACKAGE}?action=download" -O - | tar -xzC "${{ runner.temp }}" + archive=$(basename "${SPARK_PACKAGE}") bash -c "mv -v "${{ runner.temp }}/\${archive/%.tgz/}" ~/spark" + fi + shell: bash + + - name: Upload Spark Binaries + uses: actions/upload-artifact@v4 + with: + name: Spark-Binaries-${{ inputs.spark-version }}-${{ inputs.hadoop-version }} + path: ~/spark + build: name: "Build" uses: "./.github/workflows/build.yml"