From 4d35579f294814e6ad57cf194ea13d0a0e1fdb28 Mon Sep 17 00:00:00 2001
From: Enrico Minack <github@enrico.minack.dev>
Date: Mon, 4 Nov 2024 12:01:28 +0100
Subject: [PATCH] Upgrade to Spark 3.4.4 + 3.5.3 and upgrade Pyhon versions
 (#259)

---
 .github/workflows/build-jvm.yml       |  8 ++--
 .github/workflows/build-python.yml    |  4 +-
 .github/workflows/build-snapshots.yml |  8 ++--
 .github/workflows/test-jvm.yml        | 55 +++------------------------
 .github/workflows/test-python.yml     |  4 +-
 .github/workflows/test-snapshots.yml  |  8 ++--
 pom.xml                               | 28 ++++++++++++++
 7 files changed, 49 insertions(+), 66 deletions(-)

diff --git a/.github/workflows/build-jvm.yml b/.github/workflows/build-jvm.yml
index bd44f507..982d2d45 100644
--- a/.github/workflows/build-jvm.yml
+++ b/.github/workflows/build-jvm.yml
@@ -31,11 +31,11 @@ jobs:
           - spark-compat-version: '3.4'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
-            spark-version: '3.4.2'
+            spark-version: '3.4.4'
           - spark-compat-version: '3.5'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
-            spark-version: '3.5.1'
+            spark-version: '3.5.3'
 
           - spark-compat-version: '3.2'
             spark-version: '3.2.4'
@@ -48,11 +48,11 @@ jobs:
           - spark-compat-version: '3.4'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-version: '3.4.2'
+            spark-version: '3.4.4'
           - spark-compat-version: '3.5'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-version: '3.5.1'
+            spark-version: '3.5.3'
 
     steps:
       - name: Checkout
diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml
index c4a14bfd..34836453 100644
--- a/.github/workflows/build-python.yml
+++ b/.github/workflows/build-python.yml
@@ -30,11 +30,11 @@ jobs:
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.2'
+            spark-version: '3.4.4'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.1'
+            spark-version: '3.5.3'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
 
diff --git a/.github/workflows/build-snapshots.yml b/.github/workflows/build-snapshots.yml
index 8e64ac9e..a33add85 100644
--- a/.github/workflows/build-snapshots.yml
+++ b/.github/workflows/build-snapshots.yml
@@ -23,12 +23,12 @@ jobs:
             scala-version: '2.12.15'
             java-compat-version: '8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.4-SNAPSHOT'
+            spark-version: '3.4.5-SNAPSHOT'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
             java-compat-version: '8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.3-SNAPSHOT'
+            spark-version: '3.5.4-SNAPSHOT'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
             java-compat-version: '8'
@@ -44,12 +44,12 @@ jobs:
             scala-version: '2.13.8'
             java-compat-version: '8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.4-SNAPSHOT'
+            spark-version: '3.4.5-SNAPSHOT'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
             java-compat-version: '8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.3-SNAPSHOT'
+            spark-version: '3.5.4-SNAPSHOT'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
             java-compat-version: '8'
diff --git a/.github/workflows/test-jvm.yml b/.github/workflows/test-jvm.yml
index 34caa47a..4f1825d0 100644
--- a/.github/workflows/test-jvm.yml
+++ b/.github/workflows/test-jvm.yml
@@ -12,8 +12,8 @@ jobs:
       fail-fast: false
       matrix:
         scala-compat-version: ['2.12', '2.13']
-        spark-compat-version: ['3.2', '3.3']
-        spark-patch-version: ['0', '1', '2', '3']
+        spark-compat-version: ['3.2', '3.3', '3.4', '3.5']
+        spark-patch-version: ['0', '1', '2', '3', '4']
 
         include:
           - spark-compat-version: '3.0'
@@ -30,11 +30,6 @@ jobs:
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
             hadoop-version: '2.7'
-          - spark-compat-version: '3.2'
-            scala-compat-version: '2.12'
-            scala-version: '2.12.15'
-            spark-patch-version: '4'
-            hadoop-version: '2.7'
           - spark-compat-version: '3.3'
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
@@ -52,72 +47,32 @@ jobs:
           - spark-compat-version: '3.4'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
-            spark-patch-version: '0'
-            hadoop-version: '3'
-          - spark-compat-version: '3.4'
-            scala-compat-version: '2.12'
-            scala-version: '2.12.17'
-            spark-patch-version: '1'
-            hadoop-version: '3'
-          - spark-compat-version: '3.4'
-            scala-compat-version: '2.12'
-            scala-version: '2.12.17'
-            spark-patch-version: '2'
-            hadoop-version: '3'
-          - spark-compat-version: '3.5'
-            scala-compat-version: '2.12'
-            scala-version: '2.12.18'
-            spark-patch-version: '0'
             hadoop-version: '3'
           - spark-compat-version: '3.5'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
-            spark-patch-version: '1'
             hadoop-version: '3'
 
           - spark-compat-version: '3.2'
             scala-compat-version: '2.13'
             scala-version: '2.13.5'
             hadoop-version: '3.2'
-          - spark-compat-version: '3.2'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.5'
-            spark-patch-version: '4'
-            hadoop-version: '3.2'
-          - spark-compat-version: '3.3'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.8'
-            hadoop-version: '3'
           - spark-compat-version: '3.3'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-patch-version: '4'
-            hadoop-version: '3'
-          - spark-compat-version: '3.4'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.8'
-            spark-patch-version: '0'
-            hadoop-version: '3'
-          - spark-compat-version: '3.4'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.8'
-            spark-patch-version: '1'
             hadoop-version: '3'
           - spark-compat-version: '3.4'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-patch-version: '2'
             hadoop-version: '3'
           - spark-compat-version: '3.5'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-patch-version: '0'
             hadoop-version: '3'
+
+        exclude:
           - spark-compat-version: '3.5'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.8'
-            spark-patch-version: '1'
-            hadoop-version: '3'
+            spark-patch-version: '4'
 
     steps:
       - name: Checkout
diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
index a83d5599..c36cc02b 100644
--- a/.github/workflows/test-python.yml
+++ b/.github/workflows/test-python.yml
@@ -40,12 +40,12 @@ jobs:
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.2'
+            spark-version: '3.4.4'
             hadoop-version: '3'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.1'
+            spark-version: '3.5.3'
             hadoop-version: '3'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
diff --git a/.github/workflows/test-snapshots.yml b/.github/workflows/test-snapshots.yml
index 8f13f1a5..b32ee8bb 100644
--- a/.github/workflows/test-snapshots.yml
+++ b/.github/workflows/test-snapshots.yml
@@ -23,12 +23,12 @@ jobs:
             scala-version: '2.12.15'
             java-compat-version: '8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.4-SNAPSHOT'
+            spark-version: '3.4.5-SNAPSHOT'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
             java-compat-version: '8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.3-SNAPSHOT'
+            spark-version: '3.5.4-SNAPSHOT'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
             java-compat-version: '8'
@@ -44,12 +44,12 @@ jobs:
             scala-version: '2.13.8'
             java-compat-version: '8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.4-SNAPSHOT'
+            spark-version: '3.4.5-SNAPSHOT'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
             java-compat-version: '8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.3-SNAPSHOT'
+            spark-version: '3.5.4-SNAPSHOT'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
             java-compat-version: '8'
diff --git a/pom.xml b/pom.xml
index 8b7b91dc..b21fe994 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,17 +106,45 @@
   </dependencies>
 
   <repositories>
+    <!-- the Maven central repository for releases -->
+    <repository>
+      <id>central</id>
+      <name>Maven Central</name>
+      <layout>default</layout>
+      <url>https://repo1.maven.org/maven2</url>
+      <releases>
+        <enabled>true</enabled>
+        <updatePolicy>never</updatePolicy>
+      </releases>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
+    </repository>
     <!-- the official source for Spark releases, mirrored to Maven central -->
     <repository>
       <id>apache releases</id>
       <name>Apache Releases</name>
       <url>https://repository.apache.org/content/repositories/releases/</url>
+      <releases>
+        <enabled>true</enabled>
+        <updatePolicy>never</updatePolicy>
+      </releases>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
     </repository>
     <!-- required to resolve Spark snapshot versions -->
     <repository>
       <id>apache snapshots</id>
       <name>Apache Snapshots</name>
       <url>https://repository.apache.org/snapshots/</url>
+      <releases>
+        <enabled>false</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+        <updatePolicy>daily</updatePolicy>
+      </snapshots>
     </repository>
   </repositories>