Upgrade to Spark 3.4.4 + 3.5.3 and upgrade Pyhon versions (#259)

G-Research · Nov 4, 2024 · 4d35579 · 4d35579
1 parent 8aee1ab
commit 4d35579
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 66 deletions.
diff --git a/.github/workflows/build-jvm.yml b/.github/workflows/build-jvm.yml
@@ -31,11 +31,11 @@ jobs:
           - spark-compat-version: '3.4'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
-            spark-version: '3.4.2'
+            spark-version: '3.4.4'
           - spark-compat-version: '3.5'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
-            spark-version: '3.5.1'
+            spark-version: '3.5.3'
 
           - spark-compat-version: '3.2'
             spark-version: '3.2.4'
@@ -48,11 +48,11 @@ jobs:
           - spark-compat-version: '3.4'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-version: '3.4.2'
+            spark-version: '3.4.4'
           - spark-compat-version: '3.5'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-version: '3.5.1'
+            spark-version: '3.5.3'
 
     steps:
       - name: Checkout

diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml
@@ -30,11 +30,11 @@ jobs:
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.2'
+            spark-version: '3.4.4'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.1'
+            spark-version: '3.5.3'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
 

diff --git a/.github/workflows/build-snapshots.yml b/.github/workflows/build-snapshots.yml
@@ -23,12 +23,12 @@ jobs:
             scala-version: '2.12.15'
             java-compat-version: '8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.4-SNAPSHOT'
+            spark-version: '3.4.5-SNAPSHOT'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
             java-compat-version: '8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.3-SNAPSHOT'
+            spark-version: '3.5.4-SNAPSHOT'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
             java-compat-version: '8'
@@ -44,12 +44,12 @@ jobs:
             scala-version: '2.13.8'
             java-compat-version: '8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.4-SNAPSHOT'
+            spark-version: '3.4.5-SNAPSHOT'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
             java-compat-version: '8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.3-SNAPSHOT'
+            spark-version: '3.5.4-SNAPSHOT'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
             java-compat-version: '8'

diff --git a/.github/workflows/test-jvm.yml b/.github/workflows/test-jvm.yml
@@ -12,8 +12,8 @@ jobs:
       fail-fast: false
       matrix:
         scala-compat-version: ['2.12', '2.13']
-        spark-compat-version: ['3.2', '3.3']
-        spark-patch-version: ['0', '1', '2', '3']
+        spark-compat-version: ['3.2', '3.3', '3.4', '3.5']
+        spark-patch-version: ['0', '1', '2', '3', '4']
 
         include:
           - spark-compat-version: '3.0'
@@ -30,11 +30,6 @@ jobs:
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
             hadoop-version: '2.7'
-          - spark-compat-version: '3.2'
-            scala-compat-version: '2.12'
-            scala-version: '2.12.15'
-            spark-patch-version: '4'
-            hadoop-version: '2.7'
           - spark-compat-version: '3.3'
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
@@ -52,72 +47,32 @@ jobs:
           - spark-compat-version: '3.4'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
-            spark-patch-version: '0'
-            hadoop-version: '3'
-          - spark-compat-version: '3.4'
-            scala-compat-version: '2.12'
-            scala-version: '2.12.17'
-            spark-patch-version: '1'
-            hadoop-version: '3'
-          - spark-compat-version: '3.4'
-            scala-compat-version: '2.12'
-            scala-version: '2.12.17'
-            spark-patch-version: '2'
-            hadoop-version: '3'
-          - spark-compat-version: '3.5'
-            scala-compat-version: '2.12'
-            scala-version: '2.12.18'
-            spark-patch-version: '0'
             hadoop-version: '3'
           - spark-compat-version: '3.5'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
-            spark-patch-version: '1'
             hadoop-version: '3'
 
           - spark-compat-version: '3.2'
             scala-compat-version: '2.13'
             scala-version: '2.13.5'
             hadoop-version: '3.2'
-          - spark-compat-version: '3.2'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.5'
-            spark-patch-version: '4'
-            hadoop-version: '3.2'
-          - spark-compat-version: '3.3'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.8'
-            hadoop-version: '3'
           - spark-compat-version: '3.3'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-patch-version: '4'
-            hadoop-version: '3'
-          - spark-compat-version: '3.4'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.8'
-            spark-patch-version: '0'
-            hadoop-version: '3'
-          - spark-compat-version: '3.4'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.8'
-            spark-patch-version: '1'
             hadoop-version: '3'
           - spark-compat-version: '3.4'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-patch-version: '2'
             hadoop-version: '3'
           - spark-compat-version: '3.5'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
-            spark-patch-version: '0'
             hadoop-version: '3'
+
+        exclude:
           - spark-compat-version: '3.5'
-            scala-compat-version: '2.13'
-            scala-version: '2.13.8'
-            spark-patch-version: '1'
-            hadoop-version: '3'
+            spark-patch-version: '4'
 
     steps:
       - name: Checkout

diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
@@ -40,12 +40,12 @@ jobs:
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.2'
+            spark-version: '3.4.4'
             hadoop-version: '3'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.1'
+            spark-version: '3.5.3'
             hadoop-version: '3'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'

diff --git a/.github/workflows/test-snapshots.yml b/.github/workflows/test-snapshots.yml
@@ -23,12 +23,12 @@ jobs:
             scala-version: '2.12.15'
             java-compat-version: '8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.4-SNAPSHOT'
+            spark-version: '3.4.5-SNAPSHOT'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
             java-compat-version: '8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.3-SNAPSHOT'
+            spark-version: '3.5.4-SNAPSHOT'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
             java-compat-version: '8'
@@ -44,12 +44,12 @@ jobs:
             scala-version: '2.13.8'
             java-compat-version: '8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.4-SNAPSHOT'
+            spark-version: '3.4.5-SNAPSHOT'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
             java-compat-version: '8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.3-SNAPSHOT'
+            spark-version: '3.5.4-SNAPSHOT'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
             java-compat-version: '8'

diff --git a/pom.xml b/pom.xml
@@ -106,17 +106,45 @@
   </dependencies>
 
   <repositories>
+    <!-- the Maven central repository for releases -->
+    <repository>
+      <id>central</id>
+      <name>Maven Central</name>
+      <layout>default</layout>
+      <url>https://repo1.maven.org/maven2</url>
+      <releases>
+        <enabled>true</enabled>
+        <updatePolicy>never</updatePolicy>
+      </releases>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
+    </repository>
     <!-- the official source for Spark releases, mirrored to Maven central -->
     <repository>
       <id>apache releases</id>
       <name>Apache Releases</name>
       <url>https://repository.apache.org/content/repositories/releases/</url>
+      <releases>
+        <enabled>true</enabled>
+        <updatePolicy>never</updatePolicy>
+      </releases>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
     </repository>
     <!-- required to resolve Spark snapshot versions -->
     <repository>
       <id>apache snapshots</id>
       <name>Apache Snapshots</name>
       <url>https://repository.apache.org/snapshots/</url>
+      <releases>
+        <enabled>false</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+        <updatePolicy>daily</updatePolicy>
+      </snapshots>
     </repository>
   </repositories>