Merge pull request #563 from boozallen/561-harden-spark-images

[#561] spark image hardening
boozallen · Feb 4, 2025 · 4728f16 · 4728f16
2 parents df4be07 + 4889471
commit 4728f16
Show file tree

Hide file tree

Showing 17 changed files with 267 additions and 937 deletions.
diff --git a/build-parent/pom.xml b/build-parent/pom.xml
@@ -88,9 +88,9 @@
         <version.sedona>1.4.0</version.sedona>
         <version.geotools.wrapper>1.4.0-28.2</version.geotools.wrapper>
         <version.mysql-connector>8.0.30</version.mysql-connector>
-        <version.hadoop>3.3.4</version.hadoop>
+        <version.hadoop>3.4.1</version.hadoop>
         <version.neo4j>4.1.5_for_spark_3</version.neo4j>
-        <version.aws.sdk.bundle>1.12.780</version.aws.sdk.bundle>
+        <version.aws.sdk.bundle>2.25.53</version.aws.sdk.bundle>
         <version.baton>1.1.1</version.baton>
         <version.quarkus.cucumber>1.0.0</version.quarkus.cucumber>
         <version.quarkus.cucumber.java>${version.cucumber}</version.quarkus.cucumber.java>
@@ -552,6 +552,7 @@
                             <VERSION_AISSEMBLE>${version.aissemble}</VERSION_AISSEMBLE>
                             <SPARK_VERSION>${version.spark}</SPARK_VERSION>
                             <SCALA_VERSION>${version.scala.minor}</SCALA_VERSION>
+                            <HADOOP_VERSION>${version.hadoop}</HADOOP_VERSION>
                         </buildArgs>
                     </configuration>
                 </plugin>
@@ -602,6 +603,7 @@
                                         <VERSION_AISSEMBLE>${version.aissemble}</VERSION_AISSEMBLE>
                                         <SPARK_VERSION>${version.spark}</SPARK_VERSION>
                                         <SCALA_VERSION>${version.scala.minor}</SCALA_VERSION>
+                                        <HADOOP_VERSION>${version.hadoop}</HADOOP_VERSION>
                                     </args>
                                     <buildx>
                                         <builderName>default</builderName>

diff --git a/extensions/extensions-data-delivery/extensions-data-delivery-spark/pom.xml b/extensions/extensions-data-delivery/extensions-data-delivery-spark/pom.xml
@@ -148,21 +148,6 @@
             <version>${version.cucumber.reporting.plugin}</version>
             <scope>test</scope>
         </dependency>
-        <!-- These are large dependencies only used for ObjectStoreValidator so they're being set to provided since we
-          aren't really leveraging the ObjectStore. If we pick the ObjectStore effort back up, we should make this more
-          modular, so you're only pulling it in if you need it. -->
-        <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-s3</artifactId>
-            <version>1.12.456</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-aws</artifactId>
-            <version>${version.hadoop}</version>
-            <scope>provided</scope>
-        </dependency>
     </dependencies>
 
 </project>
diff --git a/.../src/main/java/com/boozallen/aiops/data/delivery/spark/objectstore/ObjectStoreConfig.java b/.../src/main/java/com/boozallen/aiops/data/delivery/spark/objectstore/ObjectStoreConfig.java
diff --git a/...rk/src/main/java/com/boozallen/aiops/data/delivery/spark/objectstore/ObjectStoreEnum.java b/...rk/src/main/java/com/boozallen/aiops/data/delivery/spark/objectstore/ObjectStoreEnum.java