upgrading tests

fabiogouw · Jun 2, 2024 · ca749e8 · ca749e8
1 parent 820255b
commit ca749e8
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 7 deletions.
diff --git a/spark-aws-messaging/build.gradle b/spark-aws-messaging/build.gradle
@@ -8,7 +8,7 @@ plugins {
 
 group = 'com.fabiogouw'
 archivesBaseName = 'spark-aws-messaging'
-version = '1.1.0'
+version = '1.2.0'
 
 repositories {
     mavenCentral()

diff --git a/...ws-messaging/src/test/java/com/fabiogouw/spark/awsmessaging/sqs/SparkIntegrationTest.java b/...ws-messaging/src/test/java/com/fabiogouw/spark/awsmessaging/sqs/SparkIntegrationTest.java
@@ -1,6 +1,5 @@
 package com.fabiogouw.spark.awsmessaging.sqs;
 
-import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.client.builder.AwsClientBuilder;
@@ -30,6 +29,9 @@
 public class SparkIntegrationTest {
 
     private static final Network network = Network.newNetwork();
+    private static final String libSparkAWSMessaging = "spark-aws-messaging-1.1.0.jar";
+    private static final String libAWSJavaSdkCore = "aws-java-sdk-core-1.12.13.jar";
+    private static final String libAWSJavaSdkSqs = "aws-java-sdk-sqs-1.12.13.jar";
 
     @Container
     private final LocalStackContainer localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:latest"))
@@ -40,8 +42,10 @@ public class SparkIntegrationTest {
 
     @Container
     private final GenericContainer spark = new GenericContainer(DockerImageName.parse("bitnami/spark:3.3.2"))
-            .withCopyFileToContainer(MountableFile.forHostPath("build/resources/test/.", 0744), "/home/")
-            .withCopyFileToContainer(MountableFile.forHostPath("build/libs/.", 0555), "/home/")
+            .withCopyFileToContainer(MountableFile.forHostPath("build/resources/test/.", 0445), "/home/")
+            .withCopyFileToContainer(MountableFile.forHostPath("build/libs/" + libSparkAWSMessaging, 0445), "/home/")
+            .withCopyFileToContainer(MountableFile.forHostPath("build/libs/deps/" + libAWSJavaSdkCore, 0445), "/home/")
+            .withCopyFileToContainer(MountableFile.forHostPath("build/libs/deps/" + libAWSJavaSdkSqs, 0445), "/home/")
             .withNetwork(network)
             .withEnv("AWS_ACCESS_KEY_ID", "test")
             .withEnv("AWS_SECRET_KEY", "test")
@@ -74,7 +78,7 @@ private AmazonSQS configureQueue() {
     private ExecResult execSparkJob(String script, String... args) throws IOException, InterruptedException {
         String[] command = ArrayUtils.addAll(new String[] {"spark-submit",
                 "--jars",
-                "/home/spark-aws-messaging-1.1.0.jar,/home/deps/aws-java-sdk-core-1.12.13.jar,/home/deps/aws-java-sdk-sqs-1.12.13.jar",
+                "/home/" + libSparkAWSMessaging + ",/home/" + libAWSJavaSdkCore + ",/home/" + libAWSJavaSdkSqs,
                 "--master",
                 "local",
                 script}, args);
@@ -144,7 +148,7 @@ void when_DataframeContainsDataExceedsSQSSizeLimit_should_FailWholeBatch() throw
                 "http://localstack:4566");
         // assert
         assertThat(result.getExitCode()).as("Spark job should execute fail").isNotZero();
-        assertThat(result.getStderr()).as("Spark job should fail due to exceeding size limit").contains("Batch requests cannot be longer than 262144 bytes");
+        assertThat(result.getStdout()).as("Spark job should fail due to exceeding size limit").contains("Batch requests cannot be longer than 262144 bytes");
         List<Message> messages = getMessagesPut(sqs);
         assertThat(messages).size().as("No messages should be written when the batch fails").isZero();
     }
@@ -162,7 +166,7 @@ void when_DataframeContainsLinesThatExceedsSQSMessageSizeLimit_should_ThrowAnExc
                 "http://localstack:4566");
         // assert
         assertThat(result.getExitCode()).as("Spark job should execute fail").isNotZero();
-        assertThat(result.getStderr()).as("Spark job should fail due to exceeding size limit").contains("Some messages failed to be sent to the SQS queue");
+        assertThat(result.getStdout()).as("Spark job should fail due to exceeding size limit").contains("Some messages failed to be sent to the SQS queue");
         List<Message> messages = getMessagesPut(sqs);
         assertThat(messages).size().as("Only messages up to 1024 should be written").isEqualTo(2);
     }