Merge pull request #260 from wellcomecollection/rk/use-localstack-in-…

…storage Use localstack in storage tests
wellcomecollection · Jan 6, 2025 · 22d55dd · 22d55dd
2 parents 05beca0 + b2c73b7
commit 22d55dd
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 22 deletions.
diff --git a/storage/docker-compose.yml b/storage/docker-compose.yml
@@ -5,19 +5,9 @@ services:
     ports:
       - "45678:8000"
   s3:
-    image: "zenko/cloudserver:8.1.8"
+    image: "public.ecr.aws/localstack/localstack:4.0.0"
     environment:
-      - "S3BACKEND=mem"
+      - SERVICES=s3
+      - ALLOW_NONSTANDARD_REGIONS=1
     ports:
-      - "33333:8000"
-
-    # We've seen flakiness when this container doesn't start fast enough,
-    # and the first few tests to interact with S3 fail.  This uses
-    # docker-compose healthchecks to check the container is returning
-    # a 401 Unauthorized before continuing.
-    # See https://docs.docker.com/compose/compose-file/#/healthcheck
-    healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:33333"]
-      interval: 2s
-      timeout: 10s
-      retries: 5
+      - "33333:4566"
diff --git a/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala b/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala
@@ -28,6 +28,10 @@ object S3Errors {
         if exc.getMessage.startsWith("The specified bucket is not valid") =>
       StoreReadError(exc)
 
+    case exc: S3Exception
+        if exc.getMessage.startsWith("The specified bucket does not exist") =>
+      StoreReadError(exc)
+
     case exc: SdkClientException
         if exc.getMessage.startsWith("Unable to execute HTTP request") =>
       new StoreReadError(exc) with RetryableError
@@ -54,6 +58,9 @@ object S3Errors {
     case exc: S3Exception
         if exc.getMessage.startsWith("Object key is too long") =>
       InvalidIdentifierFailure(exc)
+    case exc: S3Exception
+        if exc.getMessage.startsWith("Your key is too long") =>
+      InvalidIdentifierFailure(exc)
 
     case exc => StoreWriteError(exc)
   }

diff --git a/storage/src/test/scala/weco/storage/store/dynamo/DynamoHybridStoreTestCases.scala b/storage/src/test/scala/weco/storage/store/dynamo/DynamoHybridStoreTestCases.scala
@@ -171,7 +171,7 @@ trait DynamoHybridStoreTestCases[
 
                     value shouldBe a[StoreWriteError]
                     value.e.getMessage should startWith(
-                      "The specified bucket is not valid")
+                      "The specified bucket does not exist")
                   }
                 }
               }
@@ -206,7 +206,7 @@ trait DynamoHybridStoreTestCases[
                 val value = result.left.value
 
                 value shouldBe a[InvalidIdentifierFailure]
-                value.e.getMessage should startWith("Object key is too long")
+                value.e.getMessage should include("key is too long")
               }
             }
           }

diff --git a/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala b/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala
@@ -59,10 +59,11 @@ class S3StreamStoreTest
         withStoreImpl(initialEntries = Map.empty) { store =>
           val invalidLocation = createS3ObjectLocationWith(createInvalidBucket)
           val err = store.get(invalidLocation).left.value
-          err shouldBe a[StoreReadError]
+          err shouldBe a[DoesNotExistError]
 
           err.e shouldBe a[S3Exception]
-          err.e.getMessage should startWith("The specified bucket is not valid")
+          err.e.getMessage should startWith(
+            "The specified bucket does not exist")
         }
       }
     }
@@ -102,7 +103,7 @@ class S3StreamStoreTest
 
           val err = result.e
           err shouldBe a[S3Exception]
-          err.getMessage should startWith("The specified bucket is not valid")
+          err.getMessage should startWith("The specified bucket does not exist")
         }
       }
 
@@ -121,7 +122,7 @@ class S3StreamStoreTest
             val value = store.put(id)(entry).left.value
 
             value shouldBe a[InvalidIdentifierFailure]
-            value.e.getMessage should startWith("Object key is too long")
+            value.e.getMessage should include("key is too long")
           }
         }
       }

diff --git a/storage/src/test/scala/weco/storage/store/s3/S3TypedStoreTest.scala b/storage/src/test/scala/weco/storage/store/s3/S3TypedStoreTest.scala
@@ -61,7 +61,7 @@ class S3TypedStoreTest
   describe("S3TypedStore") {
     it("errors if the object key is too long") {
       withLocalS3Bucket { bucket =>
-        // Maximum length of an s3 key is 1024 bytes as of 25/06/2019
+        // Maximum length of a s3 key is 1024 bytes as of 25/06/2019
         // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
         val location = S3ObjectLocation(
           bucket = bucket.name,
@@ -74,7 +74,7 @@ class S3TypedStoreTest
           val value = store.put(location)(entry).left.value
 
           value shouldBe a[InvalidIdentifierFailure]
-          value.e.getMessage should startWith("Object key is too long")
+          value.e.getMessage should include("key is too long")
         }
       }
     }