diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c20cc07..f7da290e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## v32.43.3 - 2025-01-06 + +Update ElasticBuilder to generate and accept config case classes to enable +pushing typesafe config to the edge of consuming services. + ## v32.43.2 - 2024-10-07 S3StreamWritable bug fix (replace `read` by `readNBytes`). diff --git a/build.sbt b/build.sbt index fec66569..d9663395 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -val projectVersion = "32.43.2" +val projectVersion = "32.43.3" Global / excludeLintKeys += composeNoBuild diff --git a/elasticsearch_typesafe/src/main/scala/weco/elasticsearch/typesafe/ElasticBuilder.scala b/elasticsearch_typesafe/src/main/scala/weco/elasticsearch/typesafe/ElasticBuilder.scala index 01f0defd..75d327bc 100644 --- a/elasticsearch_typesafe/src/main/scala/weco/elasticsearch/typesafe/ElasticBuilder.scala +++ b/elasticsearch_typesafe/src/main/scala/weco/elasticsearch/typesafe/ElasticBuilder.scala @@ -3,11 +3,35 @@ package weco.elasticsearch.typesafe import com.sksamuel.elastic4s.ElasticClient import com.typesafe.config.Config import weco.elasticsearch.ElasticClientBuilder -import weco.typesafe.config.builders.EnrichConfig._ + +sealed trait ElasticConfig { + val host: String + val port: Int + val protocol: String +} + +case class ElasticConfigUsernamePassword( + host: String, + port: Int, + protocol: String, + username: String, + password: String +) extends ElasticConfig + +case class ElasticConfigApiKey( + host: String, + port: Int, + protocol: String, + apiKey: String +) extends ElasticConfig object ElasticBuilder { - def buildElasticClient(config: Config, - namespace: String = ""): ElasticClient = { + import weco.typesafe.config.builders.EnrichConfig._ + + def buildElasticClientConfig( + config: Config, + namespace: String = "" + ): ElasticConfig = { val hostname = config.requireString(s"es.$namespace.host") val port = config .getIntOption(s"es.$namespace.port") @@ -22,7 +46,7 @@ object ElasticBuilder { config.getStringOption(s"es.$namespace.apikey") ) match { case (Some(username), Some(password), None) => - ElasticClientBuilder.create( + ElasticConfigUsernamePassword( hostname, port, protocol, @@ -31,10 +55,37 @@ object ElasticBuilder { ) // Use an API key if specified, even if username/password are also present case (_, _, Some(apiKey)) => - ElasticClientBuilder.create(hostname, port, protocol, apiKey) + ElasticConfigApiKey(hostname, port, protocol, apiKey) case _ => throw new Throwable( - s"You must specify username and password, or apikey, in the 'es.$namespace' config") + s"You must specify username and password, or apikey, in the 'es.$namespace' config" + ) } } + + def buildElasticClient(config: ElasticConfig): ElasticClient = + config match { + case ElasticConfigUsernamePassword( + hostname, + port, + protocol, + username, + password + ) => + ElasticClientBuilder.create( + hostname, + port, + protocol, + username, + password + ) + case ElasticConfigApiKey(hostname, port, protocol, apiKey) => + ElasticClientBuilder.create(hostname, port, protocol, apiKey) + } + + def buildElasticClient( + config: Config, + namespace: String = "" + ): ElasticClient = + buildElasticClient(buildElasticClientConfig(config, namespace)) } diff --git a/elasticsearch_typesafe/src/test/scala/weco/elasticsearch/typesafe/ElasticBuilderTest.scala b/elasticsearch_typesafe/src/test/scala/weco/elasticsearch/typesafe/ElasticBuilderTest.scala index 341875ec..35020698 100644 --- a/elasticsearch_typesafe/src/test/scala/weco/elasticsearch/typesafe/ElasticBuilderTest.scala +++ b/elasticsearch_typesafe/src/test/scala/weco/elasticsearch/typesafe/ElasticBuilderTest.scala @@ -3,45 +3,116 @@ package weco.elasticsearch.typesafe import com.typesafe.config.ConfigFactory import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers +import weco.fixtures.RandomGenerators -class ElasticBuilderTest extends AnyFunSpec with Matchers { - it("builds a client when a username and password are specified") { - val config = ConfigFactory.parseString( - """ - |es.test.host = test.elastic - |es.test.username = test-username - |es.test.password = test-password - |""".stripMargin - ) +class ElasticBuilderTest + extends AnyFunSpec + with Matchers + with RandomGenerators { - noException shouldBe thrownBy( - ElasticBuilder.buildElasticClient(config, "test") + val List(namespace, host, username, password, apiKey) = + 1.to(5).map(_ => randomAlphanumeric()).toList + val defaultPort = 9200 + val defaultProtocol = "http" + + describe("when username and password are specified") { + val usernamePasswordConfig = ConfigFactory.parseString( + f""" + |es.$namespace.host = $host + |es.$namespace.username = $username + |es.$namespace.password = $password + |""".stripMargin ) + + it("can build and accept ElasticConfigUsernamePassword") { + val elasticConfig = ElasticBuilder.buildElasticClientConfig( + usernamePasswordConfig, + namespace) + + elasticConfig shouldBe ElasticConfigUsernamePassword( + host = host, + port = defaultPort, + protocol = defaultProtocol, + username = username, + password = password + ) + + noException shouldBe thrownBy( + ElasticBuilder.buildElasticClient(elasticConfig) + ) + } + + it("can build a client directly") { + noException shouldBe thrownBy( + ElasticBuilder.buildElasticClient(usernamePasswordConfig, namespace) + ) + } } - it("builds a client when an API key is specified") { - val config = ConfigFactory.parseString( - """ - |es.test.host = test.elastic - |es.test.apikey = test-key - |""".stripMargin - ) + val apiKeyConfig = ConfigFactory.parseString( + f""" + |es.$namespace.host = $host + |es.$namespace.apikey = $apiKey + |""".stripMargin + ) - noException shouldBe thrownBy( - ElasticBuilder.buildElasticClient(config, "test") - ) + describe("when an API key is specified") { + it("can build and accept ElasticConfigApiKey") { + val elasticConfig = + ElasticBuilder.buildElasticClientConfig(apiKeyConfig, namespace) + + elasticConfig shouldBe ElasticConfigApiKey( + host = host, + port = defaultPort, + protocol = defaultProtocol, + apiKey = apiKey + ) + + noException shouldBe thrownBy( + ElasticBuilder.buildElasticClient(elasticConfig) + ) + } + + it("builds a client directly") { + noException shouldBe thrownBy( + ElasticBuilder.buildElasticClient(apiKeyConfig, namespace) + ) + } + } + + describe("when an API key and username and password is specified") { + it("uses the API key") { + val config = ConfigFactory.parseString( + f""" + |es.$namespace.host = $host + |es.$namespace.username = $username + |es.$namespace.password = $password + |es.$namespace.apikey = $apiKey + |""".stripMargin + ) + + val elasticConfig = + ElasticBuilder.buildElasticClientConfig(config, namespace) + + elasticConfig shouldBe ElasticConfigApiKey( + host = host, + port = defaultPort, + protocol = defaultProtocol, + apiKey = apiKey + ) + } } it("errors if there is not enough config to build a client") { val config = ConfigFactory.parseString( - """ - |es.test.host = test.elastic - |es.test.username = test-username + f""" + |es.$namespace.host = $host + |es.$namespace.username = $username |""".stripMargin ) a[Throwable] shouldBe thrownBy( - ElasticBuilder.buildElasticClient(config, "test") + ElasticBuilder.buildElasticClient(config, namespace) ) } } diff --git a/fixtures/src/test/scala/weco/fixtures/LocalResources.scala b/fixtures/src/test/scala/weco/fixtures/LocalResources.scala index c5aef01c..a7478ca7 100644 --- a/fixtures/src/test/scala/weco/fixtures/LocalResources.scala +++ b/fixtures/src/test/scala/weco/fixtures/LocalResources.scala @@ -18,7 +18,8 @@ trait LocalResources { case Success(lines) => lines.mkString("\n") case Failure(_: NullPointerException) if name.startsWith("/") => - throw new RuntimeException(s"Could not find resource `$name`; try removing the leading slash") + throw new RuntimeException( + s"Could not find resource `$name`; try removing the leading slash") case Failure(_: NullPointerException) => throw new RuntimeException(s"Could not find resource `$name`") diff --git a/messaging/src/test/scala/weco/messaging/sns/SNSMessageSenderTest.scala b/messaging/src/test/scala/weco/messaging/sns/SNSMessageSenderTest.scala index 09cf6c60..1bb2c02e 100644 --- a/messaging/src/test/scala/weco/messaging/sns/SNSMessageSenderTest.scala +++ b/messaging/src/test/scala/weco/messaging/sns/SNSMessageSenderTest.scala @@ -34,7 +34,8 @@ class SNSMessageSenderTest val result = sender.send("hello world")( subject = "Sent from SNSMessageSenderTest", - destination = SNSConfig(topicArn = "arn:aws:sns:eu-west-1:012345678912:doesnotexist") + destination = + SNSConfig(topicArn = "arn:aws:sns:eu-west-1:012345678912:doesnotexist") ) result shouldBe a[Failure[_]] diff --git a/storage/src/test/scala/weco/storage/fixtures/S3Fixtures.scala b/storage/src/test/scala/weco/storage/fixtures/S3Fixtures.scala index 90306516..af5a2020 100644 --- a/storage/src/test/scala/weco/storage/fixtures/S3Fixtures.scala +++ b/storage/src/test/scala/weco/storage/fixtures/S3Fixtures.scala @@ -53,7 +53,8 @@ trait S3Fixtures AwsBasicCredentials.create("accessKey1", "verySecretKey1")) def createS3ClientWithEndpoint(endpoint: String): S3Client = - S3Client.builder() + S3Client + .builder() .credentialsProvider(s3Credentials) .forcePathStyle(true) .endpointOverride(new URI(endpoint)) @@ -66,13 +67,15 @@ trait S3Fixtures createS3ClientWithEndpoint("http://nope.nope") implicit val s3Presigner: S3Presigner = - S3Presigner.builder() + S3Presigner + .builder() .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create("accessKey1", "verySecretKey1")) ) .serviceConfiguration( - S3Configuration.builder() + S3Configuration + .builder() .pathStyleAccessEnabled(true) .build() ) @@ -83,14 +86,15 @@ trait S3Fixtures // This is based on a method called doesBucketExistV2 in the V1 Java SDK, // which used GetBucketAcl under the hood to check if a bucket existed. val request = - GetBucketAclRequest.builder() + GetBucketAclRequest + .builder() .bucket(bucketName) .build() Try { s3Client.getBucketAcl(request) } match { - case Success(_) => true + case Success(_) => true case Failure(e: S3Exception) if e.statusCode() == 404 => false - case Failure(e) => throw e + case Failure(e) => throw e } } @@ -103,7 +107,8 @@ trait S3Fixtures val bucketName: String = createBucketName val request = - CreateBucketRequest.builder() + CreateBucketRequest + .builder() .bucket(bucketName) .build() @@ -133,7 +138,8 @@ trait S3Fixtures def deleteBucket(bucket: Bucket): Unit = { val deleteBucketRequest = - DeleteBucketRequest.builder() + DeleteBucketRequest + .builder() .bucket(bucket.name) .build() @@ -142,7 +148,8 @@ trait S3Fixtures def deleteObject(location: S3ObjectLocation): Unit = { val deleteObjectRequest = - DeleteObjectRequest.builder() + DeleteObjectRequest + .builder() .bucket(location.bucket) .key(location.key) .build() @@ -152,7 +159,8 @@ trait S3Fixtures def getContentFromS3(location: S3ObjectLocation): String = { val getRequest = - GetObjectRequest.builder() + GetObjectRequest + .builder() .bucket(location.bucket) .key(location.key) .build() @@ -175,7 +183,8 @@ trait S3Fixtures def putString(location: S3ObjectLocation, contents: String): Unit = { val putRequest = - PutObjectRequest.builder() + PutObjectRequest + .builder() .bucket(location.bucket) .key(location.key) .build() @@ -189,13 +198,15 @@ trait S3Fixtures location: S3ObjectLocation, inputStream: InputStreamWithLength = createInputStream()): Unit = { val putRequest = - PutObjectRequest.builder() + PutObjectRequest + .builder() .bucket(location.bucket) .key(location.key) .contentLength(inputStream.length) .build() - val requestBody = RequestBody.fromInputStream(inputStream, inputStream.length) + val requestBody = + RequestBody.fromInputStream(inputStream, inputStream.length) s3Client.putObject(putRequest, requestBody) @@ -212,15 +223,21 @@ trait S3Fixtures */ def listKeysInBucket(bucket: Bucket): List[String] = { val listRequest = - ListObjectsV2Request.builder() + ListObjectsV2Request + .builder() .bucket(bucket.name) .build() - s3Client.listObjectsV2Paginator(listRequest) + s3Client + .listObjectsV2Paginator(listRequest) .iterator() .asScala - .flatMap { resp: ListObjectsV2Response => resp.contents().asScala } - .map { s3Obj: S3Object => s3Obj.key() } + .flatMap { resp: ListObjectsV2Response => + resp.contents().asScala + } + .map { s3Obj: S3Object => + s3Obj.key() + } .toList } diff --git a/storage/src/test/scala/weco/storage/store/s3/S3StreamReadableTest.scala b/storage/src/test/scala/weco/storage/store/s3/S3StreamReadableTest.scala index c59309a5..ea72fa6d 100644 --- a/storage/src/test/scala/weco/storage/store/s3/S3StreamReadableTest.scala +++ b/storage/src/test/scala/weco/storage/store/s3/S3StreamReadableTest.scala @@ -26,7 +26,8 @@ class S3StreamReadableTest override val maxRetries: Int = retries } - val s3ServerException = S3Exception.builder() + val s3ServerException = S3Exception + .builder() .message("We encountered an internal error. Please try again.") .statusCode(500) .build() @@ -41,7 +42,8 @@ class S3StreamReadableTest when(mockClient.getObject(any[GetObjectRequest])) .thenThrow( - S3Exception.builder() + S3Exception + .builder() .statusCode(404) .build() ) @@ -64,7 +66,8 @@ class S3StreamReadableTest .thenThrow(s3ServerException) .thenReturn({ val getRequest = - GetObjectRequest.builder() + GetObjectRequest + .builder() .bucket(location.bucket) .key(location.key) .build() @@ -108,7 +111,10 @@ class S3StreamReadableTest withLocalS3Bucket { bucket => val location = createS3ObjectLocationWith(bucket) - val exception = SdkClientException.builder().message("Unable to execute HTTP request").build() + val exception = SdkClientException + .builder() + .message("Unable to execute HTTP request") + .build() when(mockClient.getObject(any[GetObjectRequest])) .thenThrow(exception) diff --git a/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala b/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala index 6fc3d0e6..e0b16d04 100644 --- a/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala +++ b/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala @@ -26,7 +26,8 @@ class S3StreamStoreTest val err = result.e err shouldBe a[SdkClientException] - err.getMessage should startWith("Received an UnknownHostException when attempting to interact with a service") + err.getMessage should startWith( + "Received an UnknownHostException when attempting to interact with a service") } it("errors if the key doesn't exist") { @@ -77,7 +78,8 @@ class S3StreamStoreTest val err = result.e err shouldBe a[SdkClientException] - err.getCause.getMessage should startWith("Unable to execute HTTP request") + err.getCause.getMessage should startWith( + "Unable to execute HTTP request") } it("errors if the bucket doesn't exist") { @@ -139,11 +141,13 @@ class S3StreamStoreTest withLocalS3Bucket { bucket => val location = createS3ObjectLocationWith(bucket) - val result = store.put(location)(new InputStreamWithLength(inputStream, length)) + val result = + store.put(location)(new InputStreamWithLength(inputStream, length)) result shouldBe a[Right[_, _]] val getRequest = - GetObjectRequest.builder() + GetObjectRequest + .builder() .bucket(location.bucket) .key(location.key) .build()