From 5914453e33b488fe86b7a38c73e0667b937cc6cd Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 7 Jun 2024 12:36:39 -0400 Subject: [PATCH] review comments --- .../SplittingTransformerConfiguration.java | 8 ++++++ .../core/async/AsyncResponseTransformer.java | 7 +++++ .../awssdk/transfer/s3/S3TransferManager.java | 27 ++++++++++++++++--- .../s3/internal/TransferManagerFactory.java | 4 ++- .../multipart/DownloadObjectHelper.java | 17 ++++++++++++ 5 files changed, 58 insertions(+), 5 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java index 6c59b675900b..766213195203 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.internal.async.SplittingTransformer; +import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -72,6 +73,13 @@ public int hashCode() { return bufferSizeInBytes != null ? bufferSizeInBytes.hashCode() : 0; } + @Override + public String toString() { + return ToString.builder("SplittingTransformerConfiguration") + .add("bufferSizeInBytes", bufferSizeInBytes) + .build(); + } + @Override public Builder toBuilder() { return new DefaultBuilder(this); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java index 23c10a2dbc8c..dd4d982835a8 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java @@ -142,6 +142,13 @@ default SplitResult split(SplittingTransformerConfiguration .build(); } + default SplitResult split(Consumer splitConfig) { + SplittingTransformerConfiguration conf = SplittingTransformerConfiguration.builder() + .applyMutation(splitConfig) + .build(); + return split(conf); + } + /** * Creates an {@link AsyncResponseTransformer} that writes all the content to the given file. In the event of an error, the * SDK will attempt to delete the file (whatever has been written to it so far). If the file already exists, an exception will diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index 48c682709620..6e5be8ff746d 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -75,9 +75,23 @@ * * S3TransferManager transferManager = * S3TransferManager.builder() - * .s3AsyncClient(s3AsyncClient) + * .s3Client(s3AsyncClient) * .build(); * } + * + * Create an S3 Transfer Manager with S3 Multipart Async Client + * {@snippet : + * S3AsyncClient s3AsyncClient = s3AsyncClient.builder() + * .multipartEnabled(true) + * .multipartConfiguration(conf -> conf.apiCallBufferSizeInBytes(32 * MB)) + * .build(); + * + * S3TransferManager transferManager = + * S3TransferManager.builder() + * .s3Client(s3AsyncClient) + * .build(); + * } + * *

Common Usage Patterns

* Upload a file to S3 * {@snippet : @@ -157,9 +171,14 @@ * // Wait for the transfer to complete * CompletedCopy completedCopy = copy.completionFuture().join(); * } - * Warns the user if a multi-part operation disabled client is provided for TransferManager to use - * If no client is provided, a multi-part enabled async client is instantiated and used. - * In the event of a multi-part disabled defaultS3AsyncClient being used, a warning is logged + * The automatic parallel transfer feature (multipart upload/download) is available + * through the AWS-CRT based S3 client {@link S3AsyncClient#crtBuilder().build)} + * and Java-based S3 multipart client {@link S3AsyncClient#builder().multipartEnabled(true).build()}. + * If no client is configured, AWS-CRT based S3 client will be used if AWS CRT is in the classpath, + * otherwise, Java-based S3 multipart client will be used. + * {@snippet + * + * } */ @SdkPublicApi @ThreadSafe diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerFactory.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerFactory.java index dbf744afb738..429d534e074e 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerFactory.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerFactory.java @@ -54,7 +54,9 @@ public static S3TransferManager createTransferManager(DefaultBuilder tmBuilder) if (!s3AsyncClient.getClass().getName().equals("software.amazon.awssdk.services.s3.internal.multipart" + ".MultipartS3AsyncClient")) { - log.debug(() -> "The provided S3AsyncClient is neither an instance of S3CrtAsyncClient or MultipartS3AsyncClient, " + log.debug(() -> "The provided S3AsyncClient is neither " + + "an AWS CRT-based S3 async client (S3AsyncClient.crtBuilder().build()) or " + + "a Java-based S3 async client (S3AsyncClient.builder().multipartEnabled(true).build()), " + "and thus multipart upload/download feature may not be enabled and resumable file upload may not " + "be supported. To benefit from maximum throughput, consider using " + "S3AsyncClient.crtBuilder().build() or " diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/DownloadObjectHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/DownloadObjectHelper.java index 1d94f2ed360b..1154c908fe87 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/DownloadObjectHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/DownloadObjectHelper.java @@ -24,9 +24,12 @@ import software.amazon.awssdk.services.s3.model.ChecksumMode; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.utils.Logger; @SdkInternalApi public class DownloadObjectHelper { + private static final Logger log = Logger.loggerFor(DownloadObjectHelper.class); + private final S3AsyncClient s3AsyncClient; private final long bufferSizeInBytes; @@ -38,6 +41,7 @@ public DownloadObjectHelper(S3AsyncClient s3AsyncClient, long bufferSizeInBytes) public CompletableFuture downloadObject( GetObjectRequest getObjectRequest, AsyncResponseTransformer asyncResponseTransformer) { if (getObjectRequest.range() != null || getObjectRequest.partNumber() != null) { + logSinglePartWarning(getObjectRequest); return s3AsyncClient.getObject(getObjectRequest, asyncResponseTransformer); } GetObjectRequest requestToPerform = getObjectRequest.toBuilder().checksumMode(ChecksumMode.ENABLED).build(); @@ -57,4 +61,17 @@ private MultipartDownloaderSubscriber subscriber(GetObjectRequest getObjectReque .map(ctx -> new MultipartDownloaderSubscriber(s3AsyncClient, getObjectRequest, ctx.highestSequentialCompletedPart())) .orElseGet(() -> new MultipartDownloaderSubscriber(s3AsyncClient, getObjectRequest)); } + + private void logSinglePartWarning(GetObjectRequest getObjectRequest) { + String reason = ""; + if (getObjectRequest.range() != null) { + reason = " because getObjectRequest range is included in the request." + + " range = " + getObjectRequest.range(); + } else if (getObjectRequest.partNumber() != null) { + reason = " because getObjectRequest part number is included in the request." + + " part number = " + getObjectRequest.partNumber(); + } + String finalReason = reason; + log.debug(() -> "Using single part download" + finalReason); + } }