Skip to content

Commit

Permalink
review comments
Browse files Browse the repository at this point in the history
  • Loading branch information
L-Applin committed Jun 7, 2024
1 parent d35cb3b commit 5914453
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import software.amazon.awssdk.annotations.SdkPublicApi;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.core.internal.async.SplittingTransformer;
import software.amazon.awssdk.utils.ToString;
import software.amazon.awssdk.utils.Validate;
import software.amazon.awssdk.utils.builder.CopyableBuilder;
import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
Expand Down Expand Up @@ -72,6 +73,13 @@ public int hashCode() {
return bufferSizeInBytes != null ? bufferSizeInBytes.hashCode() : 0;
}

@Override
public String toString() {
return ToString.builder("SplittingTransformerConfiguration")
.add("bufferSizeInBytes", bufferSizeInBytes)
.build();
}

@Override
public Builder toBuilder() {
return new DefaultBuilder(this);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,13 @@ default SplitResult<ResponseT, ResultT> split(SplittingTransformerConfiguration
.build();
}

default SplitResult<ResponseT, ResultT> split(Consumer<SplittingTransformerConfiguration.Builder> splitConfig) {
SplittingTransformerConfiguration conf = SplittingTransformerConfiguration.builder()
.applyMutation(splitConfig)
.build();
return split(conf);
}

/**
* Creates an {@link AsyncResponseTransformer} that writes all the content to the given file. In the event of an error, the
* SDK will attempt to delete the file (whatever has been written to it so far). If the file already exists, an exception will
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,23 @@
*
* S3TransferManager transferManager =
* S3TransferManager.builder()
* .s3AsyncClient(s3AsyncClient)
* .s3Client(s3AsyncClient)
* .build();
* }
*
* <b>Create an S3 Transfer Manager with S3 Multipart Async Client</b>
* {@snippet :
* S3AsyncClient s3AsyncClient = s3AsyncClient.builder()
* .multipartEnabled(true)
* .multipartConfiguration(conf -> conf.apiCallBufferSizeInBytes(32 * MB))
* .build();
*
* S3TransferManager transferManager =
* S3TransferManager.builder()
* .s3Client(s3AsyncClient)
* .build();
* }
*
* <h2>Common Usage Patterns</h2>
* <b>Upload a file to S3</b>
* {@snippet :
Expand Down Expand Up @@ -157,9 +171,14 @@
* // Wait for the transfer to complete
* CompletedCopy completedCopy = copy.completionFuture().join();
* }
* <b> Warns the user if a multi-part operation disabled client is provided for TransferManager to use
* If no client is provided, a multi-part enabled async client is instantiated and used.
* In the event of a multi-part disabled defaultS3AsyncClient being used, a warning is logged </b>
* <b> The automatic parallel transfer feature (multipart upload/download) is available
* through the AWS-CRT based S3 client {@link S3AsyncClient#crtBuilder().build)}
* and Java-based S3 multipart client {@link S3AsyncClient#builder().multipartEnabled(true).build()}.
* If no client is configured, AWS-CRT based S3 client will be used if AWS CRT is in the classpath,
* otherwise, Java-based S3 multipart client will be used. </b>
* {@snippet
*
* }
*/
@SdkPublicApi
@ThreadSafe
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ public static S3TransferManager createTransferManager(DefaultBuilder tmBuilder)

if (!s3AsyncClient.getClass().getName().equals("software.amazon.awssdk.services.s3.internal.multipart"
+ ".MultipartS3AsyncClient")) {
log.debug(() -> "The provided S3AsyncClient is neither an instance of S3CrtAsyncClient or MultipartS3AsyncClient, "
log.debug(() -> "The provided S3AsyncClient is neither "
+ "an AWS CRT-based S3 async client (S3AsyncClient.crtBuilder().build()) or "
+ "a Java-based S3 async client (S3AsyncClient.builder().multipartEnabled(true).build()), "
+ "and thus multipart upload/download feature may not be enabled and resumable file upload may not "
+ "be supported. To benefit from maximum throughput, consider using "
+ "S3AsyncClient.crtBuilder().build() or "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,12 @@
import software.amazon.awssdk.services.s3.model.ChecksumMode;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.utils.Logger;

@SdkInternalApi
public class DownloadObjectHelper {
private static final Logger log = Logger.loggerFor(DownloadObjectHelper.class);

private final S3AsyncClient s3AsyncClient;
private final long bufferSizeInBytes;

Expand All @@ -38,6 +41,7 @@ public DownloadObjectHelper(S3AsyncClient s3AsyncClient, long bufferSizeInBytes)
public <T> CompletableFuture<T> downloadObject(
GetObjectRequest getObjectRequest, AsyncResponseTransformer<GetObjectResponse, T> asyncResponseTransformer) {
if (getObjectRequest.range() != null || getObjectRequest.partNumber() != null) {
logSinglePartWarning(getObjectRequest);
return s3AsyncClient.getObject(getObjectRequest, asyncResponseTransformer);
}
GetObjectRequest requestToPerform = getObjectRequest.toBuilder().checksumMode(ChecksumMode.ENABLED).build();
Expand All @@ -57,4 +61,17 @@ private MultipartDownloaderSubscriber subscriber(GetObjectRequest getObjectReque
.map(ctx -> new MultipartDownloaderSubscriber(s3AsyncClient, getObjectRequest, ctx.highestSequentialCompletedPart()))
.orElseGet(() -> new MultipartDownloaderSubscriber(s3AsyncClient, getObjectRequest));
}

private void logSinglePartWarning(GetObjectRequest getObjectRequest) {
String reason = "";
if (getObjectRequest.range() != null) {
reason = " because getObjectRequest range is included in the request."
+ " range = " + getObjectRequest.range();
} else if (getObjectRequest.partNumber() != null) {
reason = " because getObjectRequest part number is included in the request."
+ " part number = " + getObjectRequest.partNumber();
}
String finalReason = reason;
log.debug(() -> "Using single part download" + finalReason);
}
}

0 comments on commit 5914453

Please sign in to comment.