Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable setting multipart to true/false by defining it as a custom client context parameter #4903

Merged
merged 8 commits into from
Feb 12, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,9 @@
* // Wait for the transfer to complete
* CompletedCopy completedCopy = copy.completionFuture().join();
* }
* <b> Warns the user if a multi-part operation disabled client is provided for TransferManager to use
* If no client is provided, a multi-part enabled async client is instantiated and used.
* In the event of a multi-part disabled defaultS3AsyncClient being used, a warning is logged </b>
*/
@SdkPublicApi
@ThreadSafe
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,13 @@ public static S3TransferManager createTransferManager(DefaultBuilder tmBuilder)
return new CrtS3TransferManager(transferConfiguration, s3AsyncClient, isDefaultS3AsyncClient);
}

if (s3AsyncClient.getClass().getName().equals("software.amazon.awssdk.services.s3.DefaultS3AsyncClient")) {
log.warn(() -> "The provided DefaultS3AsyncClient is not an instance of S3CrtAsyncClient, and thus multipart"
+ " upload/download feature is not enabled and resumable file upload is not supported. To benefit "
+ "from maximum throughput, consider using S3AsyncClient.crtBuilder().build() instead.");
} else {
log.debug(() -> "The provided S3AsyncClient is not an instance of S3CrtAsyncClient, and thus multipart"
+ " upload/download feature may not be enabled and resumable file upload may not be supported.");
if (!s3AsyncClient.getClass().getName().equals("software.amazon.awssdk.services.s3.internal.multipart"
+ ".MultipartS3AsyncClient")) {
log.debug(() -> "The provided S3AsyncClient is neither an instance of S3CrtAsyncClient or MultipartS3AsyncClient, "
+ "and thus multipart upload/download feature may not be enabled and resumable file upload may not "
+ "be supported. To benefit from maximum throughput, consider using "
+ "S3AsyncClient.crtBuilder().build() or "
+ "S3AsyncClient.builder().multipartEnabled(true).build() instead");
}

return new GenericS3TransferManager(transferConfiguration, s3AsyncClient, isDefaultS3AsyncClient);
Expand All @@ -68,7 +68,7 @@ private static Supplier<S3AsyncClient> defaultS3AsyncClient() {
if (crtInClasspath()) {
return S3AsyncClient::crtCreate;
}
return S3AsyncClient::create;
return () -> S3AsyncClient.builder().multipartEnabled(true).build();
anirudh9391 marked this conversation as resolved.
Show resolved Hide resolved
}

private static boolean crtInClasspath() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@

class TransferManagerLoggingTest {

private static final String EXPECTED_DEBUG_MESSAGE = "The provided S3AsyncClient is neither an instance of S3CrtAsyncClient or MultipartS3AsyncClient, "
+ "and thus multipart upload/download feature may not be enabled and resumable file upload may not "
+ "be supported. To benefit from maximum throughput, consider using "
+ "S3AsyncClient.crtBuilder().build() or "
+ "S3AsyncClient.builder().multipartEnabled(true).build() instead";

@Test
void transferManager_withCrtClient_shouldNotLogWarnMessages() {

Expand All @@ -47,21 +53,75 @@ void transferManager_withCrtClient_shouldNotLogWarnMessages() {
}

@Test
void transferManager_withJavaClient_shouldLogWarnMessage() {
void transferManager_withJavaClientMultiPartNotSet_shouldLogDebugMessage() {


try (S3AsyncClient s3Crt = S3AsyncClient.builder()
.region(Region.US_WEST_2)
.credentialsProvider(() -> AwsBasicCredentials.create("foo", "bar"))
.build();
LogCaptor logCaptor = LogCaptor.create(Level.WARN);
LogCaptor logCaptor = LogCaptor.create(Level.DEBUG);
S3TransferManager tm = S3TransferManager.builder().s3Client(s3Crt).build()) {
List<LogEvent> events = logCaptor.loggedEvents();
assertLogged(events, Level.DEBUG, EXPECTED_DEBUG_MESSAGE);
}
}

@Test
void transferManager_withJavaClientMultiPartEqualsFalse_shouldLogDebugMessage() {


try (S3AsyncClient s3Crt = S3AsyncClient.builder()
.region(Region.US_WEST_2)
.credentialsProvider(() -> AwsBasicCredentials.create("foo", "bar"))
.multipartEnabled(false)
.build();
LogCaptor logCaptor = LogCaptor.create(Level.DEBUG);
S3TransferManager tm = S3TransferManager.builder().s3Client(s3Crt).build()) {
List<LogEvent> events = logCaptor.loggedEvents();
assertLogged(events, Level.DEBUG, EXPECTED_DEBUG_MESSAGE);
}
}

@Test
void transferManager_withDefaultClient_shouldNotLogDebugMessage() {


try (LogCaptor logCaptor = LogCaptor.create(Level.DEBUG);
S3TransferManager tm = S3TransferManager.builder().build()) {
List<LogEvent> events = logCaptor.loggedEvents();
assertThat(events).isEmpty();
}
}

@Test
void transferManager_withMultiPartEnabledJavaClient_shouldNotLogDebugMessage() {

try (S3AsyncClient s3Crt = S3AsyncClient.builder()
.region(Region.US_WEST_2)
.credentialsProvider(() -> AwsBasicCredentials.create("foo", "bar"))
.multipartEnabled(true)
.build();
LogCaptor logCaptor = LogCaptor.create(Level.DEBUG);
S3TransferManager tm = S3TransferManager.builder().s3Client(s3Crt).build()) {
List<LogEvent> events = logCaptor.loggedEvents();
assertLogged(events, Level.WARN, "The provided DefaultS3AsyncClient is not an instance of S3CrtAsyncClient, and "
+ "thus multipart upload/download feature is not enabled and resumable file upload"
+ " is "
+ "not supported. To benefit from maximum throughput, consider using "
+ "S3AsyncClient.crtBuilder().build() instead.");
assertThat(events).isEmpty();
}
}

@Test
void transferManager_withMultiPartEnabledAndCrossRegionEnabledJavaClient_shouldNotLogDebugMessage() {

try (S3AsyncClient s3Crt = S3AsyncClient.builder()
.region(Region.US_WEST_2)
.credentialsProvider(() -> AwsBasicCredentials.create("foo", "bar"))
.multipartEnabled(true)
.crossRegionAccessEnabled(true)
.build();
LogCaptor logCaptor = LogCaptor.create(Level.DEBUG);
S3TransferManager tm = S3TransferManager.builder().s3Client(s3Crt).build()) {
List<LogEvent> events = logCaptor.loggedEvents();
assertThat(events).isEmpty();
}
}

Expand Down
Loading