Skip to content

Commit

Permalink
Fix ChecksumIntegrationTest
Browse files Browse the repository at this point in the history 
- Some tests specificy a part number, but CRT may do a range get under the hood.
  S3 will throw an error if both a range and part number are specified. This is
an issue that needs to be fixed in CRT, but part number is not required in this
test, so removing it.

 - Rename test file to CrtCheckIntegrationTest so it gets added to CRT test
   suite
  • Loading branch information
@dagnir
dagnir committed Dec 21, 2023
1 parent 3568ec6 commit 0d06311
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 22 deletions.
45 changes: 27 additions & 18 deletions ...vices/s3/crt/ChecksumIntegrationTest.java → ...es/s3/crt/CrtChecksumIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,30 +22,36 @@
import java.nio.file.Files;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.async.AsyncRequestBody;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.crt.CrtResource;
import software.amazon.awssdk.core.checksums.Algorithm;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3IntegrationTestBase;
import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
import software.amazon.awssdk.services.s3.model.ChecksumMode;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectTaggingResponse;
import software.amazon.awssdk.services.s3.model.Tag;
import software.amazon.awssdk.services.s3.model.Tagging;
import software.amazon.awssdk.services.s3.utils.ChecksumUtils;
import software.amazon.awssdk.testutils.RandomTempFile;
import software.amazon.awssdk.testutils.service.AwsTestBase;

public class ChecksumIntegrationTest extends S3IntegrationTestBase {
private static final String TEST_BUCKET = temporaryBucketName(ChecksumIntegrationTest.class);
public class CrtChecksumIntegrationTest extends S3IntegrationTestBase {
private static final String TEST_BUCKET = temporaryBucketName(CrtChecksumIntegrationTest.class);
private static final String TEST_KEY = "10mib_file.dat";
private static final int OBJ_SIZE = 10 * 1024 * 1024;

private static RandomTempFile testFile;

private static String testFileSha1;
private static String testFileCrc32;

private static S3AsyncClient s3Crt;

@BeforeAll
Expand All @@ -54,10 +60,15 @@ public static void setup() throws Exception {
S3IntegrationTestBase.createBucket(TEST_BUCKET);

testFile = new RandomTempFile(TEST_KEY, OBJ_SIZE);
testFileSha1 = ChecksumUtils.calculatedChecksum(testFile.toPath(), Algorithm.SHA1);
testFileCrc32 = ChecksumUtils.calculatedChecksum(testFile.toPath(), Algorithm.CRC32);

s3Crt = S3CrtAsyncClient.builder()
.credentialsProvider(AwsTestBase.CREDENTIALS_PROVIDER_CHAIN)
.region(S3IntegrationTestBase.DEFAULT_REGION)
// make sure we don't do a multipart upload, it will mess with validation against the precomputed
// checksums above
.thresholdInBytes(2L * OBJ_SIZE)
.build();
}

Expand All @@ -72,37 +83,35 @@ public static void teardown() throws IOException {
@Test
void noChecksumCustomization_crc32ShouldBeUsed() {
AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath());
PutObjectResponse putObjectResponse =
s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join();
assertThat(putObjectResponse).isNotNull();
s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join();

ResponseBytes<GetObjectResponse> getObjectResponseResponseBytes =
s3Crt.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY).partNumber(1), AsyncResponseTransformer.toBytes()).join();
s3Crt.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), AsyncResponseTransformer.toBytes()).join();
String getObjectChecksum = getObjectResponseResponseBytes.response().checksumCRC32();
assertThat(getObjectChecksum).isNotNull();
assertThat(getObjectChecksum).isEqualTo(testFileCrc32);
}

@Test
void putObject_checksumProvidedInRequest_shouldTakePrecendence() {
AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath());
PutObjectResponse putObjectResponse =
s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY).checksumAlgorithm(ChecksumAlgorithm.SHA1), body).join();
assertThat(putObjectResponse).isNotNull();
s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY).checksumAlgorithm(ChecksumAlgorithm.SHA1), body).join();

ResponseBytes<GetObjectResponse> getObjectResponseResponseBytes =
s3Crt.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY).partNumber(1), AsyncResponseTransformer.toBytes()).join();
s3Crt.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), AsyncResponseTransformer.toBytes()).join();
String getObjectChecksum = getObjectResponseResponseBytes.response().checksumSHA1();
assertThat(getObjectChecksum).isNotNull();
// TODO: We compare against the precomputed SHA-1, but this shouldn't be necessary since the PutObjectResponse should
// have this info. This looks like an issue with CRT.
assertThat(getObjectChecksum).isEqualTo(testFileSha1);
}

@Test
void checksumDisabled_shouldNotPerformChecksumValidationByDefault() {

try (S3AsyncClient s3Crt = S3CrtAsyncClient.builder()
.credentialsProvider(AwsTestBase.CREDENTIALS_PROVIDER_CHAIN)
.region(S3IntegrationTestBase.DEFAULT_REGION)
.checksumValidationEnabled(Boolean.FALSE)
.build()) {
.credentialsProvider(AwsTestBase.CREDENTIALS_PROVIDER_CHAIN)
.region(S3IntegrationTestBase.DEFAULT_REGION)
.checksumValidationEnabled(Boolean.FALSE)
.build()) {
AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath());
PutObjectResponse putObjectResponse =
s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join();
Expand Down
20 changes: 16 additions & 4 deletions services/s3/src/test/java/software/amazon/awssdk/services/s3/utils/ChecksumUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,15 @@

package software.amazon.awssdk.services.s3.utils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import software.amazon.awssdk.core.checksums.Algorithm;
Expand Down Expand Up @@ -88,6 +86,20 @@ public static String calculatedChecksum(String contentString, Algorithm algorith
return BinaryUtils.toBase64(sdkChecksum.getChecksumBytes());
}

public static String calculatedChecksum(Path path, Algorithm algorithm) {
SdkChecksum sdkChecksum = SdkChecksum.forAlgorithm(algorithm);
try (InputStream is = Files.newInputStream(path)) {
byte[] buffer = new byte[4096];
int read;
while ((read = is.read(buffer)) != -1) {
sdkChecksum.update(buffer, 0, read);
}
return BinaryUtils.toBase64(sdkChecksum.getChecksumBytes());
} catch (IOException e) {
throw new RuntimeException(e);
}
}

public static String createDataOfSize(int dataSize, char contentCharacter) {
return IntStream.range(0, dataSize).mapToObj(i -> String.valueOf(contentCharacter)).collect(Collectors.joining());
}
Expand Down

0 comments on commit 0d06311

Please sign in to comment.